diff --git a/Makefile b/Makefile index f7764d23d..c4fac8bf5 100644 --- a/Makefile +++ b/Makefile @@ -78,17 +78,20 @@ ifndef IMAGE $(error IMAGE is required. Usage: make build-system-image IMAGE=lb) endif @if [ -f scripts/images/$(IMAGE).conf ]; then \ - ./scripts/build-system-image.sh scripts/images/$(IMAGE).conf; \ + ./scripts/build-system-image.sh scripts/images/$(IMAGE).conf $(if $(IMPORT),--import); \ elif [ -f scripts/images/$(IMAGE)/manifest.conf ]; then \ - ./scripts/build-system-image.sh scripts/images/$(IMAGE)/manifest.conf; \ + ./scripts/build-system-image.sh scripts/images/$(IMAGE)/manifest.conf $(if $(IMPORT),--import); \ else \ echo "ERROR: no manifest at scripts/images/$(IMAGE).conf or scripts/images/$(IMAGE)/manifest.conf"; \ exit 1; \ fi -build-eks-server-image: ## Build the eks-server AMI (K3s control-plane variant) +build-eks-server-image: ## Build the eks-server AMI (K3s control-plane variant; IMPORT=1 to register it) $(MAKE) build-system-image IMAGE=eks-server +import-eks-server-image: ## Build + register the eks-server AMI (requires a running cluster) + $(MAKE) build-system-image IMAGE=eks-server IMPORT=1 + build-eks-agent-image: ## Build the eks-agent AMI (K3s worker variant) $(MAKE) build-system-image IMAGE=eks-agent @@ -338,9 +341,22 @@ ansible-dev-version: ansible-dev-vpc: cd scripts/ansible && ansible-playbook playbooks/dev-vpc.yml $(ANSIBLE_EXTRA) -.PHONY: build build-ui build-installer build-lb-agent build-system-image build-eks-server-image build-eks-agent-image build-microvm-image install-microvm go_build go_run preflight test test-cover test-race diff-coverage bench run test-actions test-harness manifest-check diff-coverage bench run \ +# Multi-node cluster bootstrap against a tofu env (env1, env2, bryn, etc). +# Inventory is generated from scripts/tofu-cluster/envs/.tfvars. +# make ansible-cluster-bootstrap ENV=env1 POOL=192.168.1.150-192.168.1.159 +ENV ?= +POOL ?= +ansible-cluster-bootstrap: + @test -n "$(ENV)" || { echo "ENV= required (e.g. env1, env2)"; exit 64; } + cd scripts/ansible && CLUSTER_ENV=$(ENV) ansible-playbook -i inventory/tofu.py playbooks/cluster-bootstrap.yml \ + -e cluster_env=$(ENV) \ + $(if $(POOL),-e cluster_external_pool=$(POOL),) \ + $(_ANSIBLE_EXTRA) + +.PHONY: build build-ui build-installer build-lb-agent build-system-image build-eks-server-image import-eks-server-image build-eks-agent-image build-microvm-image install-microvm go_build go_run preflight test test-cover test-race diff-coverage bench run test-actions test-harness manifest-check diff-coverage bench run \ deploy reinstall clean \ install-system install-go install-aws quickinstall \ lint fix govulncheck \ distro distro-amd64 distro-arm64 distro-clean \ - ansible-dev-preflight ansible-dev-teardown ansible-dev-install ansible-dev-reset ansible-dev-deploy ansible-dev-status ansible-dev-logs ansible-dev-snapshot ansible-dev-restore ansible-dev-version ansible-dev-vpc + ansible-dev-preflight ansible-dev-teardown ansible-dev-install ansible-dev-reset ansible-dev-deploy ansible-dev-status ansible-dev-logs ansible-dev-snapshot ansible-dev-restore ansible-dev-version ansible-dev-vpc \ + ansible-cluster-bootstrap diff --git a/cmd/spinifex/cmd/admin.go b/cmd/spinifex/cmd/admin.go index 44f201de7..188700d33 100644 --- a/cmd/spinifex/cmd/admin.go +++ b/cmd/spinifex/cmd/admin.go @@ -325,6 +325,7 @@ func init() { imagesImportCmd.Flags().String("tmp-dir", os.TempDir(), "Temporary directory for image import processing") imagesImportCmd.Flags().String("name", "", "Import specified image by name") + imagesImportCmd.Flags().String("ami-name", "", "Override the registered AMI name (DescribeImages name). Defaults to ami-{distro}-{version}-{arch}. Use for locally-built appliances (e.g. --ami-name spinifex-eks-server).") imagesImportCmd.Flags().String("file", "", "Import file from specified path (raw, qcow2, compressed)") imagesImportCmd.Flags().String("distro", "", "Specified distro name (e.g debian)") imagesImportCmd.Flags().String("version", "", "Specified distro version (e.g 12)") @@ -380,6 +381,7 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) { // and the file is used directly (no download). When only --file is set, // metadata comes from flags and no catalog tags are applied. imageName, _ := cmd.Flags().GetString("name") + amiNameOverride, _ := cmd.Flags().GetString("ami-name") localFile, _ := cmd.Flags().GetString("file") if imageName == "" && localFile == "" { @@ -535,6 +537,9 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) { // Calculate the size manifest.AMIMetadata.Name = fmt.Sprintf("ami-%s-%s-%s", image.Distro, image.Version, image.Arch) + if amiNameOverride != "" { + manifest.AMIMetadata.Name = amiNameOverride + } volumeId := utils.GenerateResourceID("ami") manifest.AMIMetadata.ImageID = volumeId diff --git a/docs/COMMANDS.md b/docs/COMMANDS.md index d01dae7bf..e07b01ff2 100644 --- a/docs/COMMANDS.md +++ b/docs/COMMANDS.md @@ -89,7 +89,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t | Command | Flags | Prerequisites | Basic Logic | Test Cases | Status | |---------|-------|---------------|-------------|------------|--------| -| `spx admin images import` | `--name`, `--file`, `--distro`, `--version`, `--arch`, `--platform`, `--boot-mode` (bios/uefi/uefi-preferred), `--tag`, `--force`, `--skip-verify` | Cluster must be running; either `--name` (catalog download) or `--file` (operator-supplied media). `--file` mode additionally requires `--distro`, `--version`, `--arch`, `--boot-mode`. | Catalog imports (`--name`) download the image, fetch the catalog `Checksum` URL, verify the SHA-256/SHA-512 digest, and inherit `BootMode` from the catalog entry. `--boot-mode` overrides the catalog value when set. Mismatch fails closed; the cached file is left on disk and `--force` re-downloads. `--file` imports skip checksum verification (operator-supplied media is outside Spinifex's trust boundary, the skip is logged at INFO for audit) and require an explicit `--boot-mode` because there is no catalog metadata to inherit from. `--skip-verify` bypasses verification for catalog imports and emits a WARN slog + stderr notice; use only for debugging or when upstream mirrors are confirmed-broken. | 1. Import valid catalog image (verifies checksum)
2. Tampered cache hit fails with `ErrChecksumMismatch`
3. `--force` recovers after a mismatch
4. `--file` import skips verification
5. `--skip-verify` bypasses checksum with WARN
6. `--file` without `--boot-mode` rejected
7. `--boot-mode=invalid` rejected | **DONE** | +| `spx admin images import` | `--name`, `--ami-name`, `--file`, `--distro`, `--version`, `--arch`, `--platform`, `--boot-mode` (bios/uefi/uefi-preferred), `--tag`, `--force`, `--skip-verify` | Cluster must be running; either `--name` (catalog download) or `--file` (operator-supplied media). `--file` mode additionally requires `--distro`, `--version`, `--arch`, `--boot-mode`. | Catalog imports (`--name`) download the image, fetch the catalog `Checksum` URL, verify the SHA-256/SHA-512 digest, and inherit `BootMode` from the catalog entry. `--boot-mode` overrides the catalog value when set. Mismatch fails closed; the cached file is left on disk and `--force` re-downloads. `--file` imports skip checksum verification (operator-supplied media is outside Spinifex's trust boundary, the skip is logged at INFO for audit) and require an explicit `--boot-mode` because there is no catalog metadata to inherit from. `--ami-name` overrides the registered AMI name (the `DescribeImages` name), which otherwise defaults to `ami-{distro}-{version}-{arch}`; use it for locally-built appliance images (e.g. `--ami-name spinifex-eks-server`) not in the catalog. `--skip-verify` bypasses verification for catalog imports and emits a WARN slog + stderr notice; use only for debugging or when upstream mirrors are confirmed-broken. | 1. Import valid catalog image (verifies checksum)
2. Tampered cache hit fails with `ErrChecksumMismatch`
3. `--force` recovers after a mismatch
4. `--file` import skips verification
5. `--skip-verify` bypasses checksum with WARN
6. `--file` without `--boot-mode` rejected
7. `--boot-mode=invalid` rejected | **DONE** | | `spx admin images list` | — | None | Lists available OS images that can be imported or downloaded | 1. List available images | **DONE** | | `spx admin images remove` | `--image-id` (required), `--force`, `--yes` | Cluster must be running; predastore reachable. Targets AMIs imported via `spx admin images import` (non-account `ImageOwnerAlias`, e.g. `"system"`). | Loads `ami-/config.json`, walks transitive dependents — copied snapshots whose `VolumeID == imageID`, volumes whose `SnapshotID` references the internal `snap-ami-` or any derived snap, and account AMIs created via `CopyImage` whose `SnapshotID` is a derived snap — then prompts (skipped with `--yes`) before deleting `ami-/config.json` (the DescribeImages barrier) followed by the rest of `ami-/` and `snap-ami-/`. Account-owned AMIs are refused with a hint pointing at `aws ec2 deregister-image` + `aws ec2 delete-snapshot`. `--force` bypasses the dependency, ownership and config-corrupt checks for salvage of orphaned blocks. | 1. Happy path: system AMI, no deps → deleted
2. Account-owned → refused with AWS-flow hint
3. Missing/corrupt config → `InvalidAMIID.NotFound` (salvageable with `--force`)
4. Dependent volume (direct or via copied snap) → refused
5. Dependent account AMI → refused
6. `--force` overrides dependents
7. Idempotent re-run after partial delete | **DONE** | diff --git a/scripts/build-system-image.sh b/scripts/build-system-image.sh index eaff2b855..2ec0501da 100755 --- a/scripts/build-system-image.sh +++ b/scripts/build-system-image.sh @@ -192,6 +192,9 @@ if [[ -f "$OUTPUT_RAW" ]] && [[ $(( $(date +%s) - $(stat -c %Y "$OUTPUT_RAW") )) echo "Importing as AMI..." rm -f "$OUTPUT_IMAGE" IMPORT_ARGS=(--file "$OUTPUT_RAW" --distro "${DISTRO}" --version "${DISTRO_VERSION}" --arch x86_64 --boot-mode "${BOOT_MODE}") + if [[ -n "${AMI_NAME:-}" ]]; then + IMPORT_ARGS+=(--ami-name "$AMI_NAME") + fi if [[ -n "${SYSTEM_TAG:-}" ]]; then IMPORT_ARGS+=(--tag "$SYSTEM_TAG") fi @@ -453,7 +456,7 @@ if [[ "$DO_IMPORT" == true ]]; then rm -f "$OUTPUT_IMAGE" IMPORT_ARGS=(--file "$OUTPUT_RAW" --distro "${DISTRO}" --version "${DISTRO_VERSION}" --arch x86_64 --boot-mode "${BOOT_MODE}") if [[ -n "${AMI_NAME:-}" ]]; then - IMPORT_ARGS+=(--name "$AMI_NAME") + IMPORT_ARGS+=(--ami-name "$AMI_NAME") fi if [[ -n "${SYSTEM_TAG:-}" ]]; then IMPORT_ARGS+=(--tag "$SYSTEM_TAG") @@ -465,7 +468,7 @@ else echo " --file $OUTPUT_RAW \\" NAME_HINT="" if [[ -n "${AMI_NAME:-}" ]]; then - NAME_HINT=" \\\n --name ${AMI_NAME}" + NAME_HINT=" \\\n --ami-name ${AMI_NAME}" fi if [[ -n "${SYSTEM_TAG:-}" ]]; then echo -e " --distro ${DISTRO} --version ${DISTRO_VERSION} --arch x86_64 --boot-mode ${BOOT_MODE}${NAME_HINT} \\" diff --git a/scripts/dev-install.sh b/scripts/dev-install.sh index f3f221534..7044d105e 100755 --- a/scripts/dev-install.sh +++ b/scripts/dev-install.sh @@ -90,6 +90,17 @@ fi echo "=== Building and installing microVM artifacts ===" cd "$PROJECT_ROOT" && make build-lb-agent install-microvm +# EKS control-plane AMI. Heavy (full Alpine + K3s build, needs sudo + network), +# so it is opt-in: set DEV_INSTALL_EKS=1 to build + register it. Without it, +# `aws eks create-cluster` fails AMI resolution until the operator runs +# `make import-eks-server-image` manually. +if [ "${DEV_INSTALL_EKS:-0}" = "1" ]; then + echo "=== Building + importing eks-server AMI ===" + cd "$PROJECT_ROOT" && make import-eks-server-image +else + echo "=== Skipping eks-server AMI (set DEV_INSTALL_EKS=1 to enable, or run 'make import-eks-server-image') ===" +fi + echo "=== Done ===" echo "Services: sudo systemctl status spinifex.target" echo "Logs: journalctl -u 'spinifex-*' -f" diff --git a/scripts/images/eks-server/k3s-first-boot.initd b/scripts/images/eks-server/k3s-first-boot.initd index ab7029a06..2e6fa6da7 100644 --- a/scripts/images/eks-server/k3s-first-boot.initd +++ b/scripts/images/eks-server/k3s-first-boot.initd @@ -4,11 +4,18 @@ # once the sentinel file is gone. description="K3s first-boot publisher (node-token + admin kubeconfig)" +# Backgrounded: the publisher waits up to 5min for K3s /healthz before it can +# read the node-token + kubeconfig, so it must not block the default runlevel. +# command_background is also required for output_log/error_log — start-stop-daemon +# rejects --stdout/--stderr without --background ("Failed to start k3s-first-boot"). command="/usr/local/sbin/k3s-first-boot" -command_background="no" +command_background="yes" pidfile="/run/k3s-first-boot.pid" -output_log="/var/log/k3s-first-boot.log" -error_log="/var/log/k3s-first-boot.log" +# Route to the serial console so the bootstrap publisher's progress (healthz +# wait, token/kubeconfig publish, errors) is visible in the orchestrator's +# host-side console capture — the VM has no operator login. Low volume. +output_log="/dev/console" +error_log="/dev/console" depend() { need k3s diff --git a/scripts/images/eks-server/k3s-first-boot.sh b/scripts/images/eks-server/k3s-first-boot.sh index bd871fefc..991ac5060 100644 --- a/scripts/images/eks-server/k3s-first-boot.sh +++ b/scripts/images/eks-server/k3s-first-boot.sh @@ -2,15 +2,16 @@ set -eu # k3s-first-boot — runs once after the K3s server reaches a healthy state. -# Reads the bootstrap node-token and admin kubeconfig that K3s writes during -# `--cluster-init`, rewrites the kubeconfig server address to the cluster's +# Reads the bootstrap node-token and admin kubeconfig that K3s writes at +# server startup, rewrites the kubeconfig server address to the cluster's # NLB endpoint (so workers and external kubectl can use it), and publishes # both as one-shot NATS messages for the spinifex cluster reconciler to # consume into KV. # # Required env (from cloud-init user-data /etc/spinifex-eks/first-boot.env): # SPINIFEX_NATS_URL nats://... -# SPINIFEX_NATS_CREDS_FILE /etc/spinifex-eks/nats.creds +# SPINIFEX_NATS_TOKEN shared NATS auth token +# SPINIFEX_NATS_CA /etc/spinifex-eks/nats-ca.pem (TLS CA bundle) # EKS_ACCOUNT_ID # EKS_CLUSTER_NAME # EKS_NLB_ENDPOINT https://{cluster}.{accountID}.eks.{region}.{suffix} @@ -43,23 +44,51 @@ for v in SPINIFEX_NATS_URL EKS_ACCOUNT_ID EKS_CLUSTER_NAME EKS_NLB_ENDPOINT; do [ -n "${val}" ] || die "env ${v} not set" done -# 1. Wait for K3s /healthz. K3s self-signs initially; --insecure-skip is fine -# here, we are on loopback. -log "waiting for K3s /healthz on 127.0.0.1:6443" -i=0 -while [ "${i}" -lt 300 ]; do - if curl -sk --max-time 2 https://127.0.0.1:6443/healthz | grep -q '^ok$'; then - log "K3s healthz ok after ${i}s" - break - fi - i=$((i + 2)) - sleep 2 -done -[ "${i}" -lt 300 ] || die "K3s did not become healthy within 5 minutes" - -# 2. Read node-token + admin kubeconfig. +# Paths K3s writes during server startup: the bootstrap node-token and the +# admin kubeconfig (used both to gate readiness and to publish downstream). TOKEN_FILE=/var/lib/rancher/k3s/server/node-token KUBECONFIG_FILE=/etc/rancher/k3s/k3s.yaml + +# 1. Wait for the apiserver to be ready to serve, gating on /readyz. K3s runs +# the apiserver with anonymous-auth=false, so an anonymous probe returns 401, +# never "ok" — the probe must be authenticated. Use the node's admin +# kubeconfig (kubectl get --raw), which k3s writes early in startup. kubectl +# exits 0 only when /readyz returns 200; on a failing sub-check it exits +# non-zero, so the body "ok" check plus exit status gates correctly. Every +# 30s the failing /readyz checks are logged so a stuck boot is diagnosable +# from the captured serial console. +log "waiting for K3s apiserver readiness (/readyz) on 127.0.0.1:6443" +i=0 +ready=0 +while [ "${i}" -lt 300 ]; do + if [ -r "${KUBECONFIG_FILE}" ] && \ + [ "$(kubectl --kubeconfig "${KUBECONFIG_FILE}" get --raw='/readyz' 2>/dev/null)" = "ok" ]; then + log "K3s apiserver ready after ${i}s" + ready=1 + break + fi + if [ $((i % 30)) -eq 0 ]; then + log "apiserver not ready after ${i}s:" + if [ -r "${KUBECONFIG_FILE}" ]; then + kubectl --kubeconfig "${KUBECONFIG_FILE}" get --raw='/readyz?verbose' 2>&1 \ + | grep -E '^\[-\]|failed$' | head -20 | while IFS= read -r l; do log " ${l}"; done + else + log " (admin kubeconfig ${KUBECONFIG_FILE} not written yet)" + fi + fi + i=$((i + 5)) + sleep 5 +done +if [ "${ready}" -ne 1 ]; then + log "K3s apiserver not ready within 5 minutes; last /readyz body:" + kubectl --kubeconfig "${KUBECONFIG_FILE}" get --raw='/readyz?verbose' 2>&1 \ + | head -40 | while IFS= read -r l; do log " ${l}"; done + log "last 40 lines of /var/log/k3s.log follow:" + tail -n 40 /var/log/k3s.log 2>/dev/null || log "(no /var/log/k3s.log)" + die "K3s apiserver not ready within 5 minutes" +fi + +# 2. Read the four bootstrap artifacts K3s wrote at server startup. [ -r "${TOKEN_FILE}" ] || die "${TOKEN_FILE} unreadable" [ -r "${KUBECONFIG_FILE}" ] || die "${KUBECONFIG_FILE} unreadable" @@ -67,23 +96,38 @@ NODE_TOKEN=$(cat "${TOKEN_FILE}") # K3s ships kubeconfig with server: https://127.0.0.1:6443 — rewrite to the # NLB endpoint so it works from outside the control plane VM. KUBECONFIG_REWRITTEN=$(sed "s|server: https://127\.0\.0\.1:6443|server: ${EKS_NLB_ENDPOINT}|" "${KUBECONFIG_FILE}") +# The cluster CA the daemon records (base64 PEM) is exactly the +# certificate-authority-data the admin kubeconfig already embeds. +CA_B64=$(awk '/certificate-authority-data:/ {print $2; exit}' "${KUBECONFIG_FILE}") +[ -n "${CA_B64}" ] || die "no certificate-authority-data in ${KUBECONFIG_FILE}" +# The OIDC JWKS the apiserver serves from the signing key cloud-init seeded. The +# daemon cross-checks its kid/kty against the controller-generated keypair. +JWKS=$(kubectl --kubeconfig "${KUBECONFIG_FILE}" get --raw='/openid/v1/jwks' 2>/dev/null) +[ -n "${JWKS}" ] || die "apiserver returned empty /openid/v1/jwks" -# 3. Publish one-shot NATS messages. Subjects per eks-v1.md Q14 + Q15. -TOKEN_SUBJ="eks.bus.${EKS_ACCOUNT_ID}.${EKS_CLUSTER_NAME}.k3s-bootstrap-token" -KUBE_SUBJ="eks.bus.${EKS_ACCOUNT_ID}.${EKS_CLUSTER_NAME}.k3s-admin-kubeconfig" - +# 3. Publish the four one-shot bootstrap messages. Each is a BootstrapEnvelope +# JSON document (handlers/eks/nats_bootstrap.go); jq encodes the values so +# embedded newlines/quotes in the kubeconfig and JWKS stay valid JSON. NATS_ARGS="-s ${SPINIFEX_NATS_URL}" -if [ -n "${SPINIFEX_NATS_CREDS_FILE:-}" ] && [ -f "${SPINIFEX_NATS_CREDS_FILE}" ]; then - NATS_ARGS="${NATS_ARGS} --creds ${SPINIFEX_NATS_CREDS_FILE}" +if [ -n "${SPINIFEX_NATS_TOKEN:-}" ]; then + NATS_ARGS="${NATS_ARGS} --token ${SPINIFEX_NATS_TOKEN}" +fi +if [ -n "${SPINIFEX_NATS_CA:-}" ] && [ -f "${SPINIFEX_NATS_CA}" ]; then + NATS_ARGS="${NATS_ARGS} --tlsca ${SPINIFEX_NATS_CA}" fi -log "publishing node-token to ${TOKEN_SUBJ}" -# shellcheck disable=SC2086 -printf '%s' "${NODE_TOKEN}" | nats ${NATS_ARGS} pub "${TOKEN_SUBJ}" --stdin +# publish_envelope : reads the envelope JSON from stdin. +publish_envelope() { + subj="eks.bus.${EKS_ACCOUNT_ID}.${EKS_CLUSTER_NAME}.$1" + log "publishing $1 to ${subj}" + # shellcheck disable=SC2086 + nats ${NATS_ARGS} pub "${subj}" --force-stdin +} -log "publishing admin kubeconfig to ${KUBE_SUBJ}" -# shellcheck disable=SC2086 -printf '%s' "${KUBECONFIG_REWRITTEN}" | nats ${NATS_ARGS} pub "${KUBE_SUBJ}" --stdin +jq -n --arg t "${NODE_TOKEN}" '{token: $t}' | publish_envelope k3s-bootstrap-token +jq -n --arg k "${KUBECONFIG_REWRITTEN}" '{adminKubeconfig: $k}' | publish_envelope k3s-admin-kubeconfig +jq -n --arg j "${JWKS}" '{jwks: $j}' | publish_envelope k3s-oidc-jwks +jq -n --arg c "${CA_B64}" '{certificateAuthority: $c}' | publish_envelope k3s-ca # 4. Self-disable. Remove sentinel, pull from runlevel. rm -f "${SENTINEL}" diff --git a/scripts/images/eks-server/k3s.initd b/scripts/images/eks-server/k3s.initd index ca69f2269..04ed37d3b 100644 --- a/scripts/images/eks-server/k3s.initd +++ b/scripts/images/eks-server/k3s.initd @@ -1,9 +1,9 @@ #!/sbin/openrc-run -# K3s server control plane. v1 = single-node embedded etcd (eks-v1.md Q6). -# --cluster-init bootstraps a new etcd quorum on first start; subsequent -# starts pick up the existing data dir at /var/lib/rancher/k3s. v2 HA grows -# the cluster by launching additional eks-server VMs with --server pointing -# at the existing NLB endpoint (Q17) — no AMI change required. +# K3s server control plane. v1 = single-node embedded SQLite (kine) datastore; +# the per-cluster config.yaml omits cluster-init so k3s uses SQLite, not etcd +# (etcd's per-commit fsync stalls on the viperblock root volume). Data lives at +# /var/lib/rancher/k3s; subsequent starts reuse it. Multi-server HA needs etcd +# and is deferred. description="K3s Kubernetes server (Mulga EKS control plane)" command="/usr/local/bin/k3s" @@ -13,8 +13,8 @@ pidfile="/run/k3s.pid" output_log="/var/log/k3s.log" error_log="/var/log/k3s.log" -# Generous start timeout — K3s bootstraps containerd, runs etcd cluster-init, -# pulls system images. First boot can take 2-3 minutes on cold cache. +# Generous start timeout — K3s bootstraps containerd, initialises the SQLite +# datastore, pulls system images. First boot can take 2-3 minutes on cold cache. start_stop_daemon_args="--wait 5000" depend() { diff --git a/scripts/images/eks-server/manifest.conf b/scripts/images/eks-server/manifest.conf index aca0fab02..1dcaadce0 100644 --- a/scripts/images/eks-server/manifest.conf +++ b/scripts/images/eks-server/manifest.conf @@ -3,14 +3,15 @@ IMAGE_DESCRIPTION="EKS K3s server VM — Alpine + K3s v1.32.5+k3s1 + eks-token-w DISTRO="alpine" ALPINE_VERSION="3.21.7" -# 4G fits Alpine + K3s server binary (~80MB) + etcd snapshot working set. -# K3s embedded etcd lives on the data volume the orchestrator attaches at -# LaunchSystemInstance time; root disk only carries the image + logs. +# 4G fits Alpine + K3s server binary (~80MB) + SQLite datastore working set. +# v1 single-node uses the embedded SQLite (kine) datastore under +# /var/lib/rancher/k3s on the root disk, which also carries the image + logs. IMAGE_SIZE="4G" # Packages required at runtime (K3s pulls cgroups + iptables; nats-cli used by -# first-boot publisher + state-report; etcd-client used by snapshot script). -APK_PACKAGES="ca-certificates curl iptables ip6tables conntrack-tools cgroup-tools etcd-ctl openrc" +# first-boot publisher + state-report; jq builds the bootstrap-message JSON +# envelopes; etcd-client used by snapshot script). +APK_PACKAGES="ca-certificates curl jq iptables ip6tables conntrack-tools cgroup-tools etcd-ctl openrc" # k3s binary downloaded by setup.sh (network access inside chroot); SHA256 # verified at install time per spec Q12. diff --git a/scripts/images/eks-server/setup.sh b/scripts/images/eks-server/setup.sh index 5803b0a3b..c2a1a6be5 100644 --- a/scripts/images/eks-server/setup.sh +++ b/scripts/images/eks-server/setup.sh @@ -68,7 +68,7 @@ mkdir -p /etc/rancher/k3s cat > /etc/rancher/k3s/config.yaml.skel <<'EOF' # Populated at first boot by cloud-init user-data via k3s-first-boot.sh. # Fields documented at https://docs.k3s.io/installation/configuration. -cluster-init: true +# No cluster-init: single-node v1 uses the embedded SQLite datastore, not etcd. disable: - traefik - servicelb @@ -82,4 +82,15 @@ touch /var/lib/spinifex-eks/first-boot.pending 2>/dev/null || { touch /var/lib/spinifex-eks/first-boot.pending } +# Bind /dev/console to the serial port so userspace boot output — OpenRC +# service starts, cloud-init, and k3s-first-boot diagnostics — reaches ttyS0, +# which the orchestrator captures to a host-side log. The stock Alpine cloud +# image lists `console=tty0` LAST in default_kernel_opts; Linux makes the last +# console= the controlling /dev/console, so userspace logs to the framebuffer +# and the serial capture sees only kernel dmesg. Reorder so ttyS0 is last in +# both the generator config and the already-rendered extlinux.conf. +sed -i \ + 's|console=ttyS0,115200n8 console=ttyAMA0,115200n8 console=tty0|console=tty0 console=ttyAMA0,115200n8 console=ttyS0,115200n8|' \ + /etc/update-extlinux.conf /boot/extlinux.conf + echo "[eks-server-setup] done" diff --git a/spinifex/awserrors/awserrors.go b/spinifex/awserrors/awserrors.go index 9d982e7f8..4466632b3 100644 --- a/spinifex/awserrors/awserrors.go +++ b/spinifex/awserrors/awserrors.go @@ -1,5 +1,7 @@ package awserrors +import "strings" + type ErrorMessage struct { HTTPCode int Message string @@ -366,6 +368,9 @@ var ( ErrorResourceCountExceeded = "ResourceCountExceeded" ErrorResourceCountLimitExceeded = "ResourceCountLimitExceeded" ErrorResourceLimitExceeded = "ResourceLimitExceeded" + ErrorResourceNotFound = "ResourceNotFound" + ErrorEKSResourceInUse = "ResourceInUseException" + ErrorEKSResourceNotFound = "ResourceNotFoundException" ErrorRetryableError = "RetryableError" ErrorRouteAlreadyExists = "RouteAlreadyExists" ErrorRouteLimitExceeded = "RouteLimitExceeded" @@ -481,6 +486,15 @@ func ValidErrorCode(code string) string { return ErrorServerInternal } +// IsErrorCode reports whether err carries the named AWS error code. Service +// handlers return these codes as plain strings (errors.New(code)), commonly +// wrapped with %w by orchestration callers, so the match is a substring test +// against err.Error() rather than equality. Use it to classify a delete error +// as idempotent (already-gone) or retryable (still-in-use) in teardown paths. +func IsErrorCode(err error, code string) bool { + return err != nil && strings.Contains(err.Error(), code) +} + var ErrorLookup = map[string]ErrorMessage{ ErrorAccountDisabled: {HTTPCode: 400, Message: "The functionality you have requested has been administratively disabled for this account."}, ErrorActiveVpcPeeringConnectionPerVpcLimitExceeded: {HTTPCode: 400, Message: "You've reached the limit on the number of active VPC peering connections you can have for the specified VPC."}, @@ -842,6 +856,9 @@ var ErrorLookup = map[string]ErrorMessage{ ErrorResourceCountExceeded: {HTTPCode: 400, Message: "You have exceeded the number of resources allowed for this request; for example, if you try to launch more instances than AWS allows in a single request. This limit is separate from your individual resource limit. If you get this error, break up your request into smaller requests; for example, if you are launching 15 instances, try launching 5 instances in 3 separate requests."}, ErrorResourceCountLimitExceeded: {HTTPCode: 400, Message: "You have exceeded a resource limit for creating routes."}, ErrorResourceLimitExceeded: {HTTPCode: 400, Message: "You have exceeded an Amazon EC2 resource limit. For example, you might have too many snapshot copies in progress."}, + ErrorResourceNotFound: {HTTPCode: 404, Message: "The specified resource was not found."}, + ErrorEKSResourceInUse: {HTTPCode: 409, Message: "A cluster with this name already exists in this account and Region. Delete the existing cluster, or choose a different name."}, + ErrorEKSResourceNotFound: {HTTPCode: 404, Message: "The specified cluster could not be found. You can view your available clusters with ListClusters. Amazon EKS clusters are Region specific."}, ErrorRetryableError: {HTTPCode: 400, Message: "A request submitted by an AWS service on your behalf could not be completed. The requesting service might automatically retry the request."}, ErrorRouteAlreadyExists: {HTTPCode: 409, Message: "A route for the specified CIDR block already exists in this route table."}, ErrorRouteLimitExceeded: {HTTPCode: 400, Message: "You've reached the limit on the number of routes that you can add to a route table."}, diff --git a/spinifex/daemon/daemon.go b/spinifex/daemon/daemon.go index 6fd535d65..8b4003837 100644 --- a/spinifex/daemon/daemon.go +++ b/spinifex/daemon/daemon.go @@ -1332,11 +1332,14 @@ func (d *Daemon) startCluster() error { } d.eksService, err = initServiceWithRetry("EKS service", func() (*handlers_eks.EKSServiceImpl, error) { - return handlers_eks.NewEKSServiceImplWithNATS(d.config, d.natsConn) + return handlers_eks.NewEKSServiceImpl(d.buildEKSServiceDeps()) }) if err != nil { return fmt.Errorf("failed to initialize EKS service: %w", err) } + if err := d.eksService.SpawnRegisteredReconcilers(); err != nil { + slog.Warn("EKS: SpawnRegisteredReconcilers failed", "err", err) + } // Ensure default VPC exists for system and admin accounts // (matches AWS: every account has a default VPC with IGW + default SG) @@ -1931,6 +1934,11 @@ func (d *Daemon) setupShutdown() { d.elbv2Service.Close() } + // Stop EKS per-cluster reconciler + bootstrap goroutines. + if d.eksService != nil { + d.eksService.Shutdown() + } + // Final cleanup for _, sub := range d.natsSubscriptions { // Unsubscribe from each subscription @@ -2214,26 +2222,15 @@ func (d *Daemon) wireLBAgentConfig() { } // Resolve gateway URL — the address LB VMs use to reach the AWS gateway. - // Precedence: - // 1. br-mgmt present + AWSGW on a dedicated IP distinct from AdvertiseIP - // (multi-node: AWSGW on a mgmt-only IP, VPC path can't reach it) → - // gateway URL is the AWSGW bind IP and lb-agent gets a bootcmd host - // route via br-mgmt. - // 2. AdvertiseIP set (single-node, or multi-node where AWSGW binds to - // the advertised IP) → AdvertiseIP. VMs reach it via VPC → external - // (OVN's own dnat_and_snat SNATs their reply back to the ALB EIP). - // Critically, we do NOT add the mgmt host route here: when host IPs - // on the WAN share the advertiseIP, the /32 route would steal the - // return path for host-initiated ALB connections — replies would - // egress via mgmt with the VM's 10.x source IP, bypass OVN's SNAT, - // and arrive at the host with a source that doesn't match the open - // TCP socket (the client dialed the EIP, not the VM IP). - // 3. br-mgmt present + AWSGW on 0.0.0.0 → br-mgmt IP (both LB flavours - // reach the daemon via mgmt). - // 4. DevNetworking shim → 10.0.2.2. - // 5. AWSGW bound to specific IP (no br-mgmt, no advertise) → that IP. - // 6. Else: error and skip assignment — no silent empty URL. - var gatewayHost string + // Host selection is centralized in resolveGatewayHost so the OIDC issuer + // host and EKS NATS URL come from the same source (see M7). The only + // LB-specific extra here is the multi-node mgmt host route: when the host + // resolves to a dedicated AWSGW bind IP reachable only over br-mgmt, the + // lb-agent needs a bootcmd /32 route via br-mgmt. We deliberately do NOT + // add that route when the host is AdvertiseIP — WAN host IPs may share the + // advertiseIP and a /32 would steal the return path for host-initiated ALB + // connections (reply egresses mgmt with the VM's 10.x source, bypassing + // OVN's SNAT, mismatching the open TCP socket dialed against the EIP). awsgwBindIP := "" if d.config.AWSGW.Host != "" { if h, _, splitErr := net.SplitHostPort(d.config.AWSGW.Host); splitErr == nil { @@ -2243,25 +2240,13 @@ func (d *Daemon) wireLBAgentConfig() { advertiseIP := d.config.AdvertiseIP - switch { - case d.mgmtBridgeIP != "" && awsgwBindIP != "" && awsgwBindIP != "0.0.0.0" && - !net.ParseIP(awsgwBindIP).IsLoopback() && awsgwBindIP != advertiseIP: - // Multi-node: AWSGW on a dedicated mgmt IP. VMs can't reach it via - // VPC → external, so add a bootcmd host route via br-mgmt. - gatewayHost = awsgwBindIP + gatewayHost := d.resolveGatewayHost() + + // Multi-node mgmt-dedicated AWSGW: host resolved to the bind IP over + // br-mgmt (case 1 in resolveGatewayHost). Loopback / no-mgmt / advertiseIP + // paths can't satisfy all three guards, so this matches only that case. + if gatewayHost != "" && gatewayHost == awsgwBindIP && d.mgmtBridgeIP != "" && awsgwBindIP != advertiseIP { d.mgmtRouteVia = awsgwBindIP - case advertiseIP != "" && advertiseIP != "0.0.0.0": - // Single-node, or multi-node where AWSGW binds to AdvertiseIP: VMs - // reach AWSGW via the normal VPC → external path. No mgmt host route. - gatewayHost = advertiseIP - case d.mgmtBridgeIP != "": - // br-mgmt present + AWSGW on 0.0.0.0 and no advertiseIP — br-mgmt IP - // is the only reachable address. - gatewayHost = d.mgmtBridgeIP - case d.config.Daemon.DevNetworking: - gatewayHost = "10.0.2.2" - case awsgwBindIP != "" && awsgwBindIP != "0.0.0.0": - gatewayHost = awsgwBindIP } // Extract port from AWSGW host config (e.g. "0.0.0.0:9999" → "9999"). diff --git a/spinifex/daemon/daemon_handlers_eks_test.go b/spinifex/daemon/daemon_handlers_eks_test.go index 3a2265546..5d446bd2a 100644 --- a/spinifex/daemon/daemon_handlers_eks_test.go +++ b/spinifex/daemon/daemon_handlers_eks_test.go @@ -26,8 +26,10 @@ func setupEKSDaemon(t *testing.T) (*Daemon, *nats.Conn) { } // requestEKS publishes a request on the given subject and returns the response -// payload. Every EKS handler in this branch returns NotImplemented, so the -// payload always decodes to that error code. +// payload. The non-lifecycle EKS handlers all return NotImplemented so the +// payload always decodes to that error code; CreateCluster / DescribeCluster +// / ListClusters / DeleteCluster have real bodies and are covered by their +// own handlers/eks unit tests. func requestEKS(t *testing.T, nc *nats.Conn, subject string) []byte { t.Helper() msg := nats.NewMsg(subject) @@ -55,12 +57,8 @@ func TestDaemonHandleEKS_AllHandlersDispatchToService(t *testing.T) { subject string handler nats.MsgHandler }{ - {"eks.CreateCluster", d.handleEKSCreateCluster}, - {"eks.DescribeCluster", d.handleEKSDescribeCluster}, - {"eks.ListClusters", d.handleEKSListClusters}, {"eks.UpdateClusterConfig", d.handleEKSUpdateClusterConfig}, {"eks.UpdateClusterVersion", d.handleEKSUpdateClusterVersion}, - {"eks.DeleteCluster", d.handleEKSDeleteCluster}, {"eks.CreateNodegroup", d.handleEKSCreateNodegroup}, {"eks.DescribeNodegroup", d.handleEKSDescribeNodegroup}, {"eks.ListNodegroups", d.handleEKSListNodegroups}, @@ -90,7 +88,7 @@ func TestDaemonHandleEKS_AllHandlersDispatchToService(t *testing.T) { {"eks.UntagResource", d.handleEKSUntagResource}, {"eks.ListTagsForResource", d.handleEKSListTagsForResource}, } - require.Equal(t, 34, len(cases), "expected exactly one handler per AWS EKS action") + require.Equal(t, 30, len(cases), "expected exactly one handler per non-lifecycle AWS EKS action") for _, c := range cases { t.Run(c.subject, func(t *testing.T) { diff --git a/spinifex/daemon/daemon_system_instance.go b/spinifex/daemon/daemon_system_instance.go index 948c20f80..bc4c47485 100644 --- a/spinifex/daemon/daemon_system_instance.go +++ b/spinifex/daemon/daemon_system_instance.go @@ -1,6 +1,8 @@ package daemon import ( + "encoding/base64" + "errors" "fmt" "log/slog" "os" @@ -12,6 +14,7 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc" handlers_elbv2 "github.com/mulgadc/spinifex/spinifex/handlers/elbv2" + "github.com/mulgadc/spinifex/spinifex/handlers/sysinstance" "github.com/mulgadc/spinifex/spinifex/network/topology" "github.com/mulgadc/spinifex/spinifex/tags" "github.com/mulgadc/spinifex/spinifex/utils" @@ -27,6 +30,9 @@ var _ handlers_elbv2.SystemInstanceLauncher = (*Daemon)(nil) // (bundled vmlinuz+initramfs, fw_cfg-delivered config). There is no AMI, // volume, or cloud-init path. func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput) (*handlers_elbv2.SystemInstanceOutput, error) { + if input.BootMode == sysinstance.BootAMI { + return d.launchAMISystemInstance(input) + } accountID := utils.GlobalAccountID // ENI account may differ from system account — the ENI is created under // the caller's account, so lookups/updates must use that account ID. @@ -365,11 +371,129 @@ func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput) }, nil } +// launchAMISystemInstance is the BootAMI branch of LaunchSystemInstance: a full +// AMI boot (root volume cloned from an AMI, cloud-init user-data + network +// config seed) for a system-managed VM, with a management-bridge NIC so the +// guest can reach the daemon (NATS/AWSGW) off its tenant VPC subnet. +// +// It mirrors the daemon RunInstances handler's Prepare → Insert → Launch split, +// allocating the mgmt NIC on the VM record between Insert and Launch so the +// cloud-init network-config rendered during Launch carries a static mgmt0 +// interface. The instance is owned by input.AccountID (the account its +// pre-created ENI lives in) and tagged input.ManagedBy so customer listings +// hide it. +func (d *Daemon) launchAMISystemInstance(input *sysinstance.SystemInstanceInput) (*sysinstance.SystemInstanceOutput, error) { + if d.instanceService == nil { + return nil, errors.New("sysinstance: instance service not initialized") + } + if input.ImageID == "" { + return nil, errors.New("sysinstance: BootAMI requires ImageID") + } + if input.AccountID == "" { + return nil, errors.New("sysinstance: BootAMI requires AccountID") + } + if input.ENIID == "" { + return nil, errors.New("sysinstance: BootAMI requires a pre-created ENI") + } + + runInput := &ec2.RunInstancesInput{ + ImageId: aws.String(input.ImageID), + InstanceType: aws.String(input.InstanceType), + MinCount: aws.Int64(1), + MaxCount: aws.Int64(1), + NetworkInterfaces: []*ec2.InstanceNetworkInterfaceSpecification{{ + NetworkInterfaceId: aws.String(input.ENIID), + DeviceIndex: aws.Int64(0), + }}, + } + if input.UserData != "" { + runInput.UserData = aws.String(base64.StdEncoding.EncodeToString([]byte(input.UserData))) + } + if input.ManagedBy != "" { + runInput.TagSpecifications = []*ec2.TagSpecification{{ + ResourceType: aws.String("instance"), + Tags: []*ec2.Tag{{Key: aws.String(tags.ManagedByKey), Value: aws.String(input.ManagedBy)}}, + }} + } + + _, instances, instanceType, err := d.instanceService.PrepareRunInstances(runInput, input.AccountID) + if err != nil { + return nil, err + } + if len(instances) == 0 { + return nil, errors.New("sysinstance: PrepareRunInstances returned no instance") + } + inst := instances[0] + inst.ManagedBy = input.ManagedBy + + for _, instance := range instances { + d.vmMgr.Insert(instance) + } + + // Management NIC must be set before LaunchRunInstances so the cloud-init + // network-config (built during launch) renders a static mgmt0 interface. + if err := d.attachSystemMgmtNIC(inst); err != nil { + d.vmMgr.MarkFailed(inst, "mgmt_nic_setup_failed") + return nil, err + } + + d.instanceService.LaunchRunInstances(instances, runInput, instanceType) + + privateIP := input.ENIIP + if privateIP == "" && inst.Instance != nil { + privateIP = aws.StringValue(inst.Instance.PrivateIpAddress) + } + slog.Info("LaunchSystemInstance (AMI) completed", + "instanceId", inst.ID, + "managedBy", input.ManagedBy, + "imageId", input.ImageID, + "mgmtIP", inst.MgmtIP, + "privateIp", privateIP, + ) + return &sysinstance.SystemInstanceOutput{ + InstanceID: inst.ID, + PrivateIP: privateIP, + MgmtIP: inst.MgmtIP, + }, nil +} + +// attachSystemMgmtNIC allocates a management-bridge IP + MAC for a system VM and +// creates the host-side tap on br-mgmt. The mgmt NIC is mandatory for system +// instances that boot from an AMI: their VPC ENI has no route off the tenant +// subnet, so the mgmt NIC is the only path to the daemon (NATS/AWSGW). On +// failure it releases the IP and returns an error so the caller can fail the +// launch. +func (d *Daemon) attachSystemMgmtNIC(inst *vm.VM) error { + if d.mgmtIPAllocator == nil || d.mgmtBridgeIP == "" { + return errors.New("sysinstance: management bridge unavailable; system VM would have no route to the daemon") + } + mgmtBridge := "br-mgmt" + if d.config.Daemon.MgmtBridge != "" { + mgmtBridge = d.config.Daemon.MgmtBridge + } + mgmtIP, err := d.mgmtIPAllocator.Allocate(inst.ID) + if err != nil { + return fmt.Errorf("allocate mgmt IP: %w", err) + } + inst.MgmtMAC = vm.GenerateMgmtMAC(inst.ID) + inst.MgmtIP = mgmtIP + tapName := vm.MgmtTapName(inst.ID) + if err := d.networkPlumber.SetupTap(vm.TapSpec{Name: tapName, Bridge: mgmtBridge}); err != nil { + d.mgmtIPAllocator.Release(inst.ID) + inst.MgmtMAC = "" + inst.MgmtIP = "" + return fmt.Errorf("setup mgmt tap: %w", err) + } + slog.Info("System instance mgmt NIC configured", + "instanceId", inst.ID, "mgmtIP", mgmtIP, "mgmtMAC", inst.MgmtMAC, "mgmtTap", tapName) + return nil +} + // TerminateSystemInstance stops and cleans up a system-managed VM. func (d *Daemon) TerminateSystemInstance(instanceID string) error { instance, exists := d.vmMgr.Get(instanceID) if !exists { - return fmt.Errorf("instance %s not found", instanceID) + return fmt.Errorf("%w: %s", sysinstance.ErrSystemInstanceNotFound, instanceID) } // Release EIP through the EIP service for system VMs whose public IP was diff --git a/spinifex/daemon/eks_adapter.go b/spinifex/daemon/eks_adapter.go new file mode 100644 index 000000000..38e5a19ba --- /dev/null +++ b/spinifex/daemon/eks_adapter.go @@ -0,0 +1,31 @@ +package daemon + +import ( + "errors" + + handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks" +) + +// daemonEKSSubnetResolver adapts the daemon's VPCServiceImpl to the +// handlers_eks subnetVPCResolver-equivalent. handlers/eks declares the +// interface privately; the daemon needs an exported shape only because the +// constructor wires it via EKSServiceDeps.SubnetResolver = ... +// +// The translation is trivial: VPCService.GetSubnet returns a SubnetRecord +// with VpcId, which is the only field the EKS launcher needs. +type daemonEKSSubnetResolver struct { + d *Daemon +} + +func (a *daemonEKSSubnetResolver) GetSubnetVPC(accountID, subnetID string) (string, error) { + if a.d == nil || a.d.vpcService == nil { + return "", errors.New("eks: VPC service not initialized") + } + rec, err := a.d.vpcService.GetSubnet(accountID, subnetID) + if err != nil { + return "", err + } + return rec.VpcId, nil +} + +var _ handlers_eks.SubnetVPCResolver = (*daemonEKSSubnetResolver)(nil) diff --git a/spinifex/daemon/eks_deps.go b/spinifex/daemon/eks_deps.go new file mode 100644 index 000000000..2a0e63ef3 --- /dev/null +++ b/spinifex/daemon/eks_deps.go @@ -0,0 +1,159 @@ +package daemon + +import ( + "net" + "os" + "path/filepath" + + handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks" + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" + "log/slog" +) + +// buildEKSServiceDeps assembles the EKSServiceDeps the EKS service needs to +// run CreateCluster/DescribeCluster/ListClusters/DeleteCluster end-to-end. +// All collaborators must already be initialized; callers invoke this from +// Daemon.startCluster after VPC, ELBv2, Instance, and Image services are +// ready. +// +// MasterKey is loaded best-effort from the config dir (the directory holding +// spinifex.toml — same place admin init writes master.key, and the same +// resolution daemon TLS + predastore config use). A missing key disables OIDC +// envelope encryption, which in turn fails CreateCluster at the +// depsReadyForOrchestration gate — the daemon logs and continues. +func (d *Daemon) buildEKSServiceDeps() handlers_eks.EKSServiceDeps { + masterKey, err := handlers_iam.LoadMasterKey(filepath.Join(filepath.Dir(d.configPath), "master.key")) + if err != nil { + slog.Warn("EKS: LoadMasterKey failed; CreateCluster + DeleteCluster will reject until key is provisioned", + "err", err) + masterKey = nil + } + + // CA PEM the K3s server VM uses to verify the daemon's NATS TLS. Same CA + // that signs the AWSGW server cert; read here so the VM can publish its + // one-shot bootstrap messages over the token+TLS NATS endpoint. + natsCA := "" + if d.config.NATS.CACert != "" { + if caBytes, readErr := os.ReadFile(d.config.NATS.CACert); readErr == nil { + natsCA = string(caBytes) + } else { + slog.Warn("EKS: read NATS CACert failed; K3s server VM will not reach NATS over TLS", + "path", d.config.NATS.CACert, "err", readErr) + } + } + + return handlers_eks.EKSServiceDeps{ + Config: d.config, + NATSConn: d.natsConn, + MasterKey: masterKey, + GatewayBaseURL: d.resolveGatewayBaseURL(), + Region: d.config.Region, + HolderID: d.node, + NATSURL: d.resolveSystemNATSURL(), + NATSToken: d.config.NATS.ACL.Token, + NATSCACert: natsCA, + VPCSG: d.vpcService, + VPCK3s: d.vpcService, + VPCSubnet: &daemonEKSSubnetResolver{d: d}, + NLB: d.elbv2Service, + Instance: d, + Image: d.imageService, + EIP: d.eipService, + } +} + +// resolveGatewayHost is the single source of truth for the off-host-reachable +// address LB VMs, EKS OIDC issuers, and the EKS NATS URL all dial. startCluster +// and buildEKSServiceDeps both call it so the OIDC issuer host can never +// diverge from the host the lb-agent actually reaches (M7). Precedence: +// +// 1. br-mgmt present + AWSGW on a dedicated non-loopback IP distinct from +// AdvertiseIP (multi-node: AWSGW on a mgmt-only IP, VPC path can't reach +// it) → the AWSGW bind IP. startCluster additionally adds a bootcmd host +// route via br-mgmt for this case only. +// 2. AdvertiseIP set → AdvertiseIP (single-node, or multi-node where AWSGW +// binds the advertised IP; VMs reach it via VPC → external). +// 3. br-mgmt present + AWSGW on 0.0.0.0, no AdvertiseIP → the br-mgmt IP. +// 4. DevNetworking shim → 10.0.2.2. +// 5. AWSGW bound to a specific IP (no br-mgmt, no advertise) → that IP. +// +// Returns "" if no reachable host can be derived. +func (d *Daemon) resolveGatewayHost() string { + awsgwBindIP := "" + if d.config.AWSGW.Host != "" { + if h, _, splitErr := net.SplitHostPort(d.config.AWSGW.Host); splitErr == nil { + awsgwBindIP = h + } + } + advertiseIP := d.config.AdvertiseIP + + switch { + case d.mgmtBridgeIP != "" && awsgwBindIP != "" && awsgwBindIP != "0.0.0.0" && + !net.ParseIP(awsgwBindIP).IsLoopback() && awsgwBindIP != advertiseIP: + return awsgwBindIP + case advertiseIP != "" && advertiseIP != "0.0.0.0": + return advertiseIP + case d.mgmtBridgeIP != "": + return d.mgmtBridgeIP + case d.config.Daemon.DevNetworking: + return "10.0.2.2" + case awsgwBindIP != "" && awsgwBindIP != "0.0.0.0": + return awsgwBindIP + } + return "" +} + +// resolveGatewayBaseURL is the HTTPS AWSGW endpoint EKS OIDC issuers are built +// from. Returns "" if no reachable host can be derived, in which case +// CreateCluster fails at depsReadyForOrchestration. +func (d *Daemon) resolveGatewayBaseURL() string { + host := d.resolveGatewayHost() + if host == "" { + return "" + } + gatewayPort := "9999" + if d.config.AWSGW.Host != "" { + if _, port, splitErr := net.SplitHostPort(d.config.AWSGW.Host); splitErr == nil && port != "" { + gatewayPort = port + } + } + return "https://" + net.JoinHostPort(host, gatewayPort) +} + +// resolveSystemNATSURL derives the NATS URL a system instance (the K3s server +// VM) dials to publish its one-shot bootstrap messages. Unlike a customer VM, +// a system instance reaches the daemon only over the mgmt bridge: its cloud-init +// mgmt0 NIC is on-link to mgmtBridgeIP with no off-link route, so NATS must be +// dialed at the bridge address directly. NATS listens on 0.0.0.0:4222, so the +// bridge IP is reachable. Falls back to resolveNATSURL() (gateway-host +// precedence) when no mgmt bridge exists — e.g. dev-shim networking. +func (d *Daemon) resolveSystemNATSURL() string { + if d.mgmtBridgeIP == "" { + return d.resolveNATSURL() + } + natsPort := "4222" + if d.config.NATS.Host != "" { + if _, port, splitErr := net.SplitHostPort(d.config.NATS.Host); splitErr == nil && port != "" { + natsPort = port + } + } + return "nats://" + net.JoinHostPort(d.mgmtBridgeIP, natsPort) +} + +// resolveNATSURL derives the NATS URL the K3s server VM dials to publish its +// one-shot bootstrap messages. Host reuses the gateway host resolution; port +// comes from the configured NATS listen address (default 4222). Returns "" if +// no reachable host can be derived. +func (d *Daemon) resolveNATSURL() string { + host := d.resolveGatewayHost() + if host == "" { + return "" + } + natsPort := "4222" + if d.config.NATS.Host != "" { + if _, port, splitErr := net.SplitHostPort(d.config.NATS.Host); splitErr == nil && port != "" { + natsPort = port + } + } + return "nats://" + net.JoinHostPort(host, natsPort) +} diff --git a/spinifex/daemon/eks_deps_test.go b/spinifex/daemon/eks_deps_test.go new file mode 100644 index 000000000..3b5499440 --- /dev/null +++ b/spinifex/daemon/eks_deps_test.go @@ -0,0 +1,86 @@ +package daemon + +import ( + "testing" + + "github.com/mulgadc/spinifex/spinifex/config" + "github.com/stretchr/testify/assert" +) + +// TestResolveGatewayHost covers all five host-selection branches plus the +// no-reachable-host fallthrough. resolveGatewayHost is the single source of +// truth for the OIDC issuer host, EKS NATS URL, and lb-agent gateway URL, so +// each branch is pinned here to prevent silent divergence (M7). +func TestResolveGatewayHost(t *testing.T) { + tests := []struct { + name string + awsgwHost string + advertiseIP string + mgmtBridge string + devNet bool + want string + }{ + { + name: "1_mgmt_dedicated_awsgw_ip", + awsgwHost: "10.20.0.5:9999", + advertiseIP: "203.0.113.7", + mgmtBridge: "10.15.8.1", + want: "10.20.0.5", + }, + { + name: "1_skipped_when_bind_equals_advertise_falls_to_advertise", + awsgwHost: "203.0.113.7:9999", + advertiseIP: "203.0.113.7", + mgmtBridge: "10.15.8.1", + want: "203.0.113.7", + }, + { + name: "1_skipped_when_bind_loopback_falls_to_mgmt", + awsgwHost: "127.0.0.1:9999", + mgmtBridge: "10.15.8.1", + want: "10.15.8.1", + }, + { + name: "2_advertise_ip", + awsgwHost: "0.0.0.0:9999", + advertiseIP: "203.0.113.7", + want: "203.0.113.7", + }, + { + name: "3_mgmt_bridge_when_awsgw_wildcard", + awsgwHost: "0.0.0.0:9999", + mgmtBridge: "10.15.8.1", + want: "10.15.8.1", + }, + { + name: "4_dev_networking_shim", + awsgwHost: "0.0.0.0:9999", + devNet: true, + want: "10.0.2.2", + }, + { + name: "5_awsgw_specific_ip_no_mgmt_no_advertise", + awsgwHost: "198.51.100.9:9999", + want: "198.51.100.9", + }, + { + name: "6_no_reachable_host", + awsgwHost: "0.0.0.0:9999", + want: "", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + d := &Daemon{ + mgmtBridgeIP: tc.mgmtBridge, + config: &config.Config{ + AdvertiseIP: tc.advertiseIP, + AWSGW: config.AWSGWConfig{Host: tc.awsgwHost}, + Daemon: config.DaemonConfig{DevNetworking: tc.devNet}, + }, + } + assert.Equal(t, tc.want, d.resolveGatewayHost()) + }) + } +} diff --git a/spinifex/daemon/vm_adapters.go b/spinifex/daemon/vm_adapters.go index 95891d638..0210a88dd 100644 --- a/spinifex/daemon/vm_adapters.go +++ b/spinifex/daemon/vm_adapters.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "log/slog" - "strings" "time" "github.com/aws/aws-sdk-go/service/ec2" @@ -631,7 +630,7 @@ func (a *instanceCleanerAdapter) DetachAndDeleteENI(instance *vm.VM) { if _, eniErr := a.d.vpcService.DeleteNetworkInterface(&ec2.DeleteNetworkInterfaceInput{ NetworkInterfaceId: &instance.ENIId, }, instance.AccountID); eniErr != nil { - if strings.Contains(eniErr.Error(), awserrors.ErrorInvalidNetworkInterfaceIDNotFound) { + if awserrors.IsErrorCode(eniErr, awserrors.ErrorInvalidNetworkInterfaceIDNotFound) { slog.Debug("ENI already cleaned up on termination", "eni", instance.ENIId) } else { slog.Error("Failed to delete ENI on termination", diff --git a/spinifex/daemon/vpc_adapter.go b/spinifex/daemon/vpc_adapter.go index da9ca0466..b44431230 100644 --- a/spinifex/daemon/vpc_adapter.go +++ b/spinifex/daemon/vpc_adapter.go @@ -39,6 +39,22 @@ func (a *daemonENICreator) GetSubnet(accountID, subnetID string) (*handlers_ec2_ }, nil } +func (a *daemonENICreator) GetENI(accountID, eniID string) (*handlers_ec2_instance.ENIInfo, error) { + rec, err := a.d.vpcService.GetENIRecord(accountID, eniID) + if err != nil { + return nil, err + } + return &handlers_ec2_instance.ENIInfo{ + NetworkInterfaceID: rec.NetworkInterfaceId, + SubnetID: rec.SubnetId, + VpcID: rec.VpcId, + PrivateIpAddress: rec.PrivateIpAddress, + MacAddress: rec.MacAddress, + Status: rec.Status, + SecurityGroupIDs: rec.SecurityGroupIds, + }, nil +} + func (a *daemonENICreator) CreateNetworkInterface(input *ec2.CreateNetworkInterfaceInput, accountID string) (*ec2.CreateNetworkInterfaceOutput, error) { return a.d.vpcService.CreateNetworkInterface(input, accountID) } diff --git a/spinifex/gateway/eks/handler.go b/spinifex/gateway/eks/handler.go index 2c45485f2..837cc6825 100644 --- a/spinifex/gateway/eks/handler.go +++ b/spinifex/gateway/eks/handler.go @@ -12,6 +12,7 @@ import ( "log/slog" "net/http" + "github.com/aws/aws-sdk-go/private/protocol/json/jsonutil" "github.com/mulgadc/spinifex/spinifex/awserrors" ) @@ -41,9 +42,13 @@ func GenerateEKSErrorResponse(code, message string) []byte { return body } -// WriteJSONResponse serializes obj as JSON and writes it as a 200 response. +// WriteJSONResponse serializes obj as AWS REST-JSON and writes it as a 200 +// response. aws-sdk-go *Output structs tag their fields with locationName (the +// restjson wire name) and carry no json: tags, so encoding/json would emit Go +// PascalCase keys the AWS SDK cannot parse. jsonutil.BuildJSON is the same +// marshaler the SDK uses for restjson bodies and honors locationName. func WriteJSONResponse(w http.ResponseWriter, obj any) { - body, err := json.Marshal(obj) + body, err := jsonutil.BuildJSON(obj) if err != nil { slog.Error("Failed to marshal EKS response JSON", "err", err) WriteJSONError(w, awserrors.ErrorInternalError, "failed to marshal response", http.StatusInternalServerError) diff --git a/spinifex/gateway/eks/handler_test.go b/spinifex/gateway/eks/handler_test.go index 711366bc3..68b9e2e2d 100644 --- a/spinifex/gateway/eks/handler_test.go +++ b/spinifex/gateway/eks/handler_test.go @@ -7,6 +7,8 @@ import ( "net/http/httptest" "testing" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/eks" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -42,6 +44,64 @@ func TestWriteJSONResponse_SerializesObject(t *testing.T) { assert.Equal(t, "bar", got["foo"]) } +// TestWriteJSONResponse_RestJSONWireShape guards against regressing to +// encoding/json, which ignores aws-sdk-go's locationName tags and emits Go +// PascalCase keys the AWS SDK cannot parse. The body must carry the lowercase +// restjson field names, nested all the way down. +func TestWriteJSONResponse_RestJSONWireShape(t *testing.T) { + w := httptest.NewRecorder() + out := &eks.DescribeClusterOutput{ + Cluster: &eks.Cluster{ + Name: aws.String("smoke"), + Status: aws.String("ACTIVE"), + Endpoint: aws.String("https://internal-eks-smoke:443"), + Identity: &eks.Identity{ + Oidc: &eks.OIDC{Issuer: aws.String("https://gw/oidc/eks/x")}, + }, + }, + } + WriteJSONResponse(w, out) + require.Equal(t, http.StatusOK, w.Code) + + body := w.Body.String() + for _, k := range []string{`"cluster"`, `"name"`, `"status"`, `"endpoint"`, `"identity"`, `"oidc"`, `"issuer"`} { + assert.Contains(t, body, k, "restjson body missing lowercase key %s", k) + } + for _, k := range []string{`"Cluster"`, `"Status"`, `"Endpoint"`, `"Identity"`, `"Issuer"`} { + assert.NotContains(t, body, k, "restjson body leaked PascalCase key %s — encoding/json regression", k) + } + + // The AWS SDK round-trips through the same restjson nested keys. + var got struct { + Cluster struct { + Status string `json:"status"` + Endpoint string `json:"endpoint"` + Identity struct { + Oidc struct { + Issuer string `json:"issuer"` + } `json:"oidc"` + } `json:"identity"` + } `json:"cluster"` + } + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &got)) + assert.Equal(t, "ACTIVE", got.Cluster.Status) + assert.Equal(t, "https://internal-eks-smoke:443", got.Cluster.Endpoint) + assert.Equal(t, "https://gw/oidc/eks/x", got.Cluster.Identity.Oidc.Issuer) +} + +func TestWriteJSONResponse_ListClustersWireShape(t *testing.T) { + w := httptest.NewRecorder() + WriteJSONResponse(w, &eks.ListClustersOutput{ + Clusters: aws.StringSlice([]string{"alpha", "beta"}), + NextToken: aws.String("tok"), + }) + body := w.Body.String() + assert.Contains(t, body, `"clusters"`) + assert.Contains(t, body, `"nextToken"`) + assert.NotContains(t, body, `"Clusters"`) + assert.NotContains(t, body, `"NextToken"`) +} + type sampleInput struct { Name string `json:"name"` } diff --git a/spinifex/handlers/ec2/instance/service.go b/spinifex/handlers/ec2/instance/service.go index c552157c3..8c7e40d42 100644 --- a/spinifex/handlers/ec2/instance/service.go +++ b/spinifex/handlers/ec2/instance/service.go @@ -129,15 +129,32 @@ type SubnetInfo struct { MapPublicIpOnLaunch bool } +// ENIInfo carries the subset of ENI metadata RunInstances needs to attach a +// pre-created ENI as the primary interface. Translated from +// handlers/ec2/vpc.ENIRecord by the daemon adapter to avoid a cyclic +// instance↔vpc import. +type ENIInfo struct { + NetworkInterfaceID string + SubnetID string + VpcID string + PrivateIpAddress string + MacAddress string + Status string + SecurityGroupIDs []string +} + // ENICreator covers the VPC/ENI operations RunInstances performs while // auto-attaching a primary interface. Implemented via an adapter over // handlers/ec2/vpc.VPCServiceImpl on the daemon. DetachENI lives here (not on // ENIDeleter) because the launch-time NAT-rollback path needs to flip the ENI // to "available" before ENIDeleter.DeleteNetworkInterface, which rejects -// in-use ENIs. +// in-use ENIs. GetENI supports the pre-created-ENI launch path where the +// caller passes NetworkInterfaces[0].NetworkInterfaceId and RunInstances +// attaches the existing ENI instead of creating a new one. type ENICreator interface { GetDefaultSubnet(accountID string) (*SubnetInfo, error) GetSubnet(accountID, subnetID string) (*SubnetInfo, error) + GetENI(accountID, eniID string) (*ENIInfo, error) CreateNetworkInterface(input *ec2.CreateNetworkInterfaceInput, accountID string) (*ec2.CreateNetworkInterfaceOutput, error) AttachENI(accountID, eniID, instanceID string, deviceIndex int64) (string, error) DetachENI(accountID, eniID string) error diff --git a/spinifex/handlers/ec2/instance/service_impl.go b/spinifex/handlers/ec2/instance/service_impl.go index 5e0c17b53..7515443bf 100644 --- a/spinifex/handlers/ec2/instance/service_impl.go +++ b/spinifex/handlers/ec2/instance/service_impl.go @@ -631,6 +631,35 @@ func (s *InstanceServiceImpl) PrepareRunInstances(input *ec2.RunInstancesInput, } } + // Pre-created ENI path: when the caller passes + // NetworkInterfaces[0].NetworkInterfaceId, attach that existing ENI + // instead of auto-creating one. AWS-parity behaviour; consumed by the + // EKS cluster launcher which needs the ENI to live under the customer + // account before the K3s server VM is created. + preExistingENIID := "" + if len(input.NetworkInterfaces) > 0 && input.NetworkInterfaces[0] != nil { + preExistingENIID = aws.StringValue(input.NetworkInterfaces[0].NetworkInterfaceId) + } + if preExistingENIID != "" { + if s.eniCreator == nil { + slog.Error("PrepareRunInstances: pre-created ENI specified but ENI service unavailable", + "instanceId", instance.ID, "eniId", preExistingENIID) + lastRunErr = errors.New(awserrors.ErrorServerInternal) + s.resourceMgr.Deallocate(instanceType) + continue + } + if err := s.attachPreCreatedENI(accountID, preExistingENIID, instance, ec2Instance); err != nil { + slog.Error("PrepareRunInstances: pre-created ENI attach failed", + "instanceId", instance.ID, "eniId", preExistingENIID, "err", err) + lastRunErr = err + s.resourceMgr.Deallocate(instanceType) + continue + } + instances = append(instances, instance) + allEC2Instances = append(allEC2Instances, ec2Instance) + continue + } + if (input.SubnetId == nil || *input.SubnetId == "") && s.eniCreator != nil { defaultSubnet, dsErr := s.eniCreator.GetDefaultSubnet(accountID) if dsErr == nil && defaultSubnet != nil { @@ -774,6 +803,62 @@ func (s *InstanceServiceImpl) PrepareRunInstances(input *ec2.RunInstancesInput, return reservation, instances, instanceType, nil } +// attachPreCreatedENI looks up an existing ENI in accountID, verifies it is +// not already in-use, attaches it as device-0 to instance, and populates the +// VM + ec2.Instance with the ENI's network identity (private IP, MAC, subnet, +// VPC, security groups). Returns an error without mutating any state if the +// lookup or attach fails. Public-IP auto-assignment is skipped on the +// pre-created path because the caller is expected to manage that out-of-band. +func (s *InstanceServiceImpl) attachPreCreatedENI(accountID, eniID string, instance *vm.VM, ec2Instance *ec2.Instance) error { + eniInfo, err := s.eniCreator.GetENI(accountID, eniID) + if err != nil { + return err + } + if eniInfo == nil { + return errors.New(awserrors.ErrorInvalidNetworkInterfaceIDNotFound) + } + if eniInfo.Status == "in-use" { + return errors.New(awserrors.ErrorInvalidNetworkInterfaceInUse) + } + if _, err := s.eniCreator.AttachENI(accountID, eniID, instance.ID, 0); err != nil { + return err + } + + instance.ENIId = eniID + instance.ENIMac = eniInfo.MacAddress + + ec2Instance.SetPrivateIpAddress(eniInfo.PrivateIpAddress) + ec2Instance.SetSubnetId(eniInfo.SubnetID) + ec2Instance.SetVpcId(eniInfo.VpcID) + if len(eniInfo.SecurityGroupIDs) > 0 { + groups := make([]*ec2.GroupIdentifier, 0, len(eniInfo.SecurityGroupIDs)) + for _, id := range eniInfo.SecurityGroupIDs { + groups = append(groups, &ec2.GroupIdentifier{GroupId: aws.String(id)}) + } + ec2Instance.SecurityGroups = groups + } + ec2Instance.NetworkInterfaces = []*ec2.InstanceNetworkInterface{{ + NetworkInterfaceId: aws.String(eniID), + PrivateIpAddress: aws.String(eniInfo.PrivateIpAddress), + MacAddress: aws.String(eniInfo.MacAddress), + SubnetId: aws.String(eniInfo.SubnetID), + VpcId: aws.String(eniInfo.VpcID), + Status: aws.String("in-use"), + Attachment: &ec2.InstanceNetworkInterfaceAttachment{ + DeviceIndex: aws.Int64(0), + Status: aws.String("attached"), + }, + }} + + slog.Info("PrepareRunInstances: attached pre-created ENI", + "instanceId", instance.ID, + "eniId", eniID, + "privateIp", eniInfo.PrivateIpAddress, + "mac", eniInfo.MacAddress, + ) + return nil +} + // LaunchRunInstances takes the VMs created by PrepareRunInstances (already // inserted into vmMgr by the caller) and runs the heavyweight launch loop: // volume preparation, GPU claim, vmMgr.Run. Failures mark the VM as failed diff --git a/spinifex/handlers/ec2/instance/service_impl_test.go b/spinifex/handlers/ec2/instance/service_impl_test.go index 9d8e81b4f..099e64000 100644 --- a/spinifex/handlers/ec2/instance/service_impl_test.go +++ b/spinifex/handlers/ec2/instance/service_impl_test.go @@ -2451,7 +2451,10 @@ func TestStopOrTerminateInstance_TerminationProtection(t *testing.T) { type fakeENICreator struct { defaultSubnet *SubnetInfo subnet *SubnetInfo + getENIByID map[string]*ENIInfo + getENIErr error createOut *ec2.CreateNetworkInterfaceOutput + createCalls int createErr error attachErr error attachCalls int @@ -2474,7 +2477,22 @@ func (f *fakeENICreator) GetSubnet(_, _ string) (*SubnetInfo, error) { return f.subnet, nil } +func (f *fakeENICreator) GetENI(_, eniID string) (*ENIInfo, error) { + if f.getENIErr != nil { + return nil, f.getENIErr + } + if f.getENIByID == nil { + return nil, errors.New("no ENI configured") + } + info, ok := f.getENIByID[eniID] + if !ok { + return nil, errors.New("eni not found") + } + return info, nil +} + func (f *fakeENICreator) CreateNetworkInterface(_ *ec2.CreateNetworkInterfaceInput, _ string) (*ec2.CreateNetworkInterfaceOutput, error) { + f.createCalls++ if f.createErr != nil { return nil, f.createErr } @@ -3188,3 +3206,115 @@ func TestInstanceArchitecture(t *testing.T) { }) } } + +func TestPrepareRunInstances_PreCreatedENIAttachedSkipsAutoCreate(t *testing.T) { + eni := &fakeENICreator{ + getENIByID: map[string]*ENIInfo{ + "eni-pre": { + NetworkInterfaceID: "eni-pre", + SubnetID: "subnet-pre", + VpcID: "vpc-pre", + PrivateIpAddress: "10.0.5.42", + MacAddress: "aa:bb:cc:dd:ee:01", + Status: "available", + SecurityGroupIDs: []string{"sg-pre"}, + }, + }, + } + svc, _ := prepareSvcWithENI(t, eni, nil) + + _, instances, _, err := svc.PrepareRunInstances(&ec2.RunInstancesInput{ + InstanceType: aws.String("t3.micro"), + ImageId: aws.String("ami-1"), + MinCount: aws.Int64(1), + MaxCount: aws.Int64(1), + NetworkInterfaces: []*ec2.InstanceNetworkInterfaceSpecification{{ + NetworkInterfaceId: aws.String("eni-pre"), + DeviceIndex: aws.Int64(0), + }}, + }, "acc") + require.NoError(t, err) + require.Len(t, instances, 1) + assert.Equal(t, "eni-pre", instances[0].ENIId) + assert.Equal(t, "aa:bb:cc:dd:ee:01", instances[0].ENIMac) + assert.Equal(t, 1, eni.attachCalls) + assert.Equal(t, 0, eni.createCalls, "auto-create must not run when NetworkInterfaceId is specified") +} + +func TestPrepareRunInstances_PreCreatedENIInUseRejected(t *testing.T) { + eni := &fakeENICreator{ + getENIByID: map[string]*ENIInfo{ + "eni-busy": { + NetworkInterfaceID: "eni-busy", + Status: "in-use", + }, + }, + } + svc, _ := prepareSvcWithENI(t, eni, nil) + + _, _, _, err := svc.PrepareRunInstances(&ec2.RunInstancesInput{ + InstanceType: aws.String("t3.micro"), + ImageId: aws.String("ami-1"), + MinCount: aws.Int64(1), + MaxCount: aws.Int64(1), + NetworkInterfaces: []*ec2.InstanceNetworkInterfaceSpecification{{ + NetworkInterfaceId: aws.String("eni-busy"), + }}, + }, "acc") + require.Error(t, err) + assert.Equal(t, awserrors.ErrorInvalidNetworkInterfaceInUse, err.Error()) + assert.Equal(t, 0, eni.attachCalls, "in-use ENI must not be attached") + assert.Equal(t, 0, eni.createCalls, "auto-create must not run when NetworkInterfaceId is specified") +} + +func TestPrepareRunInstances_PreCreatedENILookupErrorSurfaced(t *testing.T) { + eni := &fakeENICreator{ + getENIErr: errors.New(awserrors.ErrorInvalidNetworkInterfaceIDNotFound), + } + svc, _ := prepareSvcWithENI(t, eni, nil) + + _, _, _, err := svc.PrepareRunInstances(&ec2.RunInstancesInput{ + InstanceType: aws.String("t3.micro"), + ImageId: aws.String("ami-1"), + MinCount: aws.Int64(1), + MaxCount: aws.Int64(1), + NetworkInterfaces: []*ec2.InstanceNetworkInterfaceSpecification{{ + NetworkInterfaceId: aws.String("eni-ghost"), + }}, + }, "acc") + require.Error(t, err) + assert.Equal(t, awserrors.ErrorInvalidNetworkInterfaceIDNotFound, err.Error()) + assert.Equal(t, 0, eni.attachCalls) + assert.Equal(t, 0, eni.createCalls) +} + +func TestPrepareRunInstances_PreCreatedENIAttachErrorSurfaced(t *testing.T) { + eni := &fakeENICreator{ + getENIByID: map[string]*ENIInfo{ + "eni-pre": { + NetworkInterfaceID: "eni-pre", + Status: "available", + PrivateIpAddress: "10.0.5.42", + MacAddress: "aa:bb:cc:dd:ee:01", + SubnetID: "subnet-pre", + VpcID: "vpc-pre", + }, + }, + attachErr: errors.New(awserrors.ErrorServerInternal), + } + svc, _ := prepareSvcWithENI(t, eni, nil) + + _, _, _, err := svc.PrepareRunInstances(&ec2.RunInstancesInput{ + InstanceType: aws.String("t3.micro"), + ImageId: aws.String("ami-1"), + MinCount: aws.Int64(1), + MaxCount: aws.Int64(1), + NetworkInterfaces: []*ec2.InstanceNetworkInterfaceSpecification{{ + NetworkInterfaceId: aws.String("eni-pre"), + }}, + }, "acc") + require.Error(t, err) + assert.Equal(t, awserrors.ErrorServerInternal, err.Error()) + assert.Equal(t, 1, eni.attachCalls) + assert.Equal(t, 0, eni.createCalls) +} diff --git a/spinifex/handlers/eks/awsgw_endpoint.go b/spinifex/handlers/eks/awsgw_endpoint.go new file mode 100644 index 000000000..4031512f3 --- /dev/null +++ b/spinifex/handlers/eks/awsgw_endpoint.go @@ -0,0 +1,47 @@ +package handlers_eks + +import ( + "errors" + "fmt" + "net/url" + "strings" +) + +// ClusterOIDCIssuer returns the public OIDC issuer URL for one cluster. This +// is the URL kube-apiserver advertises as service-account-issuer and the URL +// that AWS STS / external IRSA validators fetch /.well-known/openid-configuration +// against. Format per plan §10.6: +// +// {gatewayBaseURL}/oidc/eks/{region}/{accountID}/{clusterName} +// +// gatewayBaseURL is the awsgw HTTPS endpoint (e.g. "https://10.0.0.1:9999"), +// without a trailing /oidc. Validates that region/accountID/clusterName are +// non-empty and URL-safe. +func ClusterOIDCIssuer(gatewayBaseURL, region, accountID, clusterName string) (string, error) { + if gatewayBaseURL == "" { + return "", errors.New("eks: ClusterOIDCIssuer empty gatewayBaseURL") + } + if region == "" { + return "", errors.New("eks: ClusterOIDCIssuer empty region") + } + if accountID == "" { + return "", errors.New("eks: ClusterOIDCIssuer empty accountID") + } + if clusterName == "" { + return "", errors.New("eks: ClusterOIDCIssuer empty clusterName") + } + parsed, err := url.Parse(gatewayBaseURL) + if err != nil { + return "", fmt.Errorf("parse gatewayBaseURL %q: %w", gatewayBaseURL, err) + } + if parsed.Scheme == "" || parsed.Host == "" { + return "", fmt.Errorf("eks: gatewayBaseURL %q missing scheme or host", gatewayBaseURL) + } + for name, v := range map[string]string{"region": region, "accountID": accountID, "clusterName": clusterName} { + if strings.ContainsAny(v, "/?#") { + return "", fmt.Errorf("eks: %s %q contains URL-unsafe characters", name, v) + } + } + base := strings.TrimRight(gatewayBaseURL, "/") + return fmt.Sprintf("%s/oidc/eks/%s/%s/%s", base, region, accountID, clusterName), nil +} diff --git a/spinifex/handlers/eks/awsgw_endpoint_test.go b/spinifex/handlers/eks/awsgw_endpoint_test.go new file mode 100644 index 000000000..7fec30e97 --- /dev/null +++ b/spinifex/handlers/eks/awsgw_endpoint_test.go @@ -0,0 +1,62 @@ +package handlers_eks + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestClusterOIDCIssuer_HappyPath(t *testing.T) { + got, err := ClusterOIDCIssuer("https://gw.example:9999", "us-east-1", "111122223333", "alpha") + require.NoError(t, err) + assert.Equal(t, "https://gw.example:9999/oidc/eks/us-east-1/111122223333/alpha", got) +} + +func TestClusterOIDCIssuer_TrimsTrailingSlash(t *testing.T) { + got, err := ClusterOIDCIssuer("https://gw.example:9999/", "us-east-1", "111122223333", "alpha") + require.NoError(t, err) + assert.Equal(t, "https://gw.example:9999/oidc/eks/us-east-1/111122223333/alpha", got) +} + +func TestClusterOIDCIssuer_EmptyInputsRejected(t *testing.T) { + cases := []struct { + name string + base, region, accountID, clusterName, want string + }{ + {"empty base", "", "us-east-1", "111122223333", "alpha", "gatewayBaseURL"}, + {"empty region", "https://gw", "", "111122223333", "alpha", "region"}, + {"empty accountID", "https://gw", "us-east-1", "", "alpha", "accountID"}, + {"empty clusterName", "https://gw", "us-east-1", "111122223333", "", "clusterName"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := ClusterOIDCIssuer(tc.base, tc.region, tc.accountID, tc.clusterName) + require.Error(t, err) + assert.Contains(t, err.Error(), tc.want) + }) + } +} + +func TestClusterOIDCIssuer_MalformedBaseRejected(t *testing.T) { + _, err := ClusterOIDCIssuer("not-a-url", "us-east-1", "111122223333", "alpha") + require.Error(t, err) + assert.Contains(t, err.Error(), "missing scheme or host") +} + +func TestClusterOIDCIssuer_URLUnsafeInputsRejected(t *testing.T) { + cases := []struct { + field, region, accountID, clusterName string + }{ + {"region", "us/east", "111122223333", "alpha"}, + {"accountID", "us-east-1", "111?ab", "alpha"}, + {"clusterName", "us-east-1", "111122223333", "alpha#frag"}, + } + for _, tc := range cases { + t.Run(tc.field, func(t *testing.T) { + _, err := ClusterOIDCIssuer("https://gw", tc.region, tc.accountID, tc.clusterName) + require.Error(t, err) + assert.Contains(t, err.Error(), "URL-unsafe") + }) + } +} diff --git a/spinifex/handlers/eks/cluster_reconciler.go b/spinifex/handlers/eks/cluster_reconciler.go new file mode 100644 index 000000000..fc4cde12d --- /dev/null +++ b/spinifex/handlers/eks/cluster_reconciler.go @@ -0,0 +1,338 @@ +package handlers_eks + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "log/slog" + "net/http" + "time" + + "github.com/nats-io/nats.go" +) + +const ( + defaultReconcileLeaseRefresh = 20 * time.Second + defaultReconcileInterval = 30 * time.Second + defaultHealthzTimeout = 5 * time.Second + // defaultCreateTimeout bounds how long a cluster may sit in CREATING before + // the reconciler gives up and marks it FAILED. Measured from meta.CreatedAt + // (absolute), so a VM that never boots reaches a terminal state even across + // daemon restarts. Plan Risk #1. + defaultCreateTimeout = 15 * time.Minute +) + +// ErrReconcilerLeaseLost is returned by Run when the leader lease was lost +// mid-loop. Callers can re-Acquire and re-Run if they want to keep driving +// reconcile from the same node. +var ErrReconcilerLeaseLost = errors.New("eks: reconciler lease lost") + +// ErrReconcilerClusterDeleting is returned by Run when meta.Status is +// observed as DELETING; the DeleteCluster service-impl path is responsible +// for teardown, so the reconciler exits without further work. +var ErrReconcilerClusterDeleting = errors.New("eks: cluster deleting") + +// ErrReconcilerClusterFailed is returned by Run when meta.Status is FAILED. +var ErrReconcilerClusterFailed = errors.New("eks: cluster failed") + +// HTTPDoer is the minimum http.Client surface the reconciler needs for +// /healthz probes; tests stub it without a TLS server. +type HTTPDoer interface { + Do(req *http.Request) (*http.Response, error) +} + +// ClusterReconciler drives one cluster's CREATING → ACTIVE → DELETING +// lifecycle under a leader lease. One instance per (accountID, clusterName); +// the leader lease arbitrates which spinifex node actually drives reconcile. +type ClusterReconciler struct { + leaderKV nats.KeyValue + acctKV nats.KeyValue + accountID string + clusterName string + holderID string + healthURL string + + leaseRefresh time.Duration + interval time.Duration + healthTimeout time.Duration + createTimeout time.Duration + httpClient HTTPDoer +} + +// ReconcilerOption tunes a ClusterReconciler. Tests use the With* helpers to +// shrink tickers and inject a stub HTTP client. +type ReconcilerOption func(*ClusterReconciler) + +// WithReconcileInterval overrides the default reconcile loop period. +func WithReconcileInterval(d time.Duration) ReconcilerOption { + return func(r *ClusterReconciler) { r.interval = d } +} + +// WithLeaseRefresh overrides the default leader-lease refresh period. Should +// be less than KVBucketEKSLeaderTTL. +func WithLeaseRefresh(d time.Duration) ReconcilerOption { + return func(r *ClusterReconciler) { r.leaseRefresh = d } +} + +// WithHealthzTimeout overrides the per-probe HTTP timeout. +func WithHealthzTimeout(d time.Duration) ReconcilerOption { + return func(r *ClusterReconciler) { r.healthTimeout = d } +} + +// WithCreateTimeout overrides how long a cluster may remain CREATING before the +// reconciler marks it FAILED. +func WithCreateTimeout(d time.Duration) ReconcilerOption { + return func(r *ClusterReconciler) { r.createTimeout = d } +} + +// WithHTTPClient injects a stub HTTPDoer (tests). +func WithHTTPClient(c HTTPDoer) ReconcilerOption { + return func(r *ClusterReconciler) { r.httpClient = c } +} + +// NewClusterReconciler constructs a ClusterReconciler. healthURL is the +// fully-qualified probe target (e.g. https://nlb-dns:443/healthz) or empty +// to skip /healthz probing (CREATING transitions on bootstrap state alone). +func NewClusterReconciler(leaderKV, acctKV nats.KeyValue, accountID, clusterName, holderID, healthURL string, opts ...ReconcilerOption) (*ClusterReconciler, error) { + if leaderKV == nil { + return nil, errors.New("eks: NewClusterReconciler nil leaderKV") + } + if acctKV == nil { + return nil, errors.New("eks: NewClusterReconciler nil acctKV") + } + if accountID == "" { + return nil, errors.New("eks: NewClusterReconciler empty accountID") + } + if clusterName == "" { + return nil, errors.New("eks: NewClusterReconciler empty clusterName") + } + if holderID == "" { + return nil, errors.New("eks: NewClusterReconciler empty holderID") + } + r := &ClusterReconciler{ + leaderKV: leaderKV, + acctKV: acctKV, + accountID: accountID, + clusterName: clusterName, + holderID: holderID, + healthURL: healthURL, + leaseRefresh: defaultReconcileLeaseRefresh, + interval: defaultReconcileInterval, + healthTimeout: defaultHealthzTimeout, + createTimeout: defaultCreateTimeout, + httpClient: &http.Client{ + Timeout: defaultHealthzTimeout, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, //nolint:gosec // K3s NLB is self-signed; CA pinning is a v2 hardening item + }, + }, + }, + } + for _, o := range opts { + o(r) + } + return r, nil +} + +func reconcilerLeaderKey(accountID, clusterName string) string { + return accountID + "/" + clusterName +} + +// AcquireLease attempts a CAS Create on the leader bucket. Returns +// (release, true) if this caller is now the leader; (nil, false) otherwise. +// The release closure performs a best-effort kv.Delete; the bucket's 60s +// TTL reaps stale leases when release does not run (crashed leader). +func (r *ClusterReconciler) AcquireLease() (func(), bool) { + key := reconcilerLeaderKey(r.accountID, r.clusterName) + if _, err := r.leaderKV.Create(key, []byte(r.holderID)); err != nil { + slog.Debug("ClusterReconciler: lease held by another holder", + "key", key, "holderID", r.holderID, "err", err) + return nil, false + } + slog.Info("ClusterReconciler: lease acquired", "key", key, "holderID", r.holderID) + return func() { + if err := r.leaderKV.Delete(key); err != nil { + slog.Warn("ClusterReconciler: lease release failed (TTL will reap)", + "key", key, "holderID", r.holderID, "err", err) + } + }, true +} + +// RefreshLease re-asserts ownership via a CAS update on the leader key. +// Returns false if the key was taken by another holder or the CAS conflicts. +func (r *ClusterReconciler) RefreshLease() bool { + key := reconcilerLeaderKey(r.accountID, r.clusterName) + entry, err := r.leaderKV.Get(key) + if err != nil { + slog.Warn("ClusterReconciler: lease get failed", "key", key, "err", err) + return false + } + if string(entry.Value()) != r.holderID { + slog.Info("ClusterReconciler: lease now held by another holder", + "key", key, "holderID", r.holderID, "got", string(entry.Value())) + return false + } + if _, err := r.leaderKV.Update(key, []byte(r.holderID), entry.Revision()); err != nil { + slog.Warn("ClusterReconciler: lease CAS update failed", + "key", key, "holderID", r.holderID, "err", err) + return false + } + return true +} + +// Run drives reconcile until ctx is cancelled, the lease is lost, or the +// cluster reaches a terminal state. Caller is expected to AcquireLease first; +// Run does not re-acquire. +func (r *ClusterReconciler) Run(ctx context.Context) error { + refreshT := time.NewTicker(r.leaseRefresh) + defer refreshT.Stop() + reconcileT := time.NewTicker(r.interval) + defer reconcileT.Stop() + + if err := r.reconcileOnce(ctx); err != nil { + if terminalReconcileErr(err) { + return err + } + slog.Warn("ClusterReconciler: initial reconcile failed", + "cluster", r.clusterName, "err", err) + } + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-refreshT.C: + if !r.RefreshLease() { + return ErrReconcilerLeaseLost + } + case <-reconcileT.C: + if err := r.reconcileOnce(ctx); err != nil { + if terminalReconcileErr(err) { + return err + } + slog.Warn("ClusterReconciler: reconcile failed", + "cluster", r.clusterName, "err", err) + } + } + } +} + +func terminalReconcileErr(err error) bool { + return errors.Is(err, ErrReconcilerClusterDeleting) || + errors.Is(err, ErrReconcilerClusterFailed) || + errors.Is(err, ErrClusterNotFound) +} + +func (r *ClusterReconciler) reconcileOnce(ctx context.Context) error { + meta, err := GetClusterMeta(r.acctKV, r.clusterName) + if err != nil { + return err + } + switch meta.Status { + case ClusterStatusCreating: + ready, reason := r.bootstrapReady(meta) + if !ready { + slog.Debug("ClusterReconciler: bootstrap not ready", + "cluster", r.clusterName, "reason", reason) + return r.failIfCreateTimedOut(meta, "bootstrap not ready: "+reason) + } + if err := r.probeHealthz(ctx); err != nil { + slog.Debug("ClusterReconciler: /healthz still failing", + "cluster", r.clusterName, "err", err) + return r.failIfCreateTimedOut(meta, "healthz not ready: "+err.Error()) + } + if err := SetClusterStatus(r.acctKV, r.clusterName, ClusterStatusActive); err != nil { + return err + } + slog.Info("ClusterReconciler: cluster transitioned to ACTIVE", + "cluster", r.clusterName) + case ClusterStatusActive: + issue := "" + if err := r.probeHealthz(ctx); err != nil { + slog.Warn("ClusterReconciler: /healthz failing for ACTIVE cluster", + "cluster", r.clusterName, "err", err) + issue = err.Error() + } + if err := SetClusterHealth(r.acctKV, r.clusterName, issue); err != nil { + if errors.Is(err, ErrClusterNotFound) { + return ErrClusterNotFound + } + return fmt.Errorf("record cluster health: %w", err) + } + case ClusterStatusDeleting: + return ErrReconcilerClusterDeleting + case ClusterStatusFailed: + return ErrReconcilerClusterFailed + } + return nil +} + +// failIfCreateTimedOut transitions a still-CREATING cluster to FAILED once it +// has exceeded createTimeout since meta.CreatedAt, and returns +// ErrReconcilerClusterFailed so Run exits the loop. While still within the +// window it returns nil so the next tick retries. A zero or absent CreatedAt +// disables the deadline (no false-positive fail on a malformed record). +func (r *ClusterReconciler) failIfCreateTimedOut(meta *ClusterMeta, reason string) error { + if r.createTimeout <= 0 || meta.CreatedAt.IsZero() { + return nil + } + if time.Since(meta.CreatedAt) < r.createTimeout { + return nil + } + failReason := fmt.Sprintf("cluster did not become ACTIVE within %s: %s", r.createTimeout, reason) + if err := MarkClusterFailed(r.acctKV, r.clusterName, failReason); err != nil { + if errors.Is(err, ErrClusterNotFound) { + return ErrClusterNotFound + } + return fmt.Errorf("mark create-timeout failed: %w", err) + } + slog.Warn("ClusterReconciler: CREATING timed out, marked FAILED", + "cluster", r.clusterName, "timeout", r.createTimeout, "reason", reason) + return ErrReconcilerClusterFailed +} + +// bootstrapReady returns true once the four NATS bootstrap KV artifacts are +// present: node token, admin kubeconfig, the OIDC JWKS verified-marker, and the +// CA on meta. The JWKS gate is the verified-marker (written only after the VM's +// published JWKS passes the controller cross-check), NOT the pre-seeded +// OIDCJWKSKey — so a cluster never transitions to ACTIVE on an unverified or +// mismatched signing key. +func (r *ClusterReconciler) bootstrapReady(meta *ClusterMeta) (bool, string) { + if meta.CertificateAuthorityB64 == "" { + return false, "CA absent on meta" + } + keys := []string{ + NodeTokenKey(r.clusterName), + AdminKubeconfigKey(r.clusterName), + OIDCJWKSVerifiedKey(r.clusterName), + } + for _, k := range keys { + if _, err := r.acctKV.Get(k); err != nil { + return false, fmt.Sprintf("%s missing: %s", k, err) + } + } + return true, "" +} + +func (r *ClusterReconciler) probeHealthz(ctx context.Context) error { + if r.healthURL == "" { + return nil + } + probeCtx, cancel := context.WithTimeout(ctx, r.healthTimeout) + defer cancel() + req, err := http.NewRequestWithContext(probeCtx, http.MethodGet, r.healthURL, nil) + if err != nil { + return fmt.Errorf("new request: %w", err) + } + resp, err := r.httpClient.Do(req) + if err != nil { + return fmt.Errorf("healthz request: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("eks: /healthz status %d", resp.StatusCode) + } + return nil +} diff --git a/spinifex/handlers/eks/cluster_reconciler_test.go b/spinifex/handlers/eks/cluster_reconciler_test.go new file mode 100644 index 000000000..f327e5571 --- /dev/null +++ b/spinifex/handlers/eks/cluster_reconciler_test.go @@ -0,0 +1,469 @@ +package handlers_eks + +import ( + "context" + "errors" + "io" + "net/http" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/nats-io/nats.go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// stubHTTPDoer is a fake HTTPDoer that returns a configurable status code + +// optional error, and tracks the number of calls for assertion. +type stubHTTPDoer struct { + mu sync.Mutex + calls atomic.Int32 + status int + err error +} + +func (s *stubHTTPDoer) Do(_ *http.Request) (*http.Response, error) { + s.calls.Add(1) + s.mu.Lock() + err := s.err + status := s.status + s.mu.Unlock() + if err != nil { + return nil, err + } + return &http.Response{StatusCode: status, Body: io.NopCloser(strings.NewReader(""))}, nil +} + +func (s *stubHTTPDoer) setResponse(status int, err error) { + s.mu.Lock() + defer s.mu.Unlock() + s.status = status + s.err = err +} + +func newReconcilerHarness(t *testing.T, healthURL string, opts ...ReconcilerOption) (*ClusterReconciler, nats.KeyValue, nats.KeyValue) { + t.Helper() + _, _, js := testutil.StartTestJetStream(t) + leaderKV, err := InitLeaderBucket(js) + require.NoError(t, err) + acctKV, err := GetOrCreateAccountBucket(js, testAccountID) + require.NoError(t, err) + require.NoError(t, PutClusterMeta(acctKV, sampleClusterMeta("alpha"))) + + r, err := NewClusterReconciler(leaderKV, acctKV, testAccountID, "alpha", "holder-1", healthURL, opts...) + require.NoError(t, err) + return r, leaderKV, acctKV +} + +func TestNewClusterReconciler_EmptyInputsRejected(t *testing.T) { + _, _, js := testutil.StartTestJetStream(t) + leaderKV, err := InitLeaderBucket(js) + require.NoError(t, err) + acctKV, err := GetOrCreateAccountBucket(js, testAccountID) + require.NoError(t, err) + + _, err = NewClusterReconciler(nil, acctKV, testAccountID, "alpha", "h", "") + require.Error(t, err) + _, err = NewClusterReconciler(leaderKV, nil, testAccountID, "alpha", "h", "") + require.Error(t, err) + _, err = NewClusterReconciler(leaderKV, acctKV, "", "alpha", "h", "") + require.Error(t, err) + _, err = NewClusterReconciler(leaderKV, acctKV, testAccountID, "", "h", "") + require.Error(t, err) + _, err = NewClusterReconciler(leaderKV, acctKV, testAccountID, "alpha", "", "") + require.Error(t, err) +} + +func TestClusterReconciler_AcquireLeaseFirstHolderWins(t *testing.T) { + r, _, _ := newReconcilerHarness(t, "") + + release, ok := r.AcquireLease() + require.True(t, ok) + require.NotNil(t, release) + defer release() + + // Second AcquireLease from same holder must fail (Create returns KeyExists). + release2, ok2 := r.AcquireLease() + assert.False(t, ok2) + assert.Nil(t, release2) +} + +func TestClusterReconciler_AcquireLeaseSecondHolderLoses(t *testing.T) { + r1, leaderKV, acctKV := newReconcilerHarness(t, "") + + release, ok := r1.AcquireLease() + require.True(t, ok) + defer release() + + r2, err := NewClusterReconciler(leaderKV, acctKV, testAccountID, "alpha", "holder-2", "") + require.NoError(t, err) + release2, ok2 := r2.AcquireLease() + assert.False(t, ok2) + assert.Nil(t, release2) +} + +func TestClusterReconciler_RefreshLeaseFailsAfterStolen(t *testing.T) { + r, leaderKV, _ := newReconcilerHarness(t, "") + + release, ok := r.AcquireLease() + require.True(t, ok) + defer release() + + assert.True(t, r.RefreshLease(), "refresh should succeed while we own the key") + + // Steal the key by overwriting its value via Put. + _, err := leaderKV.Put(reconcilerLeaderKey(testAccountID, "alpha"), []byte("holder-2")) + require.NoError(t, err) + + assert.False(t, r.RefreshLease(), "refresh should fail after another holder stole the key") +} + +func TestClusterReconciler_RefreshLeaseFailsWhenKeyDeleted(t *testing.T) { + r, leaderKV, _ := newReconcilerHarness(t, "") + release, ok := r.AcquireLease() + require.True(t, ok) + defer release() + + require.NoError(t, leaderKV.Delete(reconcilerLeaderKey(testAccountID, "alpha"))) + assert.False(t, r.RefreshLease()) +} + +// freshenClusterCreatedAt re-stamps the cluster meta's CreatedAt to now so the +// reconciler's CREATING deadline is far from expiry. sampleClusterMeta uses a +// fixed past date, which would otherwise trip the create-timeout in tests that +// assert the cluster stays CREATING. +func freshenClusterCreatedAt(t *testing.T, kv nats.KeyValue) { + t.Helper() + meta, err := GetClusterMeta(kv, "alpha") + require.NoError(t, err) + meta.CreatedAt = time.Now().UTC() + require.NoError(t, PutClusterMeta(kv, meta)) +} + +func seedBootstrapState(t *testing.T, kv nats.KeyValue) { + t.Helper() + _, err := kv.Put(NodeTokenKey("alpha"), []byte("enc-token")) + require.NoError(t, err) + _, err = kv.Put(AdminKubeconfigKey("alpha"), []byte("enc-kc")) + require.NoError(t, err) + // Pre-seeded JWKS plus the verified-marker persistJWKS writes on a passing + // cross-check — bootstrapReady gates ACTIVE on the marker, not the seed. + _, err = kv.Put(OIDCJWKSKey("alpha"), []byte(`{"keys":[]}`)) + require.NoError(t, err) + _, err = kv.Put(OIDCJWKSVerifiedKey("alpha"), []byte("verified")) + require.NoError(t, err) + require.NoError(t, SetClusterCertificateAuthority(kv, "alpha", "ca-blob-b64")) +} + +func TestClusterReconciler_CreatingTransitionsToActiveOnReadyAndHealthz(t *testing.T) { + stub := &stubHTTPDoer{status: http.StatusOK} + r, _, acctKV := newReconcilerHarness(t, + "https://nlb.example/healthz", + WithHTTPClient(stub), + WithReconcileInterval(10*time.Millisecond), + WithLeaseRefresh(10*time.Second), + ) + release, ok := r.AcquireLease() + require.True(t, ok) + defer release() + + seedBootstrapState(t, acctKV) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + runErr := make(chan error, 1) + go func() { runErr <- r.Run(ctx) }() + + require.Eventually(t, func() bool { + meta, err := GetClusterMeta(acctKV, "alpha") + if err != nil { + return false + } + return meta.Status == ClusterStatusActive + }, 1500*time.Millisecond, 10*time.Millisecond, "CREATING should transition to ACTIVE") + + cancel() + select { + case err := <-runErr: + assert.ErrorIs(t, err, context.Canceled) + case <-time.After(2 * time.Second): + t.Fatal("Run did not return after cancel") + } + assert.Positive(t, stub.calls.Load(), "/healthz should have been probed at least once") +} + +func TestClusterReconciler_CreatingStaysWhenBootstrapMissing(t *testing.T) { + stub := &stubHTTPDoer{status: http.StatusOK} + r, _, acctKV := newReconcilerHarness(t, + "https://nlb.example/healthz", + WithHTTPClient(stub), + WithReconcileInterval(10*time.Millisecond), + WithLeaseRefresh(10*time.Second), + ) + release, ok := r.AcquireLease() + require.True(t, ok) + defer release() + + freshenClusterCreatedAt(t, acctKV) + + // Only seed two of the three KV keys (no CA, no JWKS). + _, err := acctKV.Put(NodeTokenKey("alpha"), []byte("enc-token")) + require.NoError(t, err) + _, err = acctKV.Put(AdminKubeconfigKey("alpha"), []byte("enc-kc")) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond) + defer cancel() + _ = r.Run(ctx) + + meta, err := GetClusterMeta(acctKV, "alpha") + require.NoError(t, err) + assert.Equal(t, ClusterStatusCreating, meta.Status, "must remain CREATING when bootstrap incomplete") +} + +func TestClusterReconciler_CreatingStaysWhenJWKSUnverified(t *testing.T) { + // Regression for the ACTIVE-on-pre-seeded-JWKS race: every artifact is + // present INCLUDING the controller-pre-seeded OIDCJWKSKey, but the VM's + // JWKS cross-check has not passed (no verified-marker). The cluster must + // stay CREATING — reaching ACTIVE here would mean signing IRSA tokens under + // a key the controller never confirmed. + stub := &stubHTTPDoer{status: http.StatusOK} + r, _, acctKV := newReconcilerHarness(t, + "https://nlb.example/healthz", + WithHTTPClient(stub), + WithReconcileInterval(10*time.Millisecond), + WithLeaseRefresh(10*time.Second), + ) + release, ok := r.AcquireLease() + require.True(t, ok) + defer release() + + freshenClusterCreatedAt(t, acctKV) + + _, err := acctKV.Put(NodeTokenKey("alpha"), []byte("enc-token")) + require.NoError(t, err) + _, err = acctKV.Put(AdminKubeconfigKey("alpha"), []byte("enc-kc")) + require.NoError(t, err) + require.NoError(t, SetClusterCertificateAuthority(acctKV, "alpha", "ca-blob-b64")) + // Pre-seeded JWKS present, but NO verified-marker. + _, err = acctKV.Put(OIDCJWKSKey("alpha"), []byte(`{"keys":[]}`)) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond) + defer cancel() + _ = r.Run(ctx) + + meta, err := GetClusterMeta(acctKV, "alpha") + require.NoError(t, err) + assert.Equal(t, ClusterStatusCreating, meta.Status, + "must remain CREATING until the JWKS cross-check positively passes") +} + +func TestClusterReconciler_CreatingStaysWhenHealthzFails(t *testing.T) { + stub := &stubHTTPDoer{err: errors.New("connection refused")} + r, _, acctKV := newReconcilerHarness(t, + "https://nlb.example/healthz", + WithHTTPClient(stub), + WithReconcileInterval(10*time.Millisecond), + WithLeaseRefresh(10*time.Second), + ) + release, ok := r.AcquireLease() + require.True(t, ok) + defer release() + + freshenClusterCreatedAt(t, acctKV) + seedBootstrapState(t, acctKV) + + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond) + defer cancel() + _ = r.Run(ctx) + + meta, err := GetClusterMeta(acctKV, "alpha") + require.NoError(t, err) + assert.Equal(t, ClusterStatusCreating, meta.Status, "must remain CREATING when /healthz fails") +} + +func TestClusterReconciler_CreatingTimesOutToFailed(t *testing.T) { + stub := &stubHTTPDoer{err: errors.New("connection refused")} + r, _, acctKV := newReconcilerHarness(t, + "https://nlb.example/healthz", + WithHTTPClient(stub), + WithReconcileInterval(10*time.Millisecond), + WithLeaseRefresh(10*time.Second), + WithCreateTimeout(1*time.Millisecond), + ) + release, ok := r.AcquireLease() + require.True(t, ok) + defer release() + + // Stamp CreatedAt in the past so the 1ms deadline is already exceeded; the + // cluster never becomes ready, so the reconciler must mark it FAILED. + meta, err := GetClusterMeta(acctKV, "alpha") + require.NoError(t, err) + meta.CreatedAt = time.Now().Add(-time.Hour).UTC() + require.NoError(t, PutClusterMeta(acctKV, meta)) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + err = r.Run(ctx) + assert.ErrorIs(t, err, ErrReconcilerClusterFailed) + + got, err := GetClusterMeta(acctKV, "alpha") + require.NoError(t, err) + assert.Equal(t, ClusterStatusFailed, got.Status) + assert.Contains(t, got.StatusReason, "did not become ACTIVE") +} + +func TestClusterReconciler_CreatingWithinDeadlineStaysCreating(t *testing.T) { + stub := &stubHTTPDoer{err: errors.New("connection refused")} + r, _, acctKV := newReconcilerHarness(t, + "https://nlb.example/healthz", + WithHTTPClient(stub), + WithReconcileInterval(10*time.Millisecond), + WithLeaseRefresh(10*time.Second), + WithCreateTimeout(time.Hour), + ) + release, ok := r.AcquireLease() + require.True(t, ok) + defer release() + + freshenClusterCreatedAt(t, acctKV) + + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + _ = r.Run(ctx) + + meta, err := GetClusterMeta(acctKV, "alpha") + require.NoError(t, err) + assert.Equal(t, ClusterStatusCreating, meta.Status, "within deadline must stay CREATING") +} + +func TestClusterReconciler_DeletingExitsLoop(t *testing.T) { + stub := &stubHTTPDoer{status: http.StatusOK} + r, _, acctKV := newReconcilerHarness(t, + "https://nlb.example/healthz", + WithHTTPClient(stub), + WithReconcileInterval(10*time.Millisecond), + WithLeaseRefresh(10*time.Second), + ) + release, ok := r.AcquireLease() + require.True(t, ok) + defer release() + + require.NoError(t, SetClusterStatus(acctKV, "alpha", ClusterStatusDeleting)) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + err := r.Run(ctx) + assert.ErrorIs(t, err, ErrReconcilerClusterDeleting) +} + +func TestClusterReconciler_LostLeaseExitsLoop(t *testing.T) { + stub := &stubHTTPDoer{status: http.StatusOK} + r, leaderKV, acctKV := newReconcilerHarness(t, + "https://nlb.example/healthz", + WithHTTPClient(stub), + WithReconcileInterval(10*time.Second), + WithLeaseRefresh(20*time.Millisecond), + ) + release, ok := r.AcquireLease() + require.True(t, ok) + defer release() + + seedBootstrapState(t, acctKV) + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + runErr := make(chan error, 1) + go func() { runErr <- r.Run(ctx) }() + + // Steal the lease in another holder's name. The next refresh tick should + // notice and return ErrReconcilerLeaseLost. + time.Sleep(40 * time.Millisecond) + _, err := leaderKV.Put(reconcilerLeaderKey(testAccountID, "alpha"), []byte("holder-2")) + require.NoError(t, err) + + select { + case err := <-runErr: + assert.ErrorIs(t, err, ErrReconcilerLeaseLost) + case <-time.After(2 * time.Second): + t.Fatal("Run did not exit after lease loss") + } +} + +func TestClusterReconciler_ActiveProbesAndWarnsOnHealthzFail(t *testing.T) { + stub := &stubHTTPDoer{status: http.StatusOK} + r, _, acctKV := newReconcilerHarness(t, + "https://nlb.example/healthz", + WithHTTPClient(stub), + WithReconcileInterval(10*time.Millisecond), + WithLeaseRefresh(10*time.Second), + ) + release, ok := r.AcquireLease() + require.True(t, ok) + defer release() + + require.NoError(t, SetClusterStatus(acctKV, "alpha", ClusterStatusActive)) + stub.setResponse(http.StatusServiceUnavailable, nil) + + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + _ = r.Run(ctx) + + meta, err := GetClusterMeta(acctKV, "alpha") + require.NoError(t, err) + assert.Equal(t, ClusterStatusActive, meta.Status, "ACTIVE must not flip on healthz fail in this PR") + assert.Positive(t, stub.calls.Load()) + assert.NotEmpty(t, meta.HealthIssue, "failing /healthz must be recorded on meta") + assert.False(t, meta.LastHealthProbe.IsZero(), "health probe time must be stamped") +} + +func TestClusterReconciler_ActiveHealthRecoversClearsIssue(t *testing.T) { + stub := &stubHTTPDoer{status: http.StatusServiceUnavailable} + r, _, acctKV := newReconcilerHarness(t, + "https://nlb.example/healthz", + WithHTTPClient(stub), + WithReconcileInterval(10*time.Millisecond), + WithLeaseRefresh(10*time.Second), + ) + release, ok := r.AcquireLease() + require.True(t, ok) + defer release() + + require.NoError(t, SetClusterStatus(acctKV, "alpha", ClusterStatusActive)) + + // First probe fails and records the issue. + require.Error(t, r.probeHealthz(context.Background())) + require.NoError(t, r.reconcileOnce(context.Background())) + meta, err := GetClusterMeta(acctKV, "alpha") + require.NoError(t, err) + require.NotEmpty(t, meta.HealthIssue) + + // Probe recovers: the issue must be cleared. + stub.setResponse(http.StatusOK, nil) + require.NoError(t, r.reconcileOnce(context.Background())) + meta, err = GetClusterMeta(acctKV, "alpha") + require.NoError(t, err) + assert.Empty(t, meta.HealthIssue, "recovered /healthz must clear the recorded issue") +} + +func TestClusterReconciler_ProbeHealthzEmptyURLNoop(t *testing.T) { + stub := &stubHTTPDoer{} + r, _, _ := newReconcilerHarness(t, + "", + WithHTTPClient(stub), + WithReconcileInterval(10*time.Millisecond), + WithLeaseRefresh(10*time.Second), + ) + + require.NoError(t, r.probeHealthz(context.Background())) + assert.EqualValues(t, 0, stub.calls.Load(), "no HTTP call when URL empty") +} diff --git a/spinifex/handlers/eks/cluster_state.go b/spinifex/handlers/eks/cluster_state.go new file mode 100644 index 000000000..768ffc10e --- /dev/null +++ b/spinifex/handlers/eks/cluster_state.go @@ -0,0 +1,257 @@ +package handlers_eks + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/nats-io/nats.go" +) + +// ClusterStatus is the persisted lifecycle state for a cluster. Values match +// the AWS EKS Cluster.status string set so DescribeCluster can pass them +// through verbatim. +type ClusterStatus string + +const ( + ClusterStatusCreating ClusterStatus = "CREATING" + ClusterStatusActive ClusterStatus = "ACTIVE" + ClusterStatusDeleting ClusterStatus = "DELETING" + ClusterStatusFailed ClusterStatus = "FAILED" +) + +// ClusterVpcConfig is the persisted subset of eks.VpcConfigResponse that the +// reconciler + DescribeCluster need at-rest. +type ClusterVpcConfig struct { + SubnetIds []string `json:"subnetIds"` + SecurityGroupIds []string `json:"securityGroupIds,omitempty"` + VpcId string `json:"vpcId,omitempty"` +} + +// ClusterMeta is the persisted control-plane record for one cluster. The +// blob lives at ClusterMetaKey(name) inside the per-account KV bucket and is +// the source of truth for DescribeCluster. +type ClusterMeta struct { + Name string `json:"name"` + Arn string `json:"arn"` + Status ClusterStatus `json:"status"` + StatusReason string `json:"statusReason,omitempty"` + Version string `json:"version"` + RoleArn string `json:"roleArn"` + Endpoint string `json:"endpoint,omitempty"` + OIDCIssuer string `json:"oidcIssuer,omitempty"` + CertificateAuthorityB64 string `json:"certificateAuthorityB64,omitempty"` + ResourcesVpcConfig *ClusterVpcConfig `json:"resourcesVpcConfig,omitempty"` + ControlPlaneInstanceID string `json:"controlPlaneInstanceId,omitempty"` + ControlPlaneENIID string `json:"controlPlaneEniId,omitempty"` + ControlPlaneENIIP string `json:"controlPlaneEniIp,omitempty"` + // ControlPlaneMgmtIP is the control-plane VM's br-mgmt NIC address. Until + // authoritative DNS (Eclipso/Route53) lands, the host-side reconciler cannot + // resolve or route to the VPC-internal NLB DNS endpoint, so it probes + // /healthz on this host-reachable address instead. + ControlPlaneMgmtIP string `json:"controlPlaneMgmtIp,omitempty"` + // EgressEIPAllocationID / EgressEIPPublicIP track the hidden pool address + // SNAT'd to the control-plane VM for egress-only internet (image pulls). + // Released + the snat removed on DeleteCluster. + EgressEIPAllocationID string `json:"egressEipAllocationId,omitempty"` + EgressEIPPublicIP string `json:"egressEipPublicIp,omitempty"` + NLBArn string `json:"nlbArn,omitempty"` + NLBTargetGroupArn string `json:"nlbTargetGroupArn,omitempty"` + CreatedAt time.Time `json:"createdAt"` + // HealthIssue is the last /healthz probe failure reason for an ACTIVE + // cluster; empty means healthy. DescribeCluster surfaces a non-empty value + // as a ClusterHealth issue so a dead control plane is visible behind the + // still-ACTIVE status. LastHealthProbe stamps when the health state changed. + HealthIssue string `json:"healthIssue,omitempty"` + LastHealthProbe time.Time `json:"lastHealthProbe,omitzero"` +} + +// ErrClusterNotFound is returned by GetClusterMeta / SetClusterStatus / +// DeleteClusterPrefix when the cluster meta key is absent. Callers translate +// to the AWS shape (ResourceNotFoundException) at the service boundary. +var ErrClusterNotFound = errors.New("eks: cluster not found") + +const maxClusterStateCASRetries = 5 + +// PutClusterMeta writes the meta record unconditionally. Used at +// CreateCluster time to lay down the initial CREATING record. +func PutClusterMeta(kv nats.KeyValue, meta *ClusterMeta) error { + if meta == nil { + return errors.New("eks: PutClusterMeta nil meta") + } + if meta.Name == "" { + return errors.New("eks: PutClusterMeta meta.Name empty") + } + data, err := json.Marshal(meta) + if err != nil { + return fmt.Errorf("marshal cluster meta %s: %w", meta.Name, err) + } + if _, err := kv.Put(ClusterMetaKey(meta.Name), data); err != nil { + return fmt.Errorf("kv put %s: %w", ClusterMetaKey(meta.Name), err) + } + return nil +} + +// GetClusterMeta reads the meta record. Returns ErrClusterNotFound if the +// key is absent. +func GetClusterMeta(kv nats.KeyValue, name string) (*ClusterMeta, error) { + if name == "" { + return nil, errors.New("eks: GetClusterMeta empty name") + } + entry, err := kv.Get(ClusterMetaKey(name)) + if err != nil { + if errors.Is(err, nats.ErrKeyNotFound) { + return nil, ErrClusterNotFound + } + return nil, fmt.Errorf("kv get %s: %w", ClusterMetaKey(name), err) + } + var meta ClusterMeta + if err := json.Unmarshal(entry.Value(), &meta); err != nil { + return nil, fmt.Errorf("unmarshal cluster meta %s: %w", name, err) + } + return &meta, nil +} + +// casUpdateMeta does a revision-checked read-modify-write of the cluster meta +// record. mutate receives the decoded meta and returns true if it changed a +// field (false short-circuits to a no-op success). Retries on KV CAS conflict +// (concurrent reconciler write) up to maxClusterStateCASRetries. Returns +// ErrClusterNotFound if the meta record was deleted underneath us. +func casUpdateMeta(kv nats.KeyValue, name string, mutate func(*ClusterMeta) bool) error { + if name == "" { + return errors.New("eks: casUpdateMeta empty name") + } + for range maxClusterStateCASRetries { + entry, err := kv.Get(ClusterMetaKey(name)) + if err != nil { + if errors.Is(err, nats.ErrKeyNotFound) { + return ErrClusterNotFound + } + return fmt.Errorf("kv get %s: %w", ClusterMetaKey(name), err) + } + var meta ClusterMeta + if err := json.Unmarshal(entry.Value(), &meta); err != nil { + return fmt.Errorf("unmarshal cluster meta %s: %w", name, err) + } + if !mutate(&meta) { + return nil + } + data, err := json.Marshal(&meta) + if err != nil { + return fmt.Errorf("marshal cluster meta %s: %w", name, err) + } + _, err = kv.Update(ClusterMetaKey(name), data, entry.Revision()) + if err == nil { + return nil + } + if errors.Is(err, nats.ErrKeyExists) { + continue + } + return fmt.Errorf("kv update %s: %w", ClusterMetaKey(name), err) + } + return fmt.Errorf("eks: casUpdateMeta %s exhausted CAS retries", name) +} + +// SetClusterStatus does a revision-checked update of the meta.Status field. +// Returns ErrClusterNotFound if the meta record was deleted underneath us. +func SetClusterStatus(kv nats.KeyValue, name string, status ClusterStatus) error { + if name == "" { + return errors.New("eks: SetClusterStatus empty name") + } + return casUpdateMeta(kv, name, func(m *ClusterMeta) bool { + if m.Status == status { + return false + } + m.Status = status + return true + }) +} + +// MarkClusterFailed transitions a cluster to FAILED with a human-readable +// reason, but only from CREATING — a cluster already DELETING/ACTIVE/FAILED is +// left untouched so a late bootstrap error cannot clobber a concurrent delete +// or an already-healthy cluster. Returns ErrClusterNotFound if the meta record +// was deleted underneath us. +func MarkClusterFailed(kv nats.KeyValue, name, reason string) error { + if name == "" { + return errors.New("eks: MarkClusterFailed empty name") + } + return casUpdateMeta(kv, name, func(m *ClusterMeta) bool { + if m.Status != ClusterStatusCreating { + return false + } + m.Status = ClusterStatusFailed + m.StatusReason = reason + return true + }) +} + +// SetClusterCertificateAuthority does a revision-checked update of the +// meta.CertificateAuthorityB64 field. The NATS bootstrap subscriber calls +// this once the K3s server VM publishes its CA on the bootstrap bus. Returns +// ErrClusterNotFound if the meta record was deleted underneath us. +func SetClusterCertificateAuthority(kv nats.KeyValue, name, caB64 string) error { + if name == "" { + return errors.New("eks: SetClusterCertificateAuthority empty name") + } + if caB64 == "" { + return errors.New("eks: SetClusterCertificateAuthority empty CA") + } + return casUpdateMeta(kv, name, func(m *ClusterMeta) bool { + if m.CertificateAuthorityB64 == caB64 { + return false + } + m.CertificateAuthorityB64 = caB64 + return true + }) +} + +// SetClusterHealth records the latest /healthz probe outcome for a cluster: +// issue is the failure reason (empty clears it / marks healthy). Writes only +// when the health state changes, stamping LastHealthProbe at the transition so +// a persistently-unhealthy cluster carries the time it first went bad rather +// than churning the KV every probe interval. Returns ErrClusterNotFound if the +// meta record was deleted underneath us. +func SetClusterHealth(kv nats.KeyValue, name, issue string) error { + if name == "" { + return errors.New("eks: SetClusterHealth empty name") + } + return casUpdateMeta(kv, name, func(m *ClusterMeta) bool { + if m.HealthIssue == issue { + return false + } + m.HealthIssue = issue + m.LastHealthProbe = time.Now().UTC() + return true + }) +} + +// DeleteClusterPrefix removes every KV key under clusters/{name}/. Called +// from DeleteCluster after the OIDC private key has been zeroized. +// Returns the first delete error encountered but continues sweeping so +// best-effort cleanup proceeds. +func DeleteClusterPrefix(kv nats.KeyValue, name string) error { + if name == "" { + return errors.New("eks: DeleteClusterPrefix empty name") + } + prefix := fmt.Sprintf("clusters/%s/", name) + keys, err := kv.Keys() + if err != nil { + if errors.Is(err, nats.ErrNoKeysFound) { + return nil + } + return fmt.Errorf("kv keys: %w", err) + } + var firstErr error + for _, k := range keys { + if !strings.HasPrefix(k, prefix) { + continue + } + if err := kv.Delete(k); err != nil && firstErr == nil { + firstErr = fmt.Errorf("kv delete %s: %w", k, err) + } + } + return firstErr +} diff --git a/spinifex/handlers/eks/cluster_state_test.go b/spinifex/handlers/eks/cluster_state_test.go new file mode 100644 index 000000000..d25d2ca52 --- /dev/null +++ b/spinifex/handlers/eks/cluster_state_test.go @@ -0,0 +1,256 @@ +package handlers_eks + +import ( + "encoding/json" + "errors" + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/nats-io/nats.go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newClusterStateTestKV(t *testing.T) nats.KeyValue { + t.Helper() + _, _, js := testutil.StartTestJetStream(t) + kv, err := GetOrCreateAccountBucket(js, testAccountID) + require.NoError(t, err) + return kv +} + +func sampleClusterMeta(name string) *ClusterMeta { + return &ClusterMeta{ + Name: name, + Arn: "arn:aws:eks:us-east-1:111122223333:cluster/" + name, + Status: ClusterStatusCreating, + Version: "1.32", + RoleArn: "arn:aws:iam::111122223333:role/eks-cluster", + ResourcesVpcConfig: &ClusterVpcConfig{ + SubnetIds: []string{"subnet-aaa"}, + SecurityGroupIds: []string{"sg-aaa"}, + }, + CreatedAt: time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC), + } +} + +func TestPutClusterMeta_NilOrEmptyNameRejected(t *testing.T) { + kv := newClusterStateTestKV(t) + + require.Error(t, PutClusterMeta(kv, nil)) + require.Error(t, PutClusterMeta(kv, &ClusterMeta{})) +} + +func TestPutClusterMeta_RoundTrip(t *testing.T) { + kv := newClusterStateTestKV(t) + + in := sampleClusterMeta("alpha") + require.NoError(t, PutClusterMeta(kv, in)) + + got, err := GetClusterMeta(kv, "alpha") + require.NoError(t, err) + assert.Equal(t, in.Name, got.Name) + assert.Equal(t, in.Arn, got.Arn) + assert.Equal(t, in.Status, got.Status) + assert.Equal(t, in.Version, got.Version) + assert.Equal(t, in.RoleArn, got.RoleArn) + require.NotNil(t, got.ResourcesVpcConfig) + assert.Equal(t, []string{"subnet-aaa"}, got.ResourcesVpcConfig.SubnetIds) + assert.True(t, got.CreatedAt.Equal(in.CreatedAt)) +} + +func TestGetClusterMeta_MissingReturnsErrClusterNotFound(t *testing.T) { + kv := newClusterStateTestKV(t) + + _, err := GetClusterMeta(kv, "missing") + require.ErrorIs(t, err, ErrClusterNotFound) +} + +func TestGetClusterMeta_EmptyNameRejected(t *testing.T) { + kv := newClusterStateTestKV(t) + + _, err := GetClusterMeta(kv, "") + require.Error(t, err) + assert.NotErrorIs(t, err, ErrClusterNotFound) +} + +func TestGetClusterMeta_CorruptBlobRejected(t *testing.T) { + kv := newClusterStateTestKV(t) + + _, err := kv.Put(ClusterMetaKey("corrupt"), []byte("{not json")) + require.NoError(t, err) + + _, err = GetClusterMeta(kv, "corrupt") + require.Error(t, err) + assert.NotErrorIs(t, err, ErrClusterNotFound) +} + +func TestSetClusterStatus_TransitionsAndIsIdempotent(t *testing.T) { + kv := newClusterStateTestKV(t) + require.NoError(t, PutClusterMeta(kv, sampleClusterMeta("alpha"))) + + require.NoError(t, SetClusterStatus(kv, "alpha", ClusterStatusActive)) + got, err := GetClusterMeta(kv, "alpha") + require.NoError(t, err) + assert.Equal(t, ClusterStatusActive, got.Status) + + require.NoError(t, SetClusterStatus(kv, "alpha", ClusterStatusActive)) + got, err = GetClusterMeta(kv, "alpha") + require.NoError(t, err) + assert.Equal(t, ClusterStatusActive, got.Status) +} + +func TestSetClusterStatus_DeletingAlwaysAllowedFromAnyState(t *testing.T) { + kv := newClusterStateTestKV(t) + + for _, from := range []ClusterStatus{ClusterStatusCreating, ClusterStatusActive, ClusterStatusFailed} { + meta := sampleClusterMeta("alpha") + meta.Status = from + require.NoError(t, PutClusterMeta(kv, meta)) + require.NoError(t, SetClusterStatus(kv, "alpha", ClusterStatusDeleting)) + got, err := GetClusterMeta(kv, "alpha") + require.NoError(t, err) + assert.Equal(t, ClusterStatusDeleting, got.Status, "from=%s", from) + } +} + +func TestSetClusterStatus_MissingReturnsErrClusterNotFound(t *testing.T) { + kv := newClusterStateTestKV(t) + + err := SetClusterStatus(kv, "missing", ClusterStatusActive) + require.ErrorIs(t, err, ErrClusterNotFound) +} + +func TestSetClusterStatus_RecoversFromConcurrentRevisionBump(t *testing.T) { + kv := newClusterStateTestKV(t) + require.NoError(t, PutClusterMeta(kv, sampleClusterMeta("alpha"))) + + // Read once to get a stale revision, then bump the record from another + // goroutine via Put to force one CAS conflict on the next Update. + entry, err := kv.Get(ClusterMetaKey("alpha")) + require.NoError(t, err) + + bumped := sampleClusterMeta("alpha") + bumped.Version = "1.32-bumped" + data, err := json.Marshal(bumped) + require.NoError(t, err) + _, err = kv.Update(ClusterMetaKey("alpha"), data, entry.Revision()) + require.NoError(t, err) + + // SetClusterStatus must succeed even though there has been an intervening + // revision bump — its internal loop re-reads + retries. + require.NoError(t, SetClusterStatus(kv, "alpha", ClusterStatusActive)) + got, err := GetClusterMeta(kv, "alpha") + require.NoError(t, err) + assert.Equal(t, ClusterStatusActive, got.Status) + assert.Equal(t, "1.32-bumped", got.Version) +} + +func TestMarkClusterFailed_FromCreatingSetsStatusAndReason(t *testing.T) { + kv := newClusterStateTestKV(t) + require.NoError(t, PutClusterMeta(kv, sampleClusterMeta("alpha"))) + + require.NoError(t, MarkClusterFailed(kv, "alpha", "bootstrap failed: boom")) + got, err := GetClusterMeta(kv, "alpha") + require.NoError(t, err) + assert.Equal(t, ClusterStatusFailed, got.Status) + assert.Equal(t, "bootstrap failed: boom", got.StatusReason) +} + +func TestMarkClusterFailed_NoopFromNonCreating(t *testing.T) { + kv := newClusterStateTestKV(t) + + for _, from := range []ClusterStatus{ClusterStatusActive, ClusterStatusDeleting, ClusterStatusFailed} { + meta := sampleClusterMeta("alpha") + meta.Status = from + require.NoError(t, PutClusterMeta(kv, meta)) + + require.NoError(t, MarkClusterFailed(kv, "alpha", "late bootstrap error")) + got, err := GetClusterMeta(kv, "alpha") + require.NoError(t, err) + assert.Equal(t, from, got.Status, "from=%s must be untouched", from) + assert.Empty(t, got.StatusReason, "from=%s reason must not be set", from) + } +} + +func TestMarkClusterFailed_MissingReturnsErrClusterNotFound(t *testing.T) { + kv := newClusterStateTestKV(t) + + err := MarkClusterFailed(kv, "ghost", "boom") + require.ErrorIs(t, err, ErrClusterNotFound) +} + +func TestSetClusterCertificateAuthority_WritesAndIsIdempotent(t *testing.T) { + kv := newClusterStateTestKV(t) + require.NoError(t, PutClusterMeta(kv, sampleClusterMeta("alpha"))) + + require.NoError(t, SetClusterCertificateAuthority(kv, "alpha", "ca-blob-b64")) + got, err := GetClusterMeta(kv, "alpha") + require.NoError(t, err) + assert.Equal(t, "ca-blob-b64", got.CertificateAuthorityB64) + + require.NoError(t, SetClusterCertificateAuthority(kv, "alpha", "ca-blob-b64")) + got, err = GetClusterMeta(kv, "alpha") + require.NoError(t, err) + assert.Equal(t, "ca-blob-b64", got.CertificateAuthorityB64) +} + +func TestSetClusterCertificateAuthority_MissingReturnsErrClusterNotFound(t *testing.T) { + kv := newClusterStateTestKV(t) + + err := SetClusterCertificateAuthority(kv, "ghost", "ca-blob") + require.ErrorIs(t, err, ErrClusterNotFound) +} + +func TestSetClusterCertificateAuthority_EmptyInputsRejected(t *testing.T) { + kv := newClusterStateTestKV(t) + + require.Error(t, SetClusterCertificateAuthority(kv, "", "ca-blob")) + require.Error(t, SetClusterCertificateAuthority(kv, "alpha", "")) +} + +func TestDeleteClusterPrefix_SweepsEveryKey(t *testing.T) { + kv := newClusterStateTestKV(t) + + require.NoError(t, PutClusterMeta(kv, sampleClusterMeta("alpha"))) + _, err := kv.Put(NodegroupKey("alpha", "ng-1"), []byte(`{"name":"ng-1"}`)) + require.NoError(t, err) + _, err = kv.Put(OIDCSigningKeyKey("alpha"), []byte("enc-blob")) + require.NoError(t, err) + _, err = kv.Put(OIDCJWKSKey("alpha"), []byte(`{"keys":[]}`)) + require.NoError(t, err) + _, err = kv.Put(EventKey("alpha", "1700000000"), []byte(`{"ts":"1700000000"}`)) + require.NoError(t, err) + // Sibling cluster must survive the sweep. + require.NoError(t, PutClusterMeta(kv, sampleClusterMeta("beta"))) + + require.NoError(t, DeleteClusterPrefix(kv, "alpha")) + + for _, k := range []string{ + ClusterMetaKey("alpha"), + NodegroupKey("alpha", "ng-1"), + OIDCSigningKeyKey("alpha"), + OIDCJWKSKey("alpha"), + EventKey("alpha", "1700000000"), + } { + _, err := kv.Get(k) + assert.True(t, errors.Is(err, nats.ErrKeyNotFound), "key %s should be gone, got err=%v", k, err) + } + + got, err := GetClusterMeta(kv, "beta") + require.NoError(t, err) + assert.Equal(t, "beta", got.Name) +} + +func TestDeleteClusterPrefix_EmptyBucketIsNoop(t *testing.T) { + kv := newClusterStateTestKV(t) + + require.NoError(t, DeleteClusterPrefix(kv, "ghost")) +} + +func TestDeleteClusterPrefix_EmptyNameRejected(t *testing.T) { + kv := newClusterStateTestKV(t) + + require.Error(t, DeleteClusterPrefix(kv, "")) +} diff --git a/spinifex/handlers/eks/create_cluster_test.go b/spinifex/handlers/eks/create_cluster_test.go new file mode 100644 index 000000000..084a2ec50 --- /dev/null +++ b/spinifex/handlers/eks/create_cluster_test.go @@ -0,0 +1,210 @@ +package handlers_eks + +import ( + "errors" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/eks" + "github.com/mulgadc/spinifex/spinifex/awserrors" + "github.com/mulgadc/spinifex/spinifex/tags" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func createInput(name string) *eks.CreateClusterInput { + return &eks.CreateClusterInput{ + Name: aws.String(name), + RoleArn: aws.String("arn:aws:iam::111122223333:role/eks-cluster"), + Version: aws.String("1.32"), + ResourcesVpcConfig: &eks.VpcConfigRequest{ + SubnetIds: aws.StringSlice([]string{"subnet-aaa"}), + }, + } +} + +// A create that fails after the NLB is provisioned must leave the NLB ARNs +// persisted on the (now FAILED) meta, otherwise the resources leak with no +// owning record and DeleteCluster cannot reclaim them (bead 165.3). +func TestCreateCluster_NLBArnPersistedBeforeLaterFailure(t *testing.T) { + f := newEKSServiceFixture(t) + + // Force the K3s VM launch to fail — this is after EnsureClusterNLB and the + // NLB-arn persist checkpoint, but before the final PutClusterMeta. + f.inst.launchErr = errors.New("no capacity") + + _, err := f.svc.CreateCluster(createInput("alpha"), testAccountID) + require.Error(t, err) + + meta, getErr := GetClusterMeta(f.kv, "alpha") + require.NoError(t, getErr, "meta must remain so the leaked NLB has an owning record") + assert.Equal(t, ClusterStatusFailed, meta.Status) + assert.NotEmpty(t, meta.NLBArn, "NLB ARN must be persisted before the VM-launch step") + assert.NotEmpty(t, meta.NLBTargetGroupArn) +} + +// End-to-end: a failed create followed by delete-cluster leaves zero orphaned +// NLB resources. The persisted ARNs from the partial create drive teardown. +func TestCreateCluster_FailedCreateThenDeleteReclaimsNLB(t *testing.T) { + f := newEKSServiceFixture(t) + f.inst.launchErr = errors.New("no capacity") + + _, err := f.svc.CreateCluster(createInput("alpha"), testAccountID) + require.Error(t, err) + + // The NLB the partial create provisioned exists in the fake. + require.NotEmpty(t, f.nlb.createLBCalls, "create provisioned an NLB") + + _, err = f.svc.DeleteCluster(deleteInput("alpha"), testAccountID) + require.NoError(t, err) + + assert.NotEmpty(t, f.nlb.deleteLBCalls, "DeleteCluster must tear down the NLB recorded by the failed create") + _, getErr := GetClusterMeta(f.kv, "alpha") + assert.ErrorIs(t, getErr, ErrClusterNotFound) +} + +// A live (CREATING/ACTIVE) cluster of the same name blocks create with a +// cluster-scoped ResourceInUseException, not the ELBv2 target-group message. +func TestCreateCluster_ExistingClusterReturnsResourceInUse(t *testing.T) { + f := newEKSServiceFixture(t) + + meta := sampleClusterMeta("alpha") + meta.Status = ClusterStatusActive + require.NoError(t, PutClusterMeta(f.kv, meta)) + + _, err := f.svc.CreateCluster(createInput("alpha"), testAccountID) + require.EqualError(t, err, awserrors.ErrorEKSResourceInUse) +} + +// A FAILED cluster from a prior attempt must not block a retry: create reclaims +// it (tearing down recorded resources) and proceeds to a fresh CREATING cluster. +func TestCreateCluster_FailedClusterIsReclaimedOnRetry(t *testing.T) { + f := newEKSServiceFixture(t) + + // First attempt fails at VM launch, leaving a FAILED meta with NLB ARNs. + f.inst.launchErr = errors.New("no capacity") + _, err := f.svc.CreateCluster(createInput("alpha"), testAccountID) + require.Error(t, err) + failed, err := GetClusterMeta(f.kv, "alpha") + require.NoError(t, err) + require.Equal(t, ClusterStatusFailed, failed.Status) + + // Retry now succeeds: the reclaim path purges the failed attempt's NLB and + // the create starts clean. + f.inst.launchErr = nil + _, err = f.svc.CreateCluster(createInput("alpha"), testAccountID) + require.NoError(t, err) + + got, err := GetClusterMeta(f.kv, "alpha") + require.NoError(t, err) + assert.Equal(t, ClusterStatusCreating, got.Status) + assert.NotEmpty(t, f.nlb.deleteLBCalls, "reclaim must tear down the failed attempt's NLB") +} + +// A FAILED cluster is observable via Describe and List (bead 165.6) — no state +// where create blocks but describe/list show nothing. +func TestCreateCluster_FailedClusterVisibleInDescribeAndList(t *testing.T) { + f := newEKSServiceFixture(t) + + meta := sampleClusterMeta("alpha") + meta.Status = ClusterStatusFailed + meta.StatusReason = "bootstrap failed: boom" + require.NoError(t, PutClusterMeta(f.kv, meta)) + + desc, err := f.svc.DescribeCluster(&eks.DescribeClusterInput{Name: aws.String("alpha")}, testAccountID) + require.NoError(t, err) + assert.Equal(t, eks.ClusterStatusFailed, aws.StringValue(desc.Cluster.Status)) + + list, err := f.svc.ListClusters(&eks.ListClustersInput{}, testAccountID) + require.NoError(t, err) + assert.Contains(t, aws.StringValueSlice(list.Clusters), "alpha") +} + +// An ACTIVE cluster whose reconciler recorded a /healthz failure must surface +// that as a ClusterHealth issue in describe-cluster, so a dead control plane is +// visible behind the still-ACTIVE status (bead 165.13). +func TestDescribeCluster_SurfacesHealthIssueForActiveCluster(t *testing.T) { + f := newEKSServiceFixture(t) + + meta := sampleClusterMeta("alpha") + meta.Status = ClusterStatusActive + require.NoError(t, PutClusterMeta(f.kv, meta)) + require.NoError(t, SetClusterHealth(f.kv, "alpha", "healthz request: connection refused")) + + desc, err := f.svc.DescribeCluster(&eks.DescribeClusterInput{Name: aws.String("alpha")}, testAccountID) + require.NoError(t, err) + assert.Equal(t, eks.ClusterStatusActive, aws.StringValue(desc.Cluster.Status)) + require.NotNil(t, desc.Cluster.Health) + require.Len(t, desc.Cluster.Health.Issues, 1) + assert.Equal(t, eks.ClusterIssueCodeClusterUnreachable, aws.StringValue(desc.Cluster.Health.Issues[0].Code)) + assert.Contains(t, aws.StringValue(desc.Cluster.Health.Issues[0].Message), "connection refused") +} + +// A healthy ACTIVE cluster carries no health issue in describe-cluster. +func TestDescribeCluster_HealthyActiveClusterHasNoIssues(t *testing.T) { + f := newEKSServiceFixture(t) + + meta := sampleClusterMeta("alpha") + meta.Status = ClusterStatusActive + require.NoError(t, PutClusterMeta(f.kv, meta)) + + desc, err := f.svc.DescribeCluster(&eks.DescribeClusterInput{Name: aws.String("alpha")}, testAccountID) + require.NoError(t, err) + assert.Nil(t, desc.Cluster.Health, "healthy cluster must not report a ClusterHealth issue") +} + +// A missing eks-server AMI is an operator/config gap, not an unrecoverable +// internal fault: CreateCluster must surface ServiceUnavailable (not a generic +// ServerInternal) and mark the half-built cluster FAILED (bead 165.4). +func TestCreateCluster_MissingAMIReturnsServiceUnavailable(t *testing.T) { + f := newEKSServiceFixture(t) + f.ami.describeOut = &ec2.DescribeImagesOutput{} // no images tagged managed-by=eks + + _, err := f.svc.CreateCluster(createInput("alpha"), testAccountID) + require.EqualError(t, err, awserrors.ErrorServiceUnavailable) + + meta, getErr := GetClusterMeta(f.kv, "alpha") + require.NoError(t, getErr) + assert.Equal(t, ClusterStatusFailed, meta.Status) +} + +func TestCreateCluster_HappyPathPersistsActiveCreatingMeta(t *testing.T) { + f := newEKSServiceFixture(t) + + out, err := f.svc.CreateCluster(createInput("alpha"), testAccountID) + require.NoError(t, err) + require.NotNil(t, out) + + meta, err := GetClusterMeta(f.kv, "alpha") + require.NoError(t, err) + assert.Equal(t, ClusterStatusCreating, meta.Status) + assert.NotEmpty(t, meta.NLBArn) + assert.NotEmpty(t, meta.ControlPlaneInstanceID) + assert.NotEmpty(t, meta.ControlPlaneENIID) +} + +func TestCreateCluster_AllocatesHiddenEgressEIP(t *testing.T) { + f := newEKSServiceFixture(t) + + _, err := f.svc.CreateCluster(createInput("egress"), testAccountID) + require.NoError(t, err) + + meta, err := GetClusterMeta(f.kv, "egress") + require.NoError(t, err) + assert.Equal(t, f.eip.publicIP, meta.EgressEIPPublicIP) + assert.Equal(t, f.eip.allocationID, meta.EgressEIPAllocationID) + + require.Len(t, f.eip.allocateCalls, 1) + // The pool address is tagged ManagedBy=eks so it stays out of customer + // DescribeAddresses listings. + tagged := false + for _, spec := range f.eip.allocateCalls[0].TagSpecifications { + for _, tg := range spec.Tags { + if aws.StringValue(tg.Key) == tags.ManagedByKey && aws.StringValue(tg.Value) == tags.ManagedByEKS { + tagged = true + } + } + } + assert.True(t, tagged, "egress EIP must carry the ManagedBy=eks tag") +} diff --git a/spinifex/handlers/eks/delete_cluster_test.go b/spinifex/handlers/eks/delete_cluster_test.go new file mode 100644 index 000000000..2088881c0 --- /dev/null +++ b/spinifex/handlers/eks/delete_cluster_test.go @@ -0,0 +1,149 @@ +package handlers_eks + +import ( + "errors" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/eks" + "github.com/aws/aws-sdk-go/service/elbv2" + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/nats-io/nats.go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeSubnetResolver struct{} + +var _ SubnetVPCResolver = (*fakeSubnetResolver)(nil) + +func (fakeSubnetResolver) GetSubnetVPC(_, _ string) (string, error) { return "vpc-aaa", nil } + +func deleteInput(name string) *eks.DeleteClusterInput { + return &eks.DeleteClusterInput{Name: aws.String(name)} +} + +// eksServiceFixture stands up an EKSServiceImpl with fake orchestration deps +// and an embedded JetStream KV. Tests poke the fakes' *Err fields to force +// failures at specific lifecycle steps. +type eksServiceFixture struct { + svc *EKSServiceImpl + kv nats.KeyValue + nlb *fakeNLBProvisioner + inst *fakeK3sInst + vpc *fakeK3sVPC + ami *fakeK3sAMI + eip *fakeEIPProvisioner +} + +func newEKSServiceFixture(t *testing.T) *eksServiceFixture { + t.Helper() + _, nc, js := testutil.StartTestJetStream(t) + kv, err := GetOrCreateAccountBucket(js, testAccountID) + require.NoError(t, err) + + nlb := newFakeNLBProvisioner() + inst := &fakeK3sInst{} + vpc := &fakeK3sVPC{} + ami := &fakeK3sAMI{} + eip := newFakeEIPProvisioner() + + svc, err := NewEKSServiceImpl(EKSServiceDeps{ + NATSConn: nc, + MasterKey: bootstrapTestMasterKey, + GatewayBaseURL: "https://gw.local:9999", + Region: "us-east-1", + HolderID: "node-1", + NATSURL: "nats://gw.local:4222", + NATSToken: "s3cr3t-token", + NATSCACert: "-----BEGIN CERTIFICATE-----\nFAKECA\n-----END CERTIFICATE-----\n", + VPCSG: newFakeSGProvisioner(), + VPCK3s: vpc, + VPCSubnet: fakeSubnetResolver{}, + NLB: nlb, + Instance: inst, + Image: ami, + EIP: eip, + }) + require.NoError(t, err) + t.Cleanup(svc.Shutdown) + + return &eksServiceFixture{svc: svc, kv: kv, nlb: nlb, inst: inst, vpc: vpc, ami: ami, eip: eip} +} + +// deleteClusterFixture is an eksServiceFixture pre-seeded with a CREATING +// cluster meta whose NLB/VM/SG resource references are all populated so +// DeleteCluster exercises every teardown branch. +type deleteClusterFixture struct { + svc *EKSServiceImpl + kv nats.KeyValue + nlb *fakeNLBProvisioner + inst *fakeK3sInst + vpc *fakeK3sVPC +} + +func newDeleteClusterFixture(t *testing.T, clusterName string) *deleteClusterFixture { + t.Helper() + f := newEKSServiceFixture(t) + kv := f.kv + + meta := sampleClusterMeta(clusterName) + meta.NLBArn = "arn:aws:elasticloadbalancing:us-east-1:111122223333:loadbalancer/net/" + ClusterNLBName(clusterName) + "/lb-001" + meta.NLBTargetGroupArn = "arn:aws:elasticloadbalancing:us-east-1:111122223333:targetgroup/" + ClusterTargetGroupName(clusterName) + "/tg-001" + meta.ControlPlaneENIIP = "10.0.1.42" + meta.ControlPlaneInstanceID = "i-aaa111" + meta.ControlPlaneENIID = "eni-aaa111" + meta.ResourcesVpcConfig.VpcId = "vpc-aaa" + require.NoError(t, PutClusterMeta(kv, meta)) + + // Real OIDC key so ZeroizeClusterOIDCKey has material to wipe. + _, _, err := GenerateClusterOIDCKeypair(kv, clusterName, bootstrapTestMasterKey) + require.NoError(t, err) + + return &deleteClusterFixture{svc: f.svc, kv: kv, nlb: f.nlb, inst: f.inst, vpc: f.vpc} +} + +func TestDeleteCluster_AllTeardownSucceedsSweepsKV(t *testing.T) { + f := newDeleteClusterFixture(t, "alpha") + + out, err := f.svc.DeleteCluster(deleteInput("alpha"), testAccountID) + require.NoError(t, err) + require.NotNil(t, out) + + _, err = GetClusterMeta(f.kv, "alpha") + assert.ErrorIs(t, err, ErrClusterNotFound, "meta must be swept after successful teardown") + assert.Len(t, f.inst.terminateCalls, 1) + assert.Len(t, f.vpc.deleteCalls, 1) +} + +func TestDeleteCluster_NLBFailureLeavesMetaAndDELETING(t *testing.T) { + f := newDeleteClusterFixture(t, "alpha") + + // Seed the LB so delete is attempted, then force it to fail. + lbName := ClusterNLBName("alpha") + f.nlb.lbByName[lbName] = &elbv2.LoadBalancer{ + LoadBalancerArn: aws.String("arn:lb"), + LoadBalancerName: aws.String(lbName), + } + f.nlb.deleteLBErr = errors.New("elbv2 unavailable") + + _, err := f.svc.DeleteCluster(deleteInput("alpha"), testAccountID) + require.Error(t, err, "teardown failure must surface, not be swallowed") + + meta, getErr := GetClusterMeta(f.kv, "alpha") + require.NoError(t, getErr, "meta must survive so a retry can find the resource ARNs") + assert.Equal(t, ClusterStatusDeleting, meta.Status) + assert.NotEmpty(t, meta.NLBArn, "NLB ARN must remain recorded for retry") +} + +func TestDeleteCluster_VMTerminateFailureLeavesMeta(t *testing.T) { + f := newDeleteClusterFixture(t, "alpha") + f.inst.terminateErr = errors.New("hypervisor unreachable") + + _, err := f.svc.DeleteCluster(deleteInput("alpha"), testAccountID) + require.Error(t, err) + + meta, getErr := GetClusterMeta(f.kv, "alpha") + require.NoError(t, getErr) + assert.Equal(t, ClusterStatusDeleting, meta.Status) +} diff --git a/spinifex/handlers/eks/eip_test.go b/spinifex/handlers/eks/eip_test.go new file mode 100644 index 000000000..209f6c0fc --- /dev/null +++ b/spinifex/handlers/eks/eip_test.go @@ -0,0 +1,44 @@ +package handlers_eks + +import ( + "github.com/aws/aws-sdk-go/aws" + ec2 "github.com/aws/aws-sdk-go/service/ec2" +) + +// fakeEIPProvisioner records allocate/release calls and hands back a +// deterministic pool address so CreateCluster/DeleteCluster egress wiring is +// exercised without a live EIP service. +type fakeEIPProvisioner struct { + allocateCalls []*ec2.AllocateAddressInput + releaseCalls []*ec2.ReleaseAddressInput + + publicIP string + allocationID string + allocateErr error + releaseErr error +} + +var _ eipProvisioner = (*fakeEIPProvisioner)(nil) + +func newFakeEIPProvisioner() *fakeEIPProvisioner { + return &fakeEIPProvisioner{publicIP: "203.0.113.50", allocationID: "eipalloc-fake01"} +} + +func (f *fakeEIPProvisioner) AllocateAddress(input *ec2.AllocateAddressInput, _ string) (*ec2.AllocateAddressOutput, error) { + f.allocateCalls = append(f.allocateCalls, input) + if f.allocateErr != nil { + return nil, f.allocateErr + } + return &ec2.AllocateAddressOutput{ + PublicIp: aws.String(f.publicIP), + AllocationId: aws.String(f.allocationID), + }, nil +} + +func (f *fakeEIPProvisioner) ReleaseAddress(input *ec2.ReleaseAddressInput, _ string) (*ec2.ReleaseAddressOutput, error) { + f.releaseCalls = append(f.releaseCalls, input) + if f.releaseErr != nil { + return nil, f.releaseErr + } + return &ec2.ReleaseAddressOutput{}, nil +} diff --git a/spinifex/handlers/eks/k3s_server_vm.go b/spinifex/handlers/eks/k3s_server_vm.go new file mode 100644 index 000000000..b78c95ae9 --- /dev/null +++ b/spinifex/handlers/eks/k3s_server_vm.go @@ -0,0 +1,424 @@ +package handlers_eks + +import ( + "errors" + "fmt" + "log/slog" + "net" + "strconv" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/spinifex/awserrors" + "github.com/mulgadc/spinifex/spinifex/handlers/sysinstance" + "github.com/mulgadc/spinifex/spinifex/tags" +) + +// ErrEKSServerAMINotFound is returned by the launcher when no AMI carrying the +// EKS managed-by tag exists in the account. CreateCluster maps it to a precise +// client error: it signals an operator/config gap (the eks-server image was +// never built/imported), not an unrecoverable internal fault. +var ErrEKSServerAMINotFound = errors.New("eks: eks-server AMI not found") + +// k3sVPCProvisioner is the subset of handlers_ec2_vpc.VPCService that the K3s +// server VM launcher needs. Narrow so tests can fake it without implementing +// the full VPC surface. +type k3sVPCProvisioner interface { + CreateNetworkInterface(input *ec2.CreateNetworkInterfaceInput, accountID string) (*ec2.CreateNetworkInterfaceOutput, error) + DeleteNetworkInterface(input *ec2.DeleteNetworkInterfaceInput, accountID string) (*ec2.DeleteNetworkInterfaceOutput, error) +} + +// k3sInstanceLauncher is the system-instance launch surface the K3s control-plane +// VM needs. The VM boots from the eks-server AMI (BootAMI) and must get a +// management-bridge NIC so it can reach the daemon's NATS endpoint off its +// tenant VPC subnet — which only the system-instance path provides. +// *daemon.Daemon satisfies this interface. +type k3sInstanceLauncher interface { + LaunchSystemInstance(input *sysinstance.SystemInstanceInput) (*sysinstance.SystemInstanceOutput, error) + TerminateSystemInstance(instanceID string) error +} + +// k3sAMIResolver is the subset of handlers_ec2_image.ImageService the launcher +// uses to resolve the eks-server AMI name to an AMI ID at launch time. +type k3sAMIResolver interface { + DescribeImages(input *ec2.DescribeImagesInput, accountID string) (*ec2.DescribeImagesOutput, error) +} + +const ( + // defaultK3sServerInstanceType is the spinifex instance type closest to + // the AWS EKS minimum control-plane footprint (2 vCPU / 8 GB / 40 GB). + // Callers can override via K3sServerInput.InstanceType. + defaultK3sServerInstanceType = "t3.medium" + + // k3sOIDCSigningKeyPath is the on-VM path where cloud-init drops the + // per-cluster OIDC private key PEM. K3s reads it via the + // service-account-signing-key-file kube-apiserver arg (signs SA tokens). + k3sOIDCSigningKeyPath = "/etc/rancher/k3s/oidc-signing-key.pem" + + // k3sOIDCPublicKeyPath is where cloud-init drops the matching public key + // PEM. kube-apiserver's service-account-key-file requires a PUBLIC key + // (it rejects a private-key PEM with "data does not contain any valid RSA + // or ECDSA public keys" and crash-loops), so it must be a separate file. + k3sOIDCPublicKeyPath = "/etc/rancher/k3s/oidc-signing-key.pub.pem" + + // k3sFirstBootEnvPath is the env file k3s-first-boot.sh sources at boot + // (see scripts/images/eks-server/k3s-first-boot.sh ENVFILE). + k3sFirstBootEnvPath = "/etc/spinifex-eks/first-boot.env" + + // k3sNATSCAPath is the on-VM destination for the NATS CA cert PEM. The K3s + // VM uses it to verify the daemon's NATS TLS when publishing bootstrap + // messages. Path matches k3s-first-boot.sh SPINIFEX_NATS_CA. + k3sNATSCAPath = "/etc/spinifex-eks/nats-ca.pem" + + // k3sConfigPath is the K3s server config file cloud-init writes; K3s + // reads it at startup (overrides the AMI-baked config.yaml.skel). + k3sConfigPath = "/etc/rancher/k3s/config.yaml" + + // k3sResolvConfPath is the on-VM resolver file. The Alpine eks-server AMI + // runs dhcpcd, whose resolv.conf hook fails to persist the DHCP-supplied + // nameservers ("can't create /etc/resolv.conf: nonexistent directory"), so + // the VM boots with no resolver and containerd cannot resolve registry-1. + // docker.io to pull the system-pod images. cloud-init writes a static + // resolver here; the dhcpcd hook never clobbers it (it errors before it + // would). Reachable via the cluster's egress SNAT. + k3sResolvConfPath = "/etc/resolv.conf" + + // k3sResolvConf is the static resolver content. Public anycast resolvers + // reached over the control-plane VM's egress path. + k3sResolvConf = "nameserver 1.1.1.1\nnameserver 8.8.8.8" +) + +// K3sServerInput is the launcher's input shape. AccountID is the customer +// account; the ENI + VM both live there in v1 (SystemAccount-owned VM is +// deferred behind cross-account-ENI work). Region is carried for future +// region-aware AMI lookups but not consumed today. +type K3sServerInput struct { + AccountID string + ClusterName string + Region string + SubnetID string + ControlPlaneSGID string + NLBDNS string + OIDCIssuer string + OIDCPrivateKeyPEM string + OIDCPublicKeyPEM string + NATSURL string + NATSToken string + NATSCACert string + InstanceType string +} + +// K3sServerOutput carries the identifiers the caller needs to persist into +// ClusterMeta and to register the ENI IP with the cluster NLB target group. +type K3sServerOutput struct { + InstanceID string + ENIID string + ENIIP string + MgmtIP string +} + +// LaunchK3sServerVM provisions the K3s control-plane VM for an EKS cluster. +// Sequence: resolve the eks-server AMI, pre-create the customer-account ENI +// in the supplied subnet with the control-plane SG, render cloud-init user +// data (env vars, OIDC PEM, K3s config), then call RunInstances with +// NetworkInterfaces[0].NetworkInterfaceId pointing at the pre-created ENI. +// Returns instance/ENI identity so the caller can register the ENI IP with +// the cluster NLB target group and persist the IDs in ClusterMeta. On +// RunInstances failure the pre-created ENI is deleted best-effort so the +// caller does not leak a customer-account resource. +func LaunchK3sServerVM( + vpcSvc k3sVPCProvisioner, + instSvc k3sInstanceLauncher, + amiSvc k3sAMIResolver, + in K3sServerInput, +) (*K3sServerOutput, error) { + if err := validateK3sServerInput(in); err != nil { + return nil, err + } + instanceType := in.InstanceType + if instanceType == "" { + instanceType = defaultK3sServerInstanceType + } + + amiID, err := lookupEKSServerAMI(amiSvc, in.AccountID) + if err != nil { + return nil, err + } + + eniOut, err := vpcSvc.CreateNetworkInterface(&ec2.CreateNetworkInterfaceInput{ + SubnetId: aws.String(in.SubnetID), + Description: aws.String("EKS K3s server ENI for " + in.ClusterName), + Groups: aws.StringSlice([]string{in.ControlPlaneSGID}), + TagSpecifications: []*ec2.TagSpecification{{ + ResourceType: aws.String("network-interface"), + Tags: []*ec2.Tag{ + {Key: aws.String(tags.ManagedByKey), Value: aws.String(tags.ManagedByEKS)}, + {Key: aws.String(clusterEKSClusterTagKey), Value: aws.String(in.ClusterName)}, + {Key: aws.String(clusterEKSRoleTagKey), Value: aws.String(clusterEKSRoleControlPlane)}, + }, + }}, + }, in.AccountID) + if err != nil { + return nil, fmt.Errorf("create K3s ENI in subnet %s: %w", in.SubnetID, err) + } + if eniOut == nil || eniOut.NetworkInterface == nil || + aws.StringValue(eniOut.NetworkInterface.NetworkInterfaceId) == "" || + aws.StringValue(eniOut.NetworkInterface.PrivateIpAddress) == "" { + return nil, errors.New("eks: CreateNetworkInterface returned incomplete ENI") + } + eniID := aws.StringValue(eniOut.NetworkInterface.NetworkInterfaceId) + eniIP := aws.StringValue(eniOut.NetworkInterface.PrivateIpAddress) + + userData := buildK3sUserData(in) + + sysOut, err := instSvc.LaunchSystemInstance(&sysinstance.SystemInstanceInput{ + BootMode: sysinstance.BootAMI, + ManagedBy: tags.ManagedByEKS, + InstanceType: instanceType, + ImageID: amiID, + AccountID: in.AccountID, + ENIID: eniID, + ENIMac: aws.StringValue(eniOut.NetworkInterface.MacAddress), + ENIIP: eniIP, + UserData: userData, + }) + if err != nil { + rollbackK3sENI(vpcSvc, in.AccountID, eniID) + return nil, fmt.Errorf("run K3s server instance for cluster %s: %w", in.ClusterName, err) + } + if sysOut == nil || sysOut.InstanceID == "" { + rollbackK3sENI(vpcSvc, in.AccountID, eniID) + return nil, fmt.Errorf("eks: LaunchSystemInstance returned no instance for cluster %s", in.ClusterName) + } + instanceID := sysOut.InstanceID + + slog.Info("LaunchK3sServerVM completed", + "clusterName", in.ClusterName, + "accountID", in.AccountID, + "instanceId", instanceID, + "eniId", eniID, + "eniIp", eniIP, + ) + + return &K3sServerOutput{ + InstanceID: instanceID, + ENIID: eniID, + ENIIP: eniIP, + MgmtIP: sysOut.MgmtIP, + }, nil +} + +// TerminateK3sServerVM tears down the K3s server VM for a cluster. The +// instance termination cascades ENI detach inside the customer instance path, +// then this helper deletes the ENI explicitly so DeleteCluster does not leak +// it. Missing instance/ENI is a no-op so retries stay safe. +func TerminateK3sServerVM( + vpcSvc k3sVPCProvisioner, + instSvc k3sInstanceLauncher, + accountID, instanceID, eniID string, +) error { + if instanceID == "" && eniID == "" { + return nil + } + var firstErr error + if instanceID != "" { + // A retried DeleteCluster reaches here after the VM already drained, so + // "instance not found" is the steady-state success case, not a failure — + // treat it as idempotent so teardown can proceed to the ENI/SG/KV sweep. + if err := instSvc.TerminateSystemInstance(instanceID); err != nil { + if errors.Is(err, sysinstance.ErrSystemInstanceNotFound) { + slog.Debug("TerminateK3sServerVM: instance already gone", "instanceId", instanceID) + } else { + slog.Warn("TerminateK3sServerVM: terminate failed", "instanceId", instanceID, "err", err) + firstErr = fmt.Errorf("terminate instance %s: %w", instanceID, err) + } + } + } + if eniID != "" { + if _, err := vpcSvc.DeleteNetworkInterface(&ec2.DeleteNetworkInterfaceInput{ + NetworkInterfaceId: aws.String(eniID), + }, accountID); err != nil { + switch { + case awserrors.IsErrorCode(err, awserrors.ErrorInvalidNetworkInterfaceIDNotFound), + awserrors.IsErrorCode(err, awserrors.ErrorInvalidNetworkInterfaceNotFound): + // The instance-terminate cascade (or a prior retry) already + // deleted the ENI. Idempotent success — must NOT block the SG + + // KV sweep, or the cluster wedges in DELETING permanently. + slog.Debug("TerminateK3sServerVM: ENI already gone", "eniId", eniID) + default: + // InvalidNetworkInterface.InUse (the VM is still terminating + // async and holds the ENI) and any other error are retryable: + // surface so the cluster stays DELETING and the reconciler + // retries once the instance releases the ENI. + slog.Warn("TerminateK3sServerVM: ENI delete failed", "eniId", eniID, "err", err) + if firstErr == nil { + firstErr = fmt.Errorf("delete ENI %s: %w", eniID, err) + } + } + } + } + return firstErr +} + +// lookupEKSServerAMI resolves the EKS control-plane AMI by the +// spinifex:managed-by=eks tag the build pipeline stamps on it, rather than a +// brittle exact name. This survives the planned server+agent → single EKS AMI +// unify untouched (the unified AMI keeps the tag; role is chosen per-instance +// at launch). If more than one AMI carries the tag (e.g. server + agent both +// imported pre-unify), the newest by CreationDate wins. +func lookupEKSServerAMI(amiSvc k3sAMIResolver, accountID string) (string, error) { + out, err := amiSvc.DescribeImages(&ec2.DescribeImagesInput{ + Filters: []*ec2.Filter{ + {Name: aws.String("tag:" + tags.ManagedByKey), Values: aws.StringSlice([]string{tags.ManagedByEKS})}, + }, + }, accountID) + if err != nil { + return "", fmt.Errorf("describe eks AMI (tag:%s=%s): %w", tags.ManagedByKey, tags.ManagedByEKS, err) + } + + var ( + newestID string + newestCreated string + matches int + ) + for _, img := range out.Images { + if img == nil || img.ImageId == nil || *img.ImageId == "" { + continue + } + matches++ + // CreationDate is a fixed-width RFC3339 timestamp, so lexicographic + // comparison orders it correctly without parsing. + if created := aws.StringValue(img.CreationDate); newestID == "" || created > newestCreated { + newestID, newestCreated = *img.ImageId, created + } + } + if newestID == "" { + return "", fmt.Errorf("%w (tag:%s=%s, account %s)", ErrEKSServerAMINotFound, tags.ManagedByKey, tags.ManagedByEKS, accountID) + } + if matches > 1 { + slog.Warn("eks: multiple AMIs match managed-by=eks; using newest", + "count", matches, "imageId", newestID, "created", newestCreated) + } + return newestID, nil +} + +func rollbackK3sENI(vpcSvc k3sVPCProvisioner, accountID, eniID string) { + if _, err := vpcSvc.DeleteNetworkInterface(&ec2.DeleteNetworkInterfaceInput{ + NetworkInterfaceId: aws.String(eniID), + }, accountID); err != nil { + slog.Warn("LaunchK3sServerVM: rollback ENI delete failed", "eniId", eniID, "err", err) + } +} + +func validateK3sServerInput(in K3sServerInput) error { + switch { + case in.AccountID == "": + return errors.New("eks: K3sServerInput empty AccountID") + case in.ClusterName == "": + return errors.New("eks: K3sServerInput empty ClusterName") + case in.SubnetID == "": + return errors.New("eks: K3sServerInput empty SubnetID") + case in.ControlPlaneSGID == "": + return errors.New("eks: K3sServerInput empty ControlPlaneSGID") + case in.NLBDNS == "": + return errors.New("eks: K3sServerInput empty NLBDNS") + case in.OIDCIssuer == "": + return errors.New("eks: K3sServerInput empty OIDCIssuer") + case strings.TrimSpace(in.OIDCPrivateKeyPEM) == "": + return errors.New("eks: K3sServerInput empty OIDCPrivateKeyPEM") + case strings.TrimSpace(in.OIDCPublicKeyPEM) == "": + return errors.New("eks: K3sServerInput empty OIDCPublicKeyPEM") + case in.NATSURL == "": + return errors.New("eks: K3sServerInput empty NATSURL") + case in.NATSToken == "": + return errors.New("eks: K3sServerInput empty NATSToken") + case strings.TrimSpace(in.NATSCACert) == "": + return errors.New("eks: K3sServerInput empty NATSCACert") + } + return nil +} + +// userDataFile is one entry in the cloud-config write_files block. +type userDataFile struct { + Path string + Perms string + Body string +} + +// buildK3sUserData renders the cloud-config YAML the K3s server VM consumes +// at first boot. Output is unencoded YAML; the caller base64-wraps it for +// the RunInstances UserData field. +func buildK3sUserData(in K3sServerInput) string { + nlbEndpoint := "https://" + net.JoinHostPort(in.NLBDNS, strconv.FormatInt(clusterNLBListenPort, 10)) + + envBody := strings.Join([]string{ + "SPINIFEX_NATS_URL=" + in.NATSURL, + "SPINIFEX_NATS_TOKEN=" + in.NATSToken, + "SPINIFEX_NATS_CA=" + k3sNATSCAPath, + "EKS_ACCOUNT_ID=" + in.AccountID, + "EKS_CLUSTER_NAME=" + in.ClusterName, + "EKS_NLB_ENDPOINT=" + nlbEndpoint, + "EKS_OIDC_ISSUER=" + in.OIDCIssuer, + }, "\n") + + // Omitting cluster-init selects the embedded SQLite (kine) datastore, not + // embedded etcd. etcd's per-commit fsync stalls past apiserver's 5s deadline + // on the viperblock-backed root volume, so it never reaches /healthz; kine + // tolerates the deferred-durability block path. Trade-off: no multi-server + // HA (that needs etcd) — acceptable for single-node v1. + // + // anonymous-auth=true: k3s hardens it off by default, which makes the + // apiserver answer 401 to an unauthenticated /healthz. The cluster reconciler + // probes https:///healthz anonymously to gate ACTIVE, so it must be + // reachable; the default RBAC binds only the health/version non-resource + // paths to system:unauthenticated, so this exposes nothing else. + k3sConfig := strings.Join([]string{ + "tls-san:", + " - " + in.NLBDNS, + "kube-apiserver-arg:", + " - service-account-key-file=" + k3sOIDCPublicKeyPath, + " - service-account-signing-key-file=" + k3sOIDCSigningKeyPath, + " - service-account-issuer=" + in.OIDCIssuer, + " - api-audiences=sts.amazonaws.com", + " - anonymous-auth=true", + }, "\n") + + files := []userDataFile{ + // first-boot.env carries the NATS token, so keep it root-only (0600). + {Path: k3sFirstBootEnvPath, Perms: "0600", Body: envBody}, + {Path: k3sOIDCSigningKeyPath, Perms: "0600", Body: strings.TrimRight(in.OIDCPrivateKeyPEM, "\n")}, + {Path: k3sOIDCPublicKeyPath, Perms: "0644", Body: strings.TrimRight(in.OIDCPublicKeyPEM, "\n")}, + {Path: k3sConfigPath, Perms: "0644", Body: k3sConfig}, + {Path: k3sNATSCAPath, Perms: "0644", Body: strings.TrimRight(in.NATSCACert, "\n")}, + } + + var buf strings.Builder + buf.WriteString("#cloud-config\n") + + // Resolver via bootcmd, NOT write_files: on the Alpine AMI /etc/resolv.conf + // is a dangling symlink (its target dir does not exist — which is why the + // dhcpcd hook cannot persist DHCP DNS), and pointing write_files at it makes + // cloud-init follow the dead link, fail, and abort the WHOLE write_files + // block (dropping first-boot.env + the k3s config). bootcmd runs before + // write_files in the init stage and as a shell, so it can drop the symlink + // and write a real file. Containerd needs this to resolve registry-1.docker. + // io for system-pod image pulls; reachable over the cluster egress SNAT. + buf.WriteString("bootcmd:\n") + buf.WriteString(" - rm -f " + k3sResolvConfPath + "\n") + fmt.Fprintf(&buf, " - printf '%s\\n' > %s\n", + strings.ReplaceAll(k3sResolvConf, "\n", "\\n"), k3sResolvConfPath) + + buf.WriteString("write_files:\n") + for _, f := range files { + fmt.Fprintf(&buf, " - path: %s\n", f.Path) + fmt.Fprintf(&buf, " permissions: '%s'\n", f.Perms) + buf.WriteString(" content: |\n") + for line := range strings.SplitSeq(f.Body, "\n") { + buf.WriteString(" ") + buf.WriteString(line) + buf.WriteString("\n") + } + } + return buf.String() +} diff --git a/spinifex/handlers/eks/k3s_server_vm_test.go b/spinifex/handlers/eks/k3s_server_vm_test.go new file mode 100644 index 000000000..88bde08de --- /dev/null +++ b/spinifex/handlers/eks/k3s_server_vm_test.go @@ -0,0 +1,400 @@ +package handlers_eks + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/spinifex/awserrors" + "github.com/mulgadc/spinifex/spinifex/handlers/sysinstance" + "github.com/mulgadc/spinifex/spinifex/tags" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeK3sVPC struct { + createCalls []*ec2.CreateNetworkInterfaceInput + deleteCalls []*ec2.DeleteNetworkInterfaceInput + + createOut *ec2.CreateNetworkInterfaceOutput + createErr error + deleteErr error +} + +var _ k3sVPCProvisioner = (*fakeK3sVPC)(nil) + +func (f *fakeK3sVPC) CreateNetworkInterface(input *ec2.CreateNetworkInterfaceInput, _ string) (*ec2.CreateNetworkInterfaceOutput, error) { + f.createCalls = append(f.createCalls, input) + if f.createErr != nil { + return nil, f.createErr + } + if f.createOut != nil { + return f.createOut, nil + } + return &ec2.CreateNetworkInterfaceOutput{ + NetworkInterface: &ec2.NetworkInterface{ + NetworkInterfaceId: aws.String("eni-aaa111"), + PrivateIpAddress: aws.String("10.0.1.42"), + SubnetId: input.SubnetId, + }, + }, nil +} + +func (f *fakeK3sVPC) DeleteNetworkInterface(input *ec2.DeleteNetworkInterfaceInput, _ string) (*ec2.DeleteNetworkInterfaceOutput, error) { + f.deleteCalls = append(f.deleteCalls, input) + if f.deleteErr != nil { + return nil, f.deleteErr + } + return &ec2.DeleteNetworkInterfaceOutput{}, nil +} + +type fakeK3sInst struct { + launchCalls []*sysinstance.SystemInstanceInput + terminateCalls []string + + launchOut *sysinstance.SystemInstanceOutput + launchErr error + terminateErr error +} + +var _ k3sInstanceLauncher = (*fakeK3sInst)(nil) + +func (f *fakeK3sInst) LaunchSystemInstance(input *sysinstance.SystemInstanceInput) (*sysinstance.SystemInstanceOutput, error) { + f.launchCalls = append(f.launchCalls, input) + if f.launchErr != nil { + return nil, f.launchErr + } + if f.launchOut != nil { + return f.launchOut, nil + } + return &sysinstance.SystemInstanceOutput{InstanceID: "i-aaa111"}, nil +} + +func (f *fakeK3sInst) TerminateSystemInstance(instanceID string) error { + f.terminateCalls = append(f.terminateCalls, instanceID) + return f.terminateErr +} + +type fakeK3sAMI struct { + describeCalls []*ec2.DescribeImagesInput + + describeOut *ec2.DescribeImagesOutput + describeErr error +} + +var _ k3sAMIResolver = (*fakeK3sAMI)(nil) + +func (f *fakeK3sAMI) DescribeImages(input *ec2.DescribeImagesInput, _ string) (*ec2.DescribeImagesOutput, error) { + f.describeCalls = append(f.describeCalls, input) + if f.describeErr != nil { + return nil, f.describeErr + } + if f.describeOut != nil { + return f.describeOut, nil + } + return &ec2.DescribeImagesOutput{ + Images: []*ec2.Image{{ + ImageId: aws.String("ami-eks-server-001"), + Name: aws.String("spinifex-eks-server"), + CreationDate: aws.String("2026-06-01T00:00:00.000Z"), + Tags: []*ec2.Tag{ + {Key: aws.String(tags.ManagedByKey), Value: aws.String(tags.ManagedByEKS)}, + }, + }}, + }, nil +} + +func validK3sInput() K3sServerInput { + return K3sServerInput{ + AccountID: "111122223333", + ClusterName: "alpha", + Region: "us-east-1", + SubnetID: "subnet-aaa", + ControlPlaneSGID: "sg-cp-aaa", + NLBDNS: "eks-alpha-lb-001.us-east-1.elb.spinifex.local", + OIDCIssuer: "https://oidc.spinifex.local/clusters/111122223333/alpha", + OIDCPrivateKeyPEM: "-----BEGIN PRIVATE KEY-----\nFAKEKEY\n-----END PRIVATE KEY-----\n", + OIDCPublicKeyPEM: "-----BEGIN PUBLIC KEY-----\nFAKEPUB\n-----END PUBLIC KEY-----\n", + NATSURL: "nats://localhost:4222", + NATSToken: "s3cr3t-token", + NATSCACert: "-----BEGIN CERTIFICATE-----\nFAKECA\n-----END CERTIFICATE-----\n", + } +} + +func TestLaunchK3sServerVM_EmptyInputsRejected(t *testing.T) { + mk := func(mutate func(*K3sServerInput)) K3sServerInput { + in := validK3sInput() + mutate(&in) + return in + } + cases := []struct { + name string + in K3sServerInput + }{ + {"empty AccountID", mk(func(in *K3sServerInput) { in.AccountID = "" })}, + {"empty ClusterName", mk(func(in *K3sServerInput) { in.ClusterName = "" })}, + {"empty SubnetID", mk(func(in *K3sServerInput) { in.SubnetID = "" })}, + {"empty ControlPlaneSGID", mk(func(in *K3sServerInput) { in.ControlPlaneSGID = "" })}, + {"empty NLBDNS", mk(func(in *K3sServerInput) { in.NLBDNS = "" })}, + {"empty OIDCIssuer", mk(func(in *K3sServerInput) { in.OIDCIssuer = "" })}, + {"empty OIDCPrivateKeyPEM", mk(func(in *K3sServerInput) { in.OIDCPrivateKeyPEM = " \n " })}, + {"empty OIDCPublicKeyPEM", mk(func(in *K3sServerInput) { in.OIDCPublicKeyPEM = " \n " })}, + {"empty NATSURL", mk(func(in *K3sServerInput) { in.NATSURL = "" })}, + {"empty NATSToken", mk(func(in *K3sServerInput) { in.NATSToken = "" })}, + {"empty NATSCACert", mk(func(in *K3sServerInput) { in.NATSCACert = " \n" })}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + vpc, inst, ami := &fakeK3sVPC{}, &fakeK3sInst{}, &fakeK3sAMI{} + _, err := LaunchK3sServerVM(vpc, inst, ami, tc.in) + require.Error(t, err) + assert.Empty(t, vpc.createCalls) + assert.Empty(t, inst.launchCalls) + assert.Empty(t, ami.describeCalls) + }) + } +} + +func TestLaunchK3sServerVM_AMINotFound(t *testing.T) { + vpc, inst := &fakeK3sVPC{}, &fakeK3sInst{} + ami := &fakeK3sAMI{describeOut: &ec2.DescribeImagesOutput{}} + + _, err := LaunchK3sServerVM(vpc, inst, ami, validK3sInput()) + require.ErrorIs(t, err, ErrEKSServerAMINotFound) + assert.Contains(t, err.Error(), "spinifex:managed-by=eks") + assert.Empty(t, vpc.createCalls, "no ENI created when AMI lookup fails") + assert.Empty(t, inst.launchCalls) +} + +func TestLaunchK3sServerVM_AMILookupErrorPropagated(t *testing.T) { + vpc, inst := &fakeK3sVPC{}, &fakeK3sInst{} + ami := &fakeK3sAMI{describeErr: errors.New("DescribeImages backend down")} + + _, err := LaunchK3sServerVM(vpc, inst, ami, validK3sInput()) + require.Error(t, err) + assert.Contains(t, err.Error(), "describe eks AMI") + assert.Empty(t, vpc.createCalls) + assert.Empty(t, inst.launchCalls) +} + +func TestLaunchK3sServerVM_ENICreateFailureNoRunInstances(t *testing.T) { + vpc := &fakeK3sVPC{createErr: errors.New("InsufficientFreeAddressesInSubnet")} + inst, ami := &fakeK3sInst{}, &fakeK3sAMI{} + + _, err := LaunchK3sServerVM(vpc, inst, ami, validK3sInput()) + require.Error(t, err) + assert.Contains(t, err.Error(), "create K3s ENI in subnet subnet-aaa") + assert.Empty(t, inst.launchCalls) + assert.Empty(t, vpc.deleteCalls, "no ENI to roll back when create itself failed") +} + +func TestLaunchK3sServerVM_RunInstancesFailureRollsBackENI(t *testing.T) { + vpc := &fakeK3sVPC{} + inst := &fakeK3sInst{launchErr: errors.New("InsufficientInstanceCapacity")} + ami := &fakeK3sAMI{} + + _, err := LaunchK3sServerVM(vpc, inst, ami, validK3sInput()) + require.Error(t, err) + require.Len(t, vpc.createCalls, 1) + require.Len(t, vpc.deleteCalls, 1, "ENI must roll back when RunInstances fails") + assert.Equal(t, "eni-aaa111", aws.StringValue(vpc.deleteCalls[0].NetworkInterfaceId)) +} + +func TestLaunchK3sServerVM_HappyPath(t *testing.T) { + vpc, inst, ami := &fakeK3sVPC{}, &fakeK3sInst{}, &fakeK3sAMI{} + + out, err := LaunchK3sServerVM(vpc, inst, ami, validK3sInput()) + require.NoError(t, err) + require.NotNil(t, out) + assert.Equal(t, "i-aaa111", out.InstanceID) + assert.Equal(t, "eni-aaa111", out.ENIID) + assert.Equal(t, "10.0.1.42", out.ENIIP) + + require.Len(t, vpc.createCalls, 1) + eniIn := vpc.createCalls[0] + assert.Equal(t, "subnet-aaa", aws.StringValue(eniIn.SubnetId)) + assert.Equal(t, []string{"sg-cp-aaa"}, aws.StringValueSlice(eniIn.Groups)) + require.Len(t, eniIn.TagSpecifications, 1) + assertEC2TaggedAsEKSControlPlane(t, eniIn.TagSpecifications[0].Tags, "alpha") + + require.Len(t, inst.launchCalls, 1) + runIn := inst.launchCalls[0] + assert.Equal(t, sysinstance.BootAMI, runIn.BootMode) + assert.Equal(t, tags.ManagedByEKS, runIn.ManagedBy) + assert.Equal(t, "ami-eks-server-001", runIn.ImageID) + assert.Equal(t, defaultK3sServerInstanceType, runIn.InstanceType) + assert.Equal(t, "111122223333", runIn.AccountID) + assert.Equal(t, "eni-aaa111", runIn.ENIID) + assert.Equal(t, "10.0.1.42", runIn.ENIIP) + + assert.Empty(t, vpc.deleteCalls, "no rollback on success") +} + +func TestLaunchK3sServerVM_HonorsCustomInstanceType(t *testing.T) { + vpc, inst, ami := &fakeK3sVPC{}, &fakeK3sInst{}, &fakeK3sAMI{} + in := validK3sInput() + in.InstanceType = "t3.large" + + _, err := LaunchK3sServerVM(vpc, inst, ami, in) + require.NoError(t, err) + require.Len(t, inst.launchCalls, 1) + assert.Equal(t, "t3.large", inst.launchCalls[0].InstanceType) +} + +func TestLaunchK3sServerVM_UserDataContainsAllArtifacts(t *testing.T) { + vpc, inst, ami := &fakeK3sVPC{}, &fakeK3sInst{}, &fakeK3sAMI{} + + _, err := LaunchK3sServerVM(vpc, inst, ami, validK3sInput()) + require.NoError(t, err) + require.Len(t, inst.launchCalls, 1) + + udata := inst.launchCalls[0].UserData + + assert.True(t, strings.HasPrefix(udata, "#cloud-config\n")) + assert.Contains(t, udata, "write_files:") + + // Static resolver via bootcmd (NOT write_files — /etc/resolv.conf is a + // dangling symlink on the AMI; write_files would follow it, fail, and abort + // the whole block). Without it containerd image pulls cannot resolve. + assert.Contains(t, udata, "bootcmd:") + assert.Contains(t, udata, "rm -f "+k3sResolvConfPath) + assert.Contains(t, udata, "nameserver 1.1.1.1") + // Resolver must NOT be a write_files entry (that path is what broke seeding). + assert.NotContains(t, udata, "path: "+k3sResolvConfPath) + + assert.Contains(t, udata, "path: "+k3sFirstBootEnvPath) + assert.Contains(t, udata, "SPINIFEX_NATS_URL=nats://localhost:4222") + assert.Contains(t, udata, "SPINIFEX_NATS_TOKEN=s3cr3t-token") + assert.Contains(t, udata, "SPINIFEX_NATS_CA="+k3sNATSCAPath) + assert.Contains(t, udata, "EKS_ACCOUNT_ID=111122223333") + assert.Contains(t, udata, "EKS_CLUSTER_NAME=alpha") + assert.Contains(t, udata, "EKS_NLB_ENDPOINT=https://eks-alpha-lb-001.us-east-1.elb.spinifex.local:443") + assert.Contains(t, udata, "EKS_OIDC_ISSUER=https://oidc.spinifex.local/clusters/111122223333/alpha") + + assert.Contains(t, udata, "path: "+k3sOIDCSigningKeyPath) + assert.Contains(t, udata, "permissions: '0600'") + assert.Contains(t, udata, "-----BEGIN PRIVATE KEY-----") + assert.Contains(t, udata, "FAKEKEY") + + assert.Contains(t, udata, "path: "+k3sOIDCPublicKeyPath) + assert.Contains(t, udata, "-----BEGIN PUBLIC KEY-----") + assert.Contains(t, udata, "FAKEPUB") + + assert.Contains(t, udata, "path: "+k3sConfigPath) + assert.Contains(t, udata, "tls-san:") + assert.Contains(t, udata, " - eks-alpha-lb-001.us-east-1.elb.spinifex.local") + // service-account-key-file must point at the PUBLIC key; the signing key + // (private) is a separate file — pointing key-file at the private key + // crash-loops kube-apiserver. + assert.Contains(t, udata, "service-account-key-file="+k3sOIDCPublicKeyPath) + assert.Contains(t, udata, "service-account-signing-key-file="+k3sOIDCSigningKeyPath) + assert.NotContains(t, udata, "service-account-key-file="+k3sOIDCSigningKeyPath) + assert.Contains(t, udata, "service-account-issuer=https://oidc.spinifex.local/clusters/111122223333/alpha") + assert.Contains(t, udata, "api-audiences=sts.amazonaws.com") + + assert.Contains(t, udata, "path: "+k3sNATSCAPath) + assert.Contains(t, udata, "-----BEGIN CERTIFICATE-----") + assert.Contains(t, udata, "FAKECA") +} + +func TestLaunchK3sServerVM_RunInstancesEmptyReservationRollsBack(t *testing.T) { + vpc, ami := &fakeK3sVPC{}, &fakeK3sAMI{} + inst := &fakeK3sInst{launchOut: &sysinstance.SystemInstanceOutput{}} + + _, err := LaunchK3sServerVM(vpc, inst, ami, validK3sInput()) + require.Error(t, err) + assert.Contains(t, err.Error(), "LaunchSystemInstance returned no instance") + require.Len(t, vpc.deleteCalls, 1, "must roll back ENI when reservation empty") +} + +func TestLaunchK3sServerVM_AMIFilterShape(t *testing.T) { + vpc, inst, ami := &fakeK3sVPC{}, &fakeK3sInst{}, &fakeK3sAMI{} + + _, err := LaunchK3sServerVM(vpc, inst, ami, validK3sInput()) + require.NoError(t, err) + require.Len(t, ami.describeCalls, 1) + filters := ami.describeCalls[0].Filters + require.Len(t, filters, 1) + assert.Equal(t, "tag:"+tags.ManagedByKey, aws.StringValue(filters[0].Name)) + assert.Equal(t, []string{tags.ManagedByEKS}, aws.StringValueSlice(filters[0].Values)) +} + +func TestTerminateK3sServerVM_BothNoopOnEmpty(t *testing.T) { + vpc, inst := &fakeK3sVPC{}, &fakeK3sInst{} + + require.NoError(t, TerminateK3sServerVM(vpc, inst, "111122223333", "", "")) + assert.Empty(t, inst.terminateCalls) + assert.Empty(t, vpc.deleteCalls) +} + +func TestTerminateK3sServerVM_TerminatesInstanceAndDeletesENI(t *testing.T) { + vpc, inst := &fakeK3sVPC{}, &fakeK3sInst{} + + require.NoError(t, TerminateK3sServerVM(vpc, inst, "111122223333", "i-aaa111", "eni-aaa111")) + require.Len(t, inst.terminateCalls, 1) + assert.Equal(t, "i-aaa111", inst.terminateCalls[0]) + require.Len(t, vpc.deleteCalls, 1) + assert.Equal(t, "eni-aaa111", aws.StringValue(vpc.deleteCalls[0].NetworkInterfaceId)) +} + +func TestTerminateK3sServerVM_InstanceErrorReturnedENIStillDeleted(t *testing.T) { + vpc := &fakeK3sVPC{} + inst := &fakeK3sInst{terminateErr: errors.New("IncorrectInstanceState")} + + err := TerminateK3sServerVM(vpc, inst, "111122223333", "i-aaa111", "eni-aaa111") + require.Error(t, err) + assert.Contains(t, err.Error(), "terminate instance i-aaa111") + require.Len(t, vpc.deleteCalls, 1, "ENI delete should still run after instance terminate fails") +} + +func TestTerminateK3sServerVM_ENINotFoundIsIdempotent(t *testing.T) { + // A retried delete-cluster after the instance-terminate cascade already + // removed the ENI: DeleteNetworkInterface returns NotFound. This must be + // treated as success so the SG + KV sweep downstream is not blocked. + vpc := &fakeK3sVPC{deleteErr: errors.New(awserrors.ErrorInvalidNetworkInterfaceIDNotFound)} + inst := &fakeK3sInst{} + + err := TerminateK3sServerVM(vpc, inst, "111122223333", "i-aaa111", "eni-aaa111") + require.NoError(t, err, "ENI already gone must be idempotent success, not a blocking error") + require.Len(t, vpc.deleteCalls, 1) +} + +func TestTerminateK3sServerVM_ENIInUseIsRetryable(t *testing.T) { + // The VM is still terminating async and holds the ENI. InUse must surface + // as an error so the cluster stays DELETING and the reconciler retries. + vpc := &fakeK3sVPC{deleteErr: errors.New(awserrors.ErrorInvalidNetworkInterfaceInUse)} + inst := &fakeK3sInst{} + + err := TerminateK3sServerVM(vpc, inst, "111122223333", "i-aaa111", "eni-aaa111") + require.Error(t, err, "ENI still in use must be retryable, not swallowed") + assert.Contains(t, err.Error(), "delete ENI eni-aaa111") +} + +func TestTerminateK3sServerVM_InstanceAlreadyGoneIsIdempotent(t *testing.T) { + // On a reconciler retry the VM already drained, so TerminateSystemInstance + // returns ErrSystemInstanceNotFound. This must not block teardown; the ENI + // delete still runs. + vpc := &fakeK3sVPC{} + inst := &fakeK3sInst{terminateErr: fmt.Errorf("%w: i-aaa111", sysinstance.ErrSystemInstanceNotFound)} + + err := TerminateK3sServerVM(vpc, inst, "111122223333", "i-aaa111", "eni-aaa111") + require.NoError(t, err, "instance already gone must be idempotent success") + require.Len(t, vpc.deleteCalls, 1, "ENI delete still runs after a tolerated instance-gone") +} + +func assertEC2TaggedAsEKSControlPlane(t *testing.T, ec2Tags []*ec2.Tag, clusterName string) { + t.Helper() + got := map[string]string{} + for _, tg := range ec2Tags { + if tg == nil || tg.Key == nil || tg.Value == nil { + continue + } + got[*tg.Key] = *tg.Value + } + assert.Equal(t, tags.ManagedByEKS, got[tags.ManagedByKey]) + assert.Equal(t, clusterName, got[clusterEKSClusterTagKey]) + assert.Equal(t, clusterEKSRoleControlPlane, got[clusterEKSRoleTagKey]) +} diff --git a/spinifex/handlers/eks/nats_bootstrap.go b/spinifex/handlers/eks/nats_bootstrap.go new file mode 100644 index 000000000..fd7a64dbe --- /dev/null +++ b/spinifex/handlers/eks/nats_bootstrap.go @@ -0,0 +1,386 @@ +package handlers_eks + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "log/slog" + "sync" + "time" + + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" + "github.com/nats-io/nats.go" +) + +// errBootstrapPermanent marks a bootstrap persist failure that no amount of +// retrying will fix — a malformed envelope, a non-base64 CA, or a JWKS whose +// kid does not match the controller-generated keypair. Transient failures +// (KV write blips, JWKS-not-yet-propagated) are NOT wrapped with this, so +// persistWithRetry keeps retrying them. A permanent error fails the cluster. +var errBootstrapPermanent = errors.New("bootstrap permanent error") + +const ( + // bootstrapPersistAttempts bounds inline retries of a single subject's + // persist op on transient errors. A core-NATS publication is delivered + // once, so a transient KV blip must be ridden out in-handler rather than + // by tearing down the subscription and waiting for a re-publish that + // never comes. + bootstrapPersistAttempts = 3 + bootstrapPersistBackoff = 200 * time.Millisecond +) + +// persistWithRetry runs op, retrying on transient errors up to +// bootstrapPersistAttempts. A nil result or an errBootstrapPermanent result +// returns immediately (no retry). +func persistWithRetry(op func() error) error { + var err error + for attempt := 1; attempt <= bootstrapPersistAttempts; attempt++ { + err = op() + if err == nil || errors.Is(err, errBootstrapPermanent) { + return err + } + if attempt < bootstrapPersistAttempts { + time.Sleep(bootstrapPersistBackoff) + } + } + return err +} + +// Bootstrap subject kinds. Full subject is +// "eks.bus.{accountID}.{clusterName}.{kind}" — see BootstrapSubject. +const ( + BootstrapSubjectToken = "k3s-bootstrap-token" //nolint:gosec // subject suffix, not a credential + BootstrapSubjectKubeconfig = "k3s-admin-kubeconfig" + BootstrapSubjectJWKS = "k3s-oidc-jwks" + BootstrapSubjectCA = "k3s-ca" +) + +// BootstrapSubject returns the per-cluster bootstrap subject for one kind. +func BootstrapSubject(accountID, clusterName, kind string) string { + return fmt.Sprintf("eks.bus.%s.%s.%s", accountID, clusterName, kind) +} + +// BootstrapEnvelope is the JSON wire shape every k3s-first-boot.sh +// publication uses. Each subject populates the field matching its kind; the +// rest stay empty. Versioned implicitly by field shape — adding new optional +// fields is backwards-compatible. +type BootstrapEnvelope struct { + // Token is the K3s bootstrap node-token for the cluster (k3s-bootstrap-token). + Token string `json:"token,omitempty"` + // AdminKubeconfig is the cluster-admin kubeconfig YAML + // (k3s-admin-kubeconfig). + AdminKubeconfig string `json:"adminKubeconfig,omitempty"` + // JWKS is the K3s-published RFC 7517 JWKS document. The bootstrap + // subscriber cross-checks it against the controller-generated keypair + // (k3s-oidc-jwks). + JWKS string `json:"jwks,omitempty"` + // CertificateAuthority is the base64-encoded K3s server CA PEM + // (k3s-ca). Written onto meta.CertificateAuthorityB64. + CertificateAuthority string `json:"certificateAuthority,omitempty"` +} + +// NATSBootstrap collects the four single-shot bootstrap publications from a +// K3s server VM, validates the OIDC JWKS against what the controller wrote +// pre-VM, persists each artifact into the per-cluster KV bucket, and writes +// the CA onto meta.CertificateAuthorityB64. The reconciler observes +// completion via the four KV keys + meta.CertificateAuthorityB64. +type NATSBootstrap struct { + nc *nats.Conn + kv nats.KeyValue + masterKey []byte + accountID string + clusterName string +} + +// NewNATSBootstrap validates inputs and returns a ready subscriber. Run must +// be called to actually subscribe; constructor is side-effect-free. +func NewNATSBootstrap(nc *nats.Conn, kv nats.KeyValue, masterKey []byte, accountID, clusterName string) (*NATSBootstrap, error) { + if nc == nil { + return nil, errors.New("eks: NewNATSBootstrap nil nats conn") + } + if kv == nil { + return nil, errors.New("eks: NewNATSBootstrap nil kv") + } + if len(masterKey) == 0 { + return nil, errors.New("eks: NewNATSBootstrap empty master key") + } + if accountID == "" { + return nil, errors.New("eks: NewNATSBootstrap empty accountID") + } + if clusterName == "" { + return nil, errors.New("eks: NewNATSBootstrap empty clusterName") + } + return &NATSBootstrap{ + nc: nc, + kv: kv, + masterKey: masterKey, + accountID: accountID, + clusterName: clusterName, + }, nil +} + +// BootstrapPendingKinds reports which bootstrap subjects still lack their +// persisted artifact for a cluster, so a daemon-restart resume re-subscribes +// only to what is missing. The K3s VM publishes each subject once over core +// NATS and never republishes, so re-waiting on an already-arrived subject +// would deadlock. +// +// JWKS is tracked by its verified-marker (OIDCJWKSVerifiedKey), written by +// persistJWKS only after the cross-check passes — NOT by OIDCJWKSKey, which the +// controller pre-seeds at create time. Until the marker exists the cross-check +// has not positively passed, so resume re-subscribes to JWKS like any other +// pending subject. (If the VM published JWKS during the restart window the +// re-subscribe waits for a republish that never comes and the cluster fails on +// the create timeout — the same one-shot limitation every bootstrap subject +// shares, now applied uniformly so ACTIVE always implies a verified key.) +func BootstrapPendingKinds(kv nats.KeyValue, clusterName string, meta *ClusterMeta) []string { + var pending []string + if _, err := kv.Get(NodeTokenKey(clusterName)); err != nil { + pending = append(pending, BootstrapSubjectToken) + } + if _, err := kv.Get(AdminKubeconfigKey(clusterName)); err != nil { + pending = append(pending, BootstrapSubjectKubeconfig) + } + if _, err := kv.Get(OIDCJWKSVerifiedKey(clusterName)); err != nil { + pending = append(pending, BootstrapSubjectJWKS) + } + if meta == nil || meta.CertificateAuthorityB64 == "" { + pending = append(pending, BootstrapSubjectCA) + } + return pending +} + +type bootstrapSubjectHandler struct { + subject string + kind string + handler func([]byte) error +} + +// subjectHandlers returns the subject/handler entry for each requested kind. +// An unknown kind is an error — callers pass kinds from BootstrapSubject* +// constants. A nil/empty kinds slice means all four. +func (b *NATSBootstrap) subjectHandlers(kinds []string) ([]bootstrapSubjectHandler, error) { + all := map[string]func([]byte) error{ + BootstrapSubjectToken: b.persistToken, + BootstrapSubjectKubeconfig: b.persistKubeconfig, + BootstrapSubjectJWKS: b.persistJWKS, + BootstrapSubjectCA: b.persistCA, + } + if len(kinds) == 0 { + kinds = []string{BootstrapSubjectToken, BootstrapSubjectKubeconfig, BootstrapSubjectJWKS, BootstrapSubjectCA} + } + out := make([]bootstrapSubjectHandler, 0, len(kinds)) + for _, kind := range kinds { + handler, ok := all[kind] + if !ok { + return nil, fmt.Errorf("eks: unknown bootstrap kind %q", kind) + } + out = append(out, bootstrapSubjectHandler{ + subject: BootstrapSubject(b.accountID, b.clusterName, kind), + kind: kind, + handler: handler, + }) + } + return out, nil +} + +// Run subscribes to all four bootstrap subjects and returns when all four have +// arrived, ctx is cancelled, or a handler errors. See RunForKinds. +func (b *NATSBootstrap) Run(ctx context.Context) error { + return b.RunForKinds(ctx, nil) +} + +// RunForKinds subscribes only to the requested bootstrap subjects (nil = all +// four) and returns when: +// - every requested subject has arrived and persisted successfully (nil), +// - ctx is cancelled (ctx.Err()), +// - one of the persistence handlers returns an error (first error). +// +// The subset form drives daemon-restart recovery: a cluster that already has +// some artifacts in KV re-subscribes only to the missing ones, since the K3s +// VM published each subject once over core NATS and will not republish. +// +// Each subject is consumed at most once; subsequent publications on the same +// subject are dropped. Subscriptions are always cleaned up before return. +func (b *NATSBootstrap) RunForKinds(ctx context.Context, kinds []string) error { + subs, err := b.subjectHandlers(kinds) + if err != nil { + return err + } + want := len(subs) + + errCh := make(chan error, want) + doneCh := make(chan struct{}, 1) + var ( + mu sync.Mutex + done = make(map[string]struct{}, want) + ) + + var subscriptions []*nats.Subscription + defer func() { + for _, sub := range subscriptions { + _ = sub.Unsubscribe() + } + }() + + for _, s := range subs { + sub, err := b.nc.Subscribe(s.subject, func(m *nats.Msg) { + mu.Lock() + if _, ok := done[s.kind]; ok { + mu.Unlock() + return + } + mu.Unlock() + if err := persistWithRetry(func() error { return s.handler(m.Data) }); err != nil { + errCh <- fmt.Errorf("persist %s: %w", s.kind, err) + return + } + mu.Lock() + done[s.kind] = struct{}{} + full := len(done) == want + mu.Unlock() + if full { + select { + case doneCh <- struct{}{}: + default: + } + } + }) + if err != nil { + return fmt.Errorf("subscribe %s: %w", s.subject, err) + } + subscriptions = append(subscriptions, sub) + } + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case err := <-errCh: + return err + case <-doneCh: + slog.Info("NATSBootstrap: all requested bootstrap messages received", + "accountID", b.accountID, "cluster", b.clusterName, "kinds", want) + return nil + } + } +} + +func (b *NATSBootstrap) persistToken(data []byte) error { + env, err := unmarshalBootstrapEnvelope(data) + if err != nil { + return err + } + if env.Token == "" { + return fmt.Errorf("%w: token envelope empty", errBootstrapPermanent) + } + ct, err := handlers_iam.EncryptSecret(env.Token, b.masterKey) + if err != nil { + return fmt.Errorf("encrypt token: %w", err) + } + if _, err := b.kv.Put(NodeTokenKey(b.clusterName), []byte(ct)); err != nil { + return fmt.Errorf("kv put %s: %w", NodeTokenKey(b.clusterName), err) + } + return nil +} + +func (b *NATSBootstrap) persistKubeconfig(data []byte) error { + env, err := unmarshalBootstrapEnvelope(data) + if err != nil { + return err + } + if env.AdminKubeconfig == "" { + return fmt.Errorf("%w: adminKubeconfig envelope empty", errBootstrapPermanent) + } + ct, err := handlers_iam.EncryptSecret(env.AdminKubeconfig, b.masterKey) + if err != nil { + return fmt.Errorf("encrypt kubeconfig: %w", err) + } + if _, err := b.kv.Put(AdminKubeconfigKey(b.clusterName), []byte(ct)); err != nil { + return fmt.Errorf("kv put %s: %w", AdminKubeconfigKey(b.clusterName), err) + } + return nil +} + +func (b *NATSBootstrap) persistJWKS(data []byte) error { + env, err := unmarshalBootstrapEnvelope(data) + if err != nil { + return err + } + if env.JWKS == "" { + return fmt.Errorf("%w: jwks envelope empty", errBootstrapPermanent) + } + existing, err := b.kv.Get(OIDCJWKSKey(b.clusterName)) + if err != nil { + return fmt.Errorf("kv get %s: %w", OIDCJWKSKey(b.clusterName), err) + } + if err := assertJWKSMatch(existing.Value(), []byte(env.JWKS)); err != nil { + return err + } + // Record the positive cross-check as a distinct marker. The reconciler + // gates ACTIVE on this key — never on OIDCJWKSKey, which the controller + // pre-seeds at create time — so a cluster cannot reach ACTIVE on an + // unverified or mismatched signing key. + if _, err := b.kv.Put(OIDCJWKSVerifiedKey(b.clusterName), []byte("verified")); err != nil { + return fmt.Errorf("kv put %s: %w", OIDCJWKSVerifiedKey(b.clusterName), err) + } + return nil +} + +func (b *NATSBootstrap) persistCA(data []byte) error { + env, err := unmarshalBootstrapEnvelope(data) + if err != nil { + return err + } + if env.CertificateAuthority == "" { + return fmt.Errorf("%w: CA envelope empty", errBootstrapPermanent) + } + if _, err := base64.StdEncoding.DecodeString(env.CertificateAuthority); err != nil { + return fmt.Errorf("%w: CA not base64: %v", errBootstrapPermanent, err) + } + return SetClusterCertificateAuthority(b.kv, b.clusterName, env.CertificateAuthority) +} + +func unmarshalBootstrapEnvelope(data []byte) (BootstrapEnvelope, error) { + var env BootstrapEnvelope + if err := json.Unmarshal(data, &env); err != nil { + return env, fmt.Errorf("%w: unmarshal envelope: %v", errBootstrapPermanent, err) + } + return env, nil +} + +// assertJWKSMatch verifies that every key in the incoming JWKS matches a +// controller-generated key by both key id (kid) and key type (kty). A kid +// mismatch means the VM generated its own signing key instead of consuming our +// PEM; a kty mismatch (e.g. an RSA key carrying our EC key's kid) means the +// published key cannot be the one the controller distributed even though the +// kid collides. Either rejects the publication. +func assertJWKSMatch(existing, incoming []byte) error { + var ex, in oidcJWKS + if err := json.Unmarshal(existing, &ex); err != nil { + return fmt.Errorf("unmarshal existing JWKS: %w", err) + } + if err := json.Unmarshal(incoming, &in); err != nil { + return fmt.Errorf("%w: unmarshal incoming JWKS: %v", errBootstrapPermanent, err) + } + if len(in.Keys) == 0 { + return fmt.Errorf("%w: incoming JWKS has no keys", errBootstrapPermanent) + } + have := make(map[string]oidcJWK, len(ex.Keys)) + for _, k := range ex.Keys { + have[k.Kid] = k + } + for _, k := range in.Keys { + exKey, ok := have[k.Kid] + if !ok { + return fmt.Errorf("%w: bootstrap JWKS kid %q does not match generated keypair", errBootstrapPermanent, k.Kid) + } + if k.Kty != exKey.Kty { + return fmt.Errorf("%w: bootstrap JWKS kid %q key type %q does not match generated %q", + errBootstrapPermanent, k.Kid, k.Kty, exKey.Kty) + } + } + return nil +} diff --git a/spinifex/handlers/eks/nats_bootstrap_test.go b/spinifex/handlers/eks/nats_bootstrap_test.go new file mode 100644 index 000000000..08fdc7cef --- /dev/null +++ b/spinifex/handlers/eks/nats_bootstrap_test.go @@ -0,0 +1,390 @@ +package handlers_eks + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + "time" + + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/nats-io/nats.go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + bootstrapTestCluster = "alpha" +) + +var bootstrapTestMasterKey = []byte("0123456789abcdef0123456789abcdef") + +// natsBootstrapHarness wires an embedded JetStream NATS server, a per-account +// KV bucket, a published controller JWKS (so persistJWKS has something to +// cross-check against), and a NATSBootstrap subscriber. +type natsBootstrapHarness struct { + t *testing.T + nc *nats.Conn + kv nats.KeyValue + + subscriber *NATSBootstrap +} + +func newBootstrapHarness(t *testing.T) *natsBootstrapHarness { + t.Helper() + _, nc, js := testutil.StartTestJetStream(t) + kv, err := GetOrCreateAccountBucket(js, testAccountID) + require.NoError(t, err) + + require.NoError(t, PutClusterMeta(kv, sampleClusterMeta(bootstrapTestCluster))) + + // Lay down the controller-generated JWKS so persistJWKS has something + // to cross-check against. + _, _, err = GenerateClusterOIDCKeypair(kv, bootstrapTestCluster, bootstrapTestMasterKey) + require.NoError(t, err) + + sub, err := NewNATSBootstrap(nc, kv, bootstrapTestMasterKey, testAccountID, bootstrapTestCluster) + require.NoError(t, err) + + return &natsBootstrapHarness{t: t, nc: nc, kv: kv, subscriber: sub} +} + +func (h *natsBootstrapHarness) publish(kind string, env BootstrapEnvelope) { + h.t.Helper() + data, err := json.Marshal(env) + require.NoError(h.t, err) + subject := BootstrapSubject(testAccountID, bootstrapTestCluster, kind) + require.NoError(h.t, h.nc.Publish(subject, data)) + require.NoError(h.t, h.nc.Flush()) +} + +func (h *natsBootstrapHarness) loadControllerJWKS() []byte { + h.t.Helper() + entry, err := h.kv.Get(OIDCJWKSKey(bootstrapTestCluster)) + require.NoError(h.t, err) + return entry.Value() +} + +func TestNewNATSBootstrap_RejectsBadInputs(t *testing.T) { + _, nc, js := testutil.StartTestJetStream(t) + kv, err := GetOrCreateAccountBucket(js, testAccountID) + require.NoError(t, err) + + _, err = NewNATSBootstrap(nil, kv, bootstrapTestMasterKey, testAccountID, "alpha") + require.Error(t, err) + _, err = NewNATSBootstrap(nc, nil, bootstrapTestMasterKey, testAccountID, "alpha") + require.Error(t, err) + _, err = NewNATSBootstrap(nc, kv, nil, testAccountID, "alpha") + require.Error(t, err) + _, err = NewNATSBootstrap(nc, kv, bootstrapTestMasterKey, "", "alpha") + require.Error(t, err) + _, err = NewNATSBootstrap(nc, kv, bootstrapTestMasterKey, testAccountID, "") + require.Error(t, err) +} + +func TestNATSBootstrap_HappyPath(t *testing.T) { + h := newBootstrapHarness(t) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + jwks := string(h.loadControllerJWKS()) + caB64 := base64.StdEncoding.EncodeToString([]byte("-----BEGIN CERTIFICATE-----\nfakeCA\n-----END CERTIFICATE-----\n")) + + runErr := make(chan error, 1) + go func() { runErr <- h.subscriber.Run(ctx) }() + + // Give subscriptions time to bind before publishing. + require.Eventually(t, func() bool { + // Probe by sending a no-op flush — once Run is past Subscribe the + // subjects are live; we just need a small yield to the goroutine. + return true + }, 100*time.Millisecond, 10*time.Millisecond) + time.Sleep(50 * time.Millisecond) + + h.publish(BootstrapSubjectToken, BootstrapEnvelope{Token: "k3s::server-node-token::v1"}) + h.publish(BootstrapSubjectKubeconfig, BootstrapEnvelope{AdminKubeconfig: "apiVersion: v1\nkind: Config\n"}) + h.publish(BootstrapSubjectJWKS, BootstrapEnvelope{JWKS: jwks}) + h.publish(BootstrapSubjectCA, BootstrapEnvelope{CertificateAuthority: caB64}) + + select { + case err := <-runErr: + require.NoError(t, err) + case <-time.After(3 * time.Second): + t.Fatal("Run did not return after all four publications") + } + + tokEntry, err := h.kv.Get(NodeTokenKey(bootstrapTestCluster)) + require.NoError(t, err) + dec, err := handlers_iam.DecryptSecret(string(tokEntry.Value()), bootstrapTestMasterKey) + require.NoError(t, err) + assert.Equal(t, "k3s::server-node-token::v1", dec) + + kcEntry, err := h.kv.Get(AdminKubeconfigKey(bootstrapTestCluster)) + require.NoError(t, err) + dec, err = handlers_iam.DecryptSecret(string(kcEntry.Value()), bootstrapTestMasterKey) + require.NoError(t, err) + assert.Equal(t, "apiVersion: v1\nkind: Config\n", dec) + + meta, err := GetClusterMeta(h.kv, bootstrapTestCluster) + require.NoError(t, err) + assert.Equal(t, caB64, meta.CertificateAuthorityB64) + + // The passing JWKS cross-check must record the verified-marker the + // reconciler gates ACTIVE on. + _, err = h.kv.Get(OIDCJWKSVerifiedKey(bootstrapTestCluster)) + require.NoError(t, err, "persistJWKS must write the verified-marker on a passing cross-check") +} + +func TestNATSBootstrap_ContextCancelReturnsErr(t *testing.T) { + h := newBootstrapHarness(t) + ctx, cancel := context.WithCancel(context.Background()) + + runErr := make(chan error, 1) + go func() { runErr <- h.subscriber.Run(ctx) }() + time.Sleep(50 * time.Millisecond) + + cancel() + + select { + case err := <-runErr: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(2 * time.Second): + t.Fatal("Run did not return after ctx cancel") + } +} + +func TestNATSBootstrap_JWKSMismatchReturnsErr(t *testing.T) { + h := newBootstrapHarness(t) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + + runErr := make(chan error, 1) + go func() { runErr <- h.subscriber.Run(ctx) }() + time.Sleep(50 * time.Millisecond) + + // kid mismatch — controller generated a real kid, we send a bogus one. + badJWKS := `{"keys":[{"kty":"EC","kid":"not-the-controller-kid","crv":"P-256","x":"AAA","y":"BBB"}]}` + h.publish(BootstrapSubjectJWKS, BootstrapEnvelope{JWKS: badJWKS}) + + select { + case err := <-runErr: + require.Error(t, err) + assert.Contains(t, err.Error(), "JWKS kid") + case <-time.After(3 * time.Second): + t.Fatal("Run did not return on JWKS mismatch") + } +} + +func TestAssertJWKSMatch_RejectsKtyMismatch(t *testing.T) { + // Same kid as the controller key, but RSA instead of EC — a published key + // that cannot be the one the controller distributed despite the kid + // collision. Must be rejected. + existing := []byte(`{"keys":[{"kty":"EC","kid":"abc","crv":"P-256","x":"AAA","y":"BBB"}]}`) + incoming := []byte(`{"keys":[{"kty":"RSA","kid":"abc","n":"AAA","e":"AQAB"}]}`) + + err := assertJWKSMatch(existing, incoming) + require.Error(t, err) + require.ErrorIs(t, err, errBootstrapPermanent) + assert.Contains(t, err.Error(), "key type") +} + +func TestAssertJWKSMatch_AcceptsMatchingKidAndKty(t *testing.T) { + jwks := []byte(`{"keys":[{"kty":"EC","kid":"abc","crv":"P-256","x":"AAA","y":"BBB"}]}`) + require.NoError(t, assertJWKSMatch(jwks, jwks)) +} + +func TestNATSBootstrap_EmptyEnvelopeFieldsRejected(t *testing.T) { + h := newBootstrapHarness(t) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + + runErr := make(chan error, 1) + go func() { runErr <- h.subscriber.Run(ctx) }() + time.Sleep(50 * time.Millisecond) + + h.publish(BootstrapSubjectToken, BootstrapEnvelope{}) + + select { + case err := <-runErr: + require.Error(t, err) + assert.Contains(t, err.Error(), "token envelope empty") + case <-time.After(3 * time.Second): + t.Fatal("Run did not return on empty envelope") + } +} + +func TestNATSBootstrap_OneShotIgnoresSubsequentMessages(t *testing.T) { + h := newBootstrapHarness(t) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + jwks := string(h.loadControllerJWKS()) + caB64 := base64.StdEncoding.EncodeToString([]byte("ca-pem")) + + runErr := make(chan error, 1) + go func() { runErr <- h.subscriber.Run(ctx) }() + time.Sleep(50 * time.Millisecond) + + // First, send only token so subsequent token publications are dropped. + h.publish(BootstrapSubjectToken, BootstrapEnvelope{Token: "first-token"}) + time.Sleep(50 * time.Millisecond) + // Second token publication has a different value; persistToken should not + // re-encrypt/overwrite because the one-shot map prevents reprocess. + h.publish(BootstrapSubjectToken, BootstrapEnvelope{Token: "second-token"}) + + h.publish(BootstrapSubjectKubeconfig, BootstrapEnvelope{AdminKubeconfig: "kc"}) + h.publish(BootstrapSubjectJWKS, BootstrapEnvelope{JWKS: jwks}) + h.publish(BootstrapSubjectCA, BootstrapEnvelope{CertificateAuthority: caB64}) + + select { + case err := <-runErr: + require.NoError(t, err) + case <-time.After(3 * time.Second): + t.Fatal("Run did not return") + } + + entry, err := h.kv.Get(NodeTokenKey(bootstrapTestCluster)) + require.NoError(t, err) + dec, err := handlers_iam.DecryptSecret(string(entry.Value()), bootstrapTestMasterKey) + require.NoError(t, err) + assert.Equal(t, "first-token", dec, "second token publication must be dropped") +} + +func TestNATSBootstrap_CANotBase64Rejected(t *testing.T) { + h := newBootstrapHarness(t) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + + runErr := make(chan error, 1) + go func() { runErr <- h.subscriber.Run(ctx) }() + time.Sleep(50 * time.Millisecond) + + h.publish(BootstrapSubjectCA, BootstrapEnvelope{CertificateAuthority: "not!base64@@@"}) + + select { + case err := <-runErr: + require.Error(t, err) + assert.Contains(t, err.Error(), "CA not base64") + case <-time.After(3 * time.Second): + t.Fatal("Run did not return") + } +} + +func TestPersistWithRetry(t *testing.T) { + t.Run("succeeds first try", func(t *testing.T) { + calls := 0 + err := persistWithRetry(func() error { calls++; return nil }) + require.NoError(t, err) + assert.Equal(t, 1, calls) + }) + + t.Run("retries transient then succeeds", func(t *testing.T) { + calls := 0 + err := persistWithRetry(func() error { + calls++ + if calls < 2 { + return errors.New("transient kv blip") + } + return nil + }) + require.NoError(t, err) + assert.Equal(t, 2, calls) + }) + + t.Run("permanent error returns immediately without retry", func(t *testing.T) { + calls := 0 + err := persistWithRetry(func() error { + calls++ + return fmt.Errorf("%w: malformed envelope", errBootstrapPermanent) + }) + require.ErrorIs(t, err, errBootstrapPermanent) + assert.Equal(t, 1, calls, "permanent error must not be retried") + }) + + t.Run("transient error exhausts attempts", func(t *testing.T) { + calls := 0 + err := persistWithRetry(func() error { calls++; return errors.New("always transient") }) + require.Error(t, err) + assert.Equal(t, bootstrapPersistAttempts, calls) + }) +} + +func TestBootstrapPendingKinds(t *testing.T) { + h := newBootstrapHarness(t) + + meta, err := GetClusterMeta(h.kv, bootstrapTestCluster) + require.NoError(t, err) + + // Fresh cluster: all four pending. The JWKS verified-marker is absent even + // though the controller pre-seeded OIDCJWKSKey, so JWKS is still pending. + assert.Equal(t, + []string{BootstrapSubjectToken, BootstrapSubjectKubeconfig, BootstrapSubjectJWKS, BootstrapSubjectCA}, + BootstrapPendingKinds(h.kv, bootstrapTestCluster, meta)) + + // Persist token only -> token drops out of pending. + ct, err := handlers_iam.EncryptSecret("tok", bootstrapTestMasterKey) + require.NoError(t, err) + _, err = h.kv.Put(NodeTokenKey(bootstrapTestCluster), []byte(ct)) + require.NoError(t, err) + assert.Equal(t, + []string{BootstrapSubjectKubeconfig, BootstrapSubjectJWKS, BootstrapSubjectCA}, + BootstrapPendingKinds(h.kv, bootstrapTestCluster, meta)) + + // Persist kubeconfig + CA-on-meta -> only JWKS (unverified) still pending. + _, err = h.kv.Put(AdminKubeconfigKey(bootstrapTestCluster), []byte(ct)) + require.NoError(t, err) + meta.CertificateAuthorityB64 = "ca-b64" + assert.Equal(t, + []string{BootstrapSubjectJWKS}, + BootstrapPendingKinds(h.kv, bootstrapTestCluster, meta)) + + // Write the JWKS verified-marker (persistJWKS does this on a passing + // cross-check) -> nothing pending. + _, err = h.kv.Put(OIDCJWKSVerifiedKey(bootstrapTestCluster), []byte("verified")) + require.NoError(t, err) + assert.Empty(t, BootstrapPendingKinds(h.kv, bootstrapTestCluster, meta)) +} + +func TestNATSBootstrap_RunForKindsWaitsOnlyForSubset(t *testing.T) { + h := newBootstrapHarness(t) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + caB64 := base64.StdEncoding.EncodeToString([]byte("ca-pem")) + + runErr := make(chan error, 1) + go func() { runErr <- h.subscriber.RunForKinds(ctx, []string{BootstrapSubjectCA}) }() + time.Sleep(50 * time.Millisecond) + + // Only publish CA; the subset subscriber must complete without token, + // kubeconfig, or JWKS ever arriving (they were consumed pre-restart). + h.publish(BootstrapSubjectCA, BootstrapEnvelope{CertificateAuthority: caB64}) + + select { + case err := <-runErr: + require.NoError(t, err) + case <-time.After(3 * time.Second): + t.Fatal("RunForKinds did not return after CA-only publication") + } + + meta, err := GetClusterMeta(h.kv, bootstrapTestCluster) + require.NoError(t, err) + assert.Equal(t, caB64, meta.CertificateAuthorityB64) +} + +func TestNATSBootstrap_RunForKindsRejectsUnknownKind(t *testing.T) { + h := newBootstrapHarness(t) + err := h.subscriber.RunForKinds(context.Background(), []string{"bogus-kind"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "unknown bootstrap kind") +} + +func TestBootstrapSubject_Shape(t *testing.T) { + got := BootstrapSubject("111122223333", "alpha", BootstrapSubjectToken) + assert.Equal(t, "eks.bus.111122223333.alpha.k3s-bootstrap-token", got) + assert.True(t, strings.HasPrefix(got, "eks.bus.")) +} diff --git a/spinifex/handlers/eks/nlb.go b/spinifex/handlers/eks/nlb.go new file mode 100644 index 000000000..dc7fc8b38 --- /dev/null +++ b/spinifex/handlers/eks/nlb.go @@ -0,0 +1,353 @@ +package handlers_eks + +import ( + "errors" + "fmt" + "log/slog" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/elbv2" + "github.com/mulgadc/spinifex/spinifex/tags" +) + +// nlbProvisioner is the subset of handlers_elbv2.ELBv2Service that the +// cluster NLB helpers need. Narrow so tests can fake with hand-rolled +// recorders without implementing the full ELBv2 surface. +type nlbProvisioner interface { + CreateLoadBalancer(input *elbv2.CreateLoadBalancerInput, accountID string) (*elbv2.CreateLoadBalancerOutput, error) + DescribeLoadBalancers(input *elbv2.DescribeLoadBalancersInput, accountID string) (*elbv2.DescribeLoadBalancersOutput, error) + DeleteLoadBalancer(input *elbv2.DeleteLoadBalancerInput, accountID string) (*elbv2.DeleteLoadBalancerOutput, error) + + CreateTargetGroup(input *elbv2.CreateTargetGroupInput, accountID string) (*elbv2.CreateTargetGroupOutput, error) + DescribeTargetGroups(input *elbv2.DescribeTargetGroupsInput, accountID string) (*elbv2.DescribeTargetGroupsOutput, error) + DeleteTargetGroup(input *elbv2.DeleteTargetGroupInput, accountID string) (*elbv2.DeleteTargetGroupOutput, error) + + CreateListener(input *elbv2.CreateListenerInput, accountID string) (*elbv2.CreateListenerOutput, error) + DescribeListeners(input *elbv2.DescribeListenersInput, accountID string) (*elbv2.DescribeListenersOutput, error) + + RegisterTargets(input *elbv2.RegisterTargetsInput, accountID string) (*elbv2.RegisterTargetsOutput, error) + DeregisterTargets(input *elbv2.DeregisterTargetsInput, accountID string) (*elbv2.DeregisterTargetsOutput, error) +} + +// AWS load-balancer + target-group names are bounded to 32 alphanumeric or +// hyphen characters. EnsureClusterNLB rejects clusters whose derived name +// would exceed this before issuing the create call so callers see a clear +// error instead of a generic ELBv2 InvalidParameterValue. +const maxELBv2NameLen = 32 + +// k3sAPIServerPort is the in-VM K3s api-server bind port. The NLB target +// group sends traffic to this port on the K3s server VM's ENI. +const k3sAPIServerPort int64 = 6443 + +// clusterNLBListenPort is the customer-facing port the cluster NLB exposes. +// kubectl + the AWS SDK both expect TLS on 443. +const clusterNLBListenPort int64 = 443 + +// ClusterNLB is the in-process tuple produced by EnsureClusterNLB. Each ARN +// is persisted into ClusterMeta so the reconciler + DeleteCluster can act on +// the resources without re-resolving by name. +type ClusterNLB struct { + LoadBalancerArn string + TargetGroupArn string + ListenerArn string + DNSName string +} + +// ClusterNLBName returns the deterministic NLB name for a cluster. Stable so +// DeleteCluster can locate the NLB from clusterName alone after a daemon +// restart that lost the in-memory ARN. +func ClusterNLBName(clusterName string) string { + return "eks-" + clusterName +} + +// ClusterTargetGroupName returns the deterministic target-group name for a +// cluster's control-plane TG. +func ClusterTargetGroupName(clusterName string) string { + return "eks-" + clusterName + "-cp" +} + +// EnsureClusterNLB provisions (or returns) the cluster's NLB + target group + +// :443→:6443 TCP listener. Idempotent on clusterName: existing resources are +// reused so DeleteCluster failure-retry and reconciler crash-recovery both +// converge without duplicate creates. +// +// The K3s server VM ENI IP is NOT registered here — Stage 4 +// (k3s_server_vm.go) calls RegisterClusterTarget once the VM has its ENI IP. +func EnsureClusterNLB(nlbp nlbProvisioner, accountID, clusterName string, subnetIDs []string) (*ClusterNLB, error) { + if clusterName == "" { + return nil, errors.New("eks: EnsureClusterNLB empty cluster name") + } + if len(subnetIDs) == 0 { + return nil, errors.New("eks: EnsureClusterNLB empty subnet list") + } + lbName := ClusterNLBName(clusterName) + tgName := ClusterTargetGroupName(clusterName) + if len(lbName) > maxELBv2NameLen { + return nil, fmt.Errorf("eks: NLB name %q exceeds %d chars (cluster name too long)", lbName, maxELBv2NameLen) + } + if len(tgName) > maxELBv2NameLen { + return nil, fmt.Errorf("eks: TG name %q exceeds %d chars (cluster name too long)", tgName, maxELBv2NameLen) + } + + out := &ClusterNLB{} + + if err := ensureClusterLB(nlbp, accountID, clusterName, lbName, subnetIDs, out); err != nil { + return nil, err + } + if err := ensureClusterTG(nlbp, accountID, clusterName, tgName, out); err != nil { + return nil, err + } + if err := ensureClusterListener(nlbp, accountID, lbName, out); err != nil { + return nil, err + } + return out, nil +} + +// RegisterClusterTarget attaches the K3s server ENI IP to the cluster TG so +// the NLB starts forwarding :443 traffic to :6443 on the VM. The ELBv2 +// RegisterTargets impl already dedupes on (id, port), so re-invocation by +// the reconciler is idempotent. +func RegisterClusterTarget(nlbp nlbProvisioner, accountID, tgArn, eniIP string) error { + if tgArn == "" { + return errors.New("eks: RegisterClusterTarget empty TG arn") + } + if eniIP == "" { + return errors.New("eks: RegisterClusterTarget empty ENI IP") + } + if _, err := nlbp.RegisterTargets(&elbv2.RegisterTargetsInput{ + TargetGroupArn: aws.String(tgArn), + Targets: []*elbv2.TargetDescription{{ + Id: aws.String(eniIP), + Port: aws.Int64(k3sAPIServerPort), + }}, + }, accountID); err != nil { + return fmt.Errorf("register target %s on TG %s: %w", eniIP, tgArn, err) + } + return nil +} + +// DeregisterClusterTarget removes the K3s server ENI IP from the cluster TG. +// Called by DeleteCluster before VM termination so health checks stop hitting +// a dying target. +func DeregisterClusterTarget(nlbp nlbProvisioner, accountID, tgArn, eniIP string) error { + if tgArn == "" { + return errors.New("eks: DeregisterClusterTarget empty TG arn") + } + if eniIP == "" { + return errors.New("eks: DeregisterClusterTarget empty ENI IP") + } + if _, err := nlbp.DeregisterTargets(&elbv2.DeregisterTargetsInput{ + TargetGroupArn: aws.String(tgArn), + Targets: []*elbv2.TargetDescription{{ + Id: aws.String(eniIP), + Port: aws.Int64(k3sAPIServerPort), + }}, + }, accountID); err != nil { + return fmt.Errorf("deregister target %s on TG %s: %w", eniIP, tgArn, err) + } + return nil +} + +// DeleteClusterNLB tears down the cluster NLB + TG. DeleteLoadBalancer +// cascades listener cleanup inside the ELBv2 impl, so listeners do not need +// an explicit delete pass. Missing resources are no-ops to keep DeleteCluster +// retries safe. Both sub-sweeps always run; LB delete error takes precedence +// over TG delete error when both fail. +func DeleteClusterNLB(nlbp nlbProvisioner, accountID, clusterName string) error { + if clusterName == "" { + return errors.New("eks: DeleteClusterNLB empty cluster name") + } + lbErr := deleteClusterLB(nlbp, accountID, ClusterNLBName(clusterName)) + tgErr := deleteClusterTG(nlbp, accountID, ClusterTargetGroupName(clusterName)) + if lbErr != nil { + return lbErr + } + return tgErr +} + +func deleteClusterLB(nlbp nlbProvisioner, accountID, lbName string) error { + lb, err := lookupLBByName(nlbp, accountID, lbName) + if err != nil { + slog.Warn("DeleteClusterNLB: LB lookup failed", "name", lbName, "err", err) + return err + } + if lb == nil || lb.LoadBalancerArn == nil { + return nil + } + if _, err := nlbp.DeleteLoadBalancer(&elbv2.DeleteLoadBalancerInput{LoadBalancerArn: lb.LoadBalancerArn}, accountID); err != nil { + slog.Warn("DeleteClusterNLB: delete NLB failed", "name", lbName, "err", err) + return fmt.Errorf("delete NLB %s: %w", lbName, err) + } + return nil +} + +func deleteClusterTG(nlbp nlbProvisioner, accountID, tgName string) error { + tg, err := lookupTGByName(nlbp, accountID, tgName) + if err != nil { + slog.Warn("DeleteClusterNLB: TG lookup failed", "name", tgName, "err", err) + return err + } + if tg == nil || tg.TargetGroupArn == nil { + return nil + } + if _, err := nlbp.DeleteTargetGroup(&elbv2.DeleteTargetGroupInput{TargetGroupArn: tg.TargetGroupArn}, accountID); err != nil { + slog.Warn("DeleteClusterNLB: delete TG failed", "name", tgName, "err", err) + return fmt.Errorf("delete TG %s: %w", tgName, err) + } + return nil +} + +func ensureClusterLB(nlbp nlbProvisioner, accountID, clusterName, lbName string, subnetIDs []string, out *ClusterNLB) error { + if lb, err := lookupLBByName(nlbp, accountID, lbName); err != nil { + return err + } else if lb != nil { + out.LoadBalancerArn = aws.StringValue(lb.LoadBalancerArn) + out.DNSName = aws.StringValue(lb.DNSName) + if out.LoadBalancerArn == "" || out.DNSName == "" { + return fmt.Errorf("eks: existing NLB %s missing arn or DNS name", lbName) + } + return nil + } + + created, err := nlbp.CreateLoadBalancer(&elbv2.CreateLoadBalancerInput{ + Name: aws.String(lbName), + Type: aws.String(elbv2.LoadBalancerTypeEnumNetwork), + Scheme: aws.String(elbv2.LoadBalancerSchemeEnumInternal), + Subnets: aws.StringSlice(subnetIDs), + Tags: []*elbv2.Tag{ + {Key: aws.String(tags.ManagedByKey), Value: aws.String(tags.ManagedByEKS)}, + {Key: aws.String(clusterEKSClusterTagKey), Value: aws.String(clusterName)}, + }, + }, accountID) + if err != nil { + return fmt.Errorf("create NLB %s: %w", lbName, err) + } + if created == nil || len(created.LoadBalancers) == 0 || created.LoadBalancers[0] == nil { + return fmt.Errorf("eks: CreateLoadBalancer returned no LB for %s", lbName) + } + lb := created.LoadBalancers[0] + out.LoadBalancerArn = aws.StringValue(lb.LoadBalancerArn) + out.DNSName = aws.StringValue(lb.DNSName) + if out.LoadBalancerArn == "" || out.DNSName == "" { + return fmt.Errorf("eks: CreateLoadBalancer returned empty arn or DNS name for %s", lbName) + } + return nil +} + +func ensureClusterTG(nlbp nlbProvisioner, accountID, clusterName, tgName string, out *ClusterNLB) error { + if tg, err := lookupTGByName(nlbp, accountID, tgName); err != nil { + return err + } else if tg != nil { + out.TargetGroupArn = aws.StringValue(tg.TargetGroupArn) + if out.TargetGroupArn == "" { + return fmt.Errorf("eks: existing TG %s missing arn", tgName) + } + return nil + } + + created, err := nlbp.CreateTargetGroup(&elbv2.CreateTargetGroupInput{ + Name: aws.String(tgName), + Protocol: aws.String(elbv2.ProtocolEnumTcp), + Port: aws.Int64(k3sAPIServerPort), + TargetType: aws.String(elbv2.TargetTypeEnumIp), + Tags: []*elbv2.Tag{ + {Key: aws.String(tags.ManagedByKey), Value: aws.String(tags.ManagedByEKS)}, + {Key: aws.String(clusterEKSClusterTagKey), Value: aws.String(clusterName)}, + }, + }, accountID) + if err != nil { + return fmt.Errorf("create TG %s: %w", tgName, err) + } + if created == nil || len(created.TargetGroups) == 0 || created.TargetGroups[0] == nil { + return fmt.Errorf("eks: CreateTargetGroup returned no TG for %s", tgName) + } + out.TargetGroupArn = aws.StringValue(created.TargetGroups[0].TargetGroupArn) + if out.TargetGroupArn == "" { + return fmt.Errorf("eks: CreateTargetGroup returned empty arn for %s", tgName) + } + return nil +} + +func ensureClusterListener(nlbp nlbProvisioner, accountID, lbName string, out *ClusterNLB) error { + if l, err := lookupListenerByPort(nlbp, accountID, out.LoadBalancerArn, clusterNLBListenPort); err != nil { + return err + } else if l != nil { + out.ListenerArn = aws.StringValue(l.ListenerArn) + if out.ListenerArn == "" { + return fmt.Errorf("eks: existing listener for %s:%d missing arn", lbName, clusterNLBListenPort) + } + return nil + } + + created, err := nlbp.CreateListener(&elbv2.CreateListenerInput{ + LoadBalancerArn: aws.String(out.LoadBalancerArn), + Protocol: aws.String(elbv2.ProtocolEnumTcp), + Port: aws.Int64(clusterNLBListenPort), + DefaultActions: []*elbv2.Action{{ + Type: aws.String(elbv2.ActionTypeEnumForward), + TargetGroupArn: aws.String(out.TargetGroupArn), + }}, + }, accountID) + if err != nil { + return fmt.Errorf("create listener %s:%d: %w", lbName, clusterNLBListenPort, err) + } + if created == nil || len(created.Listeners) == 0 || created.Listeners[0] == nil { + return fmt.Errorf("eks: CreateListener returned no listener for %s:%d", lbName, clusterNLBListenPort) + } + out.ListenerArn = aws.StringValue(created.Listeners[0].ListenerArn) + if out.ListenerArn == "" { + return fmt.Errorf("eks: CreateListener returned empty arn for %s:%d", lbName, clusterNLBListenPort) + } + return nil +} + +func lookupLBByName(nlbp nlbProvisioner, accountID, name string) (*elbv2.LoadBalancer, error) { + out, err := nlbp.DescribeLoadBalancers(&elbv2.DescribeLoadBalancersInput{ + Names: aws.StringSlice([]string{name}), + }, accountID) + if err != nil { + return nil, fmt.Errorf("describe NLB %s: %w", name, err) + } + for _, lb := range out.LoadBalancers { + if lb != nil { + return lb, nil + } + } + return nil, nil +} + +func lookupTGByName(nlbp nlbProvisioner, accountID, name string) (*elbv2.TargetGroup, error) { + out, err := nlbp.DescribeTargetGroups(&elbv2.DescribeTargetGroupsInput{ + Names: aws.StringSlice([]string{name}), + }, accountID) + if err != nil { + return nil, fmt.Errorf("describe TG %s: %w", name, err) + } + for _, tg := range out.TargetGroups { + if tg != nil { + return tg, nil + } + } + return nil, nil +} + +func lookupListenerByPort(nlbp nlbProvisioner, accountID, lbArn string, port int64) (*elbv2.Listener, error) { + if lbArn == "" { + return nil, errors.New("eks: lookupListenerByPort empty LB arn") + } + out, err := nlbp.DescribeListeners(&elbv2.DescribeListenersInput{ + LoadBalancerArn: aws.String(lbArn), + }, accountID) + if err != nil { + return nil, fmt.Errorf("describe listeners on %s: %w", lbArn, err) + } + for _, l := range out.Listeners { + if l == nil || l.Port == nil { + continue + } + if *l.Port == port { + return l, nil + } + } + return nil, nil +} diff --git a/spinifex/handlers/eks/nlb_test.go b/spinifex/handlers/eks/nlb_test.go new file mode 100644 index 000000000..0b60d9471 --- /dev/null +++ b/spinifex/handlers/eks/nlb_test.go @@ -0,0 +1,388 @@ +package handlers_eks + +import ( + "errors" + "strings" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/elbv2" + "github.com/mulgadc/spinifex/spinifex/tags" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeNLBProvisioner struct { + createLBCalls []*elbv2.CreateLoadBalancerInput + describeLBCalls []*elbv2.DescribeLoadBalancersInput + deleteLBCalls []*elbv2.DeleteLoadBalancerInput + createTGCalls []*elbv2.CreateTargetGroupInput + describeTGCalls []*elbv2.DescribeTargetGroupsInput + deleteTGCalls []*elbv2.DeleteTargetGroupInput + createListenerCalls []*elbv2.CreateListenerInput + describeListeners []*elbv2.DescribeListenersInput + registerCalls []*elbv2.RegisterTargetsInput + deregisterCalls []*elbv2.DeregisterTargetsInput + + lbByName map[string]*elbv2.LoadBalancer + tgByName map[string]*elbv2.TargetGroup + listenerByPort map[string]map[int64]*elbv2.Listener // lbArn → port → listener + + createLBOut *elbv2.CreateLoadBalancerOutput + createTGOut *elbv2.CreateTargetGroupOutput + createListenerOut *elbv2.CreateListenerOutput + createLBErr error + createTGErr error + createListenerErr error + deleteLBErr error + deleteTGErr error + registerErr error + deregisterErr error +} + +var _ nlbProvisioner = (*fakeNLBProvisioner)(nil) + +func newFakeNLBProvisioner() *fakeNLBProvisioner { + return &fakeNLBProvisioner{ + lbByName: map[string]*elbv2.LoadBalancer{}, + tgByName: map[string]*elbv2.TargetGroup{}, + listenerByPort: map[string]map[int64]*elbv2.Listener{}, + } +} + +func (f *fakeNLBProvisioner) CreateLoadBalancer(input *elbv2.CreateLoadBalancerInput, _ string) (*elbv2.CreateLoadBalancerOutput, error) { + f.createLBCalls = append(f.createLBCalls, input) + if f.createLBErr != nil { + return nil, f.createLBErr + } + if f.createLBOut == nil { + name := aws.StringValue(input.Name) + arn := "arn:aws:elasticloadbalancing:us-east-1:111122223333:loadbalancer/net/" + name + "/lb-001" + dns := name + "-lb-001.us-east-1.elb.spinifex.local" + f.createLBOut = &elbv2.CreateLoadBalancerOutput{ + LoadBalancers: []*elbv2.LoadBalancer{{ + LoadBalancerArn: aws.String(arn), + LoadBalancerName: aws.String(name), + DNSName: aws.String(dns), + Type: aws.String(elbv2.LoadBalancerTypeEnumNetwork), + Scheme: aws.String(elbv2.LoadBalancerSchemeEnumInternal), + }}, + } + } + out := f.createLBOut + if len(out.LoadBalancers) > 0 { + name := aws.StringValue(out.LoadBalancers[0].LoadBalancerName) + f.lbByName[name] = out.LoadBalancers[0] + } + return out, nil +} + +func (f *fakeNLBProvisioner) DescribeLoadBalancers(input *elbv2.DescribeLoadBalancersInput, _ string) (*elbv2.DescribeLoadBalancersOutput, error) { + f.describeLBCalls = append(f.describeLBCalls, input) + out := &elbv2.DescribeLoadBalancersOutput{} + for _, n := range input.Names { + if n == nil { + continue + } + if lb, ok := f.lbByName[*n]; ok { + out.LoadBalancers = append(out.LoadBalancers, lb) + } + } + return out, nil +} + +func (f *fakeNLBProvisioner) DeleteLoadBalancer(input *elbv2.DeleteLoadBalancerInput, _ string) (*elbv2.DeleteLoadBalancerOutput, error) { + f.deleteLBCalls = append(f.deleteLBCalls, input) + if f.deleteLBErr != nil { + return nil, f.deleteLBErr + } + if input.LoadBalancerArn != nil { + for name, lb := range f.lbByName { + if aws.StringValue(lb.LoadBalancerArn) == *input.LoadBalancerArn { + delete(f.lbByName, name) + } + } + } + return &elbv2.DeleteLoadBalancerOutput{}, nil +} + +func (f *fakeNLBProvisioner) CreateTargetGroup(input *elbv2.CreateTargetGroupInput, _ string) (*elbv2.CreateTargetGroupOutput, error) { + f.createTGCalls = append(f.createTGCalls, input) + if f.createTGErr != nil { + return nil, f.createTGErr + } + if f.createTGOut == nil { + name := aws.StringValue(input.Name) + arn := "arn:aws:elasticloadbalancing:us-east-1:111122223333:targetgroup/" + name + "/tg-001" + f.createTGOut = &elbv2.CreateTargetGroupOutput{ + TargetGroups: []*elbv2.TargetGroup{{ + TargetGroupArn: aws.String(arn), + TargetGroupName: aws.String(name), + Protocol: input.Protocol, + Port: input.Port, + TargetType: input.TargetType, + }}, + } + } + out := f.createTGOut + if len(out.TargetGroups) > 0 { + name := aws.StringValue(out.TargetGroups[0].TargetGroupName) + f.tgByName[name] = out.TargetGroups[0] + } + return out, nil +} + +func (f *fakeNLBProvisioner) DescribeTargetGroups(input *elbv2.DescribeTargetGroupsInput, _ string) (*elbv2.DescribeTargetGroupsOutput, error) { + f.describeTGCalls = append(f.describeTGCalls, input) + out := &elbv2.DescribeTargetGroupsOutput{} + for _, n := range input.Names { + if n == nil { + continue + } + if tg, ok := f.tgByName[*n]; ok { + out.TargetGroups = append(out.TargetGroups, tg) + } + } + return out, nil +} + +func (f *fakeNLBProvisioner) DeleteTargetGroup(input *elbv2.DeleteTargetGroupInput, _ string) (*elbv2.DeleteTargetGroupOutput, error) { + f.deleteTGCalls = append(f.deleteTGCalls, input) + if f.deleteTGErr != nil { + return nil, f.deleteTGErr + } + if input.TargetGroupArn != nil { + for name, tg := range f.tgByName { + if aws.StringValue(tg.TargetGroupArn) == *input.TargetGroupArn { + delete(f.tgByName, name) + } + } + } + return &elbv2.DeleteTargetGroupOutput{}, nil +} + +func (f *fakeNLBProvisioner) CreateListener(input *elbv2.CreateListenerInput, _ string) (*elbv2.CreateListenerOutput, error) { + f.createListenerCalls = append(f.createListenerCalls, input) + if f.createListenerErr != nil { + return nil, f.createListenerErr + } + if f.createListenerOut == nil { + lbArn := aws.StringValue(input.LoadBalancerArn) + port := aws.Int64Value(input.Port) + listenerArn := lbArn + "/listener/lst-001" + f.createListenerOut = &elbv2.CreateListenerOutput{ + Listeners: []*elbv2.Listener{{ + ListenerArn: aws.String(listenerArn), + LoadBalancerArn: input.LoadBalancerArn, + Port: input.Port, + Protocol: input.Protocol, + DefaultActions: input.DefaultActions, + }}, + } + if f.listenerByPort[lbArn] == nil { + f.listenerByPort[lbArn] = map[int64]*elbv2.Listener{} + } + f.listenerByPort[lbArn][port] = f.createListenerOut.Listeners[0] + } + return f.createListenerOut, nil +} + +func (f *fakeNLBProvisioner) DescribeListeners(input *elbv2.DescribeListenersInput, _ string) (*elbv2.DescribeListenersOutput, error) { + f.describeListeners = append(f.describeListeners, input) + out := &elbv2.DescribeListenersOutput{} + if input.LoadBalancerArn != nil { + for _, l := range f.listenerByPort[*input.LoadBalancerArn] { + out.Listeners = append(out.Listeners, l) + } + } + return out, nil +} + +func (f *fakeNLBProvisioner) RegisterTargets(input *elbv2.RegisterTargetsInput, _ string) (*elbv2.RegisterTargetsOutput, error) { + f.registerCalls = append(f.registerCalls, input) + if f.registerErr != nil { + return nil, f.registerErr + } + return &elbv2.RegisterTargetsOutput{}, nil +} + +func (f *fakeNLBProvisioner) DeregisterTargets(input *elbv2.DeregisterTargetsInput, _ string) (*elbv2.DeregisterTargetsOutput, error) { + f.deregisterCalls = append(f.deregisterCalls, input) + if f.deregisterErr != nil { + return nil, f.deregisterErr + } + return &elbv2.DeregisterTargetsOutput{}, nil +} + +func TestEnsureClusterNLB_EmptyInputsRejected(t *testing.T) { + nlbp := newFakeNLBProvisioner() + + _, err := EnsureClusterNLB(nlbp, "111122223333", "", []string{"subnet-aaa"}) + require.Error(t, err) + + _, err = EnsureClusterNLB(nlbp, "111122223333", "alpha", nil) + require.Error(t, err) + + assert.Empty(t, nlbp.createLBCalls) +} + +func TestEnsureClusterNLB_NameTooLongRejected(t *testing.T) { + nlbp := newFakeNLBProvisioner() + + longName := strings.Repeat("x", maxELBv2NameLen) // "eks-" + 32x = 36 chars + _, err := EnsureClusterNLB(nlbp, "111122223333", longName, []string{"subnet-aaa"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "exceeds") + assert.Empty(t, nlbp.createLBCalls) +} + +func TestEnsureClusterNLB_FreshCreatesAllThree(t *testing.T) { + nlbp := newFakeNLBProvisioner() + + out, err := EnsureClusterNLB(nlbp, "111122223333", "alpha", []string{"subnet-aaa", "subnet-bbb"}) + require.NoError(t, err) + require.NotNil(t, out) + assert.NotEmpty(t, out.LoadBalancerArn) + assert.NotEmpty(t, out.TargetGroupArn) + assert.NotEmpty(t, out.ListenerArn) + assert.Contains(t, out.DNSName, "eks-alpha") + + require.Len(t, nlbp.createLBCalls, 1) + lbIn := nlbp.createLBCalls[0] + assert.Equal(t, "eks-alpha", aws.StringValue(lbIn.Name)) + assert.Equal(t, elbv2.LoadBalancerTypeEnumNetwork, aws.StringValue(lbIn.Type)) + assert.Equal(t, elbv2.LoadBalancerSchemeEnumInternal, aws.StringValue(lbIn.Scheme)) + assert.Equal(t, []string{"subnet-aaa", "subnet-bbb"}, aws.StringValueSlice(lbIn.Subnets)) + assertELBv2TaggedAsEKS(t, lbIn.Tags, "alpha") + + require.Len(t, nlbp.createTGCalls, 1) + tgIn := nlbp.createTGCalls[0] + assert.Equal(t, "eks-alpha-cp", aws.StringValue(tgIn.Name)) + assert.Equal(t, elbv2.ProtocolEnumTcp, aws.StringValue(tgIn.Protocol)) + assert.Equal(t, k3sAPIServerPort, aws.Int64Value(tgIn.Port)) + assert.Equal(t, elbv2.TargetTypeEnumIp, aws.StringValue(tgIn.TargetType)) + assertELBv2TaggedAsEKS(t, tgIn.Tags, "alpha") + + require.Len(t, nlbp.createListenerCalls, 1) + lstIn := nlbp.createListenerCalls[0] + assert.Equal(t, out.LoadBalancerArn, aws.StringValue(lstIn.LoadBalancerArn)) + assert.Equal(t, elbv2.ProtocolEnumTcp, aws.StringValue(lstIn.Protocol)) + assert.Equal(t, clusterNLBListenPort, aws.Int64Value(lstIn.Port)) + require.Len(t, lstIn.DefaultActions, 1) + assert.Equal(t, elbv2.ActionTypeEnumForward, aws.StringValue(lstIn.DefaultActions[0].Type)) + assert.Equal(t, out.TargetGroupArn, aws.StringValue(lstIn.DefaultActions[0].TargetGroupArn)) +} + +func TestEnsureClusterNLB_IdempotentReusesExisting(t *testing.T) { + nlbp := newFakeNLBProvisioner() + + first, err := EnsureClusterNLB(nlbp, "111122223333", "alpha", []string{"subnet-aaa"}) + require.NoError(t, err) + + createLBCount := len(nlbp.createLBCalls) + createTGCount := len(nlbp.createTGCalls) + createListenerCount := len(nlbp.createListenerCalls) + + second, err := EnsureClusterNLB(nlbp, "111122223333", "alpha", []string{"subnet-aaa"}) + require.NoError(t, err) + + assert.Equal(t, first.LoadBalancerArn, second.LoadBalancerArn) + assert.Equal(t, first.TargetGroupArn, second.TargetGroupArn) + assert.Equal(t, first.ListenerArn, second.ListenerArn) + assert.Equal(t, first.DNSName, second.DNSName) + + assert.Equal(t, createLBCount, len(nlbp.createLBCalls), "no new LB create on idempotent call") + assert.Equal(t, createTGCount, len(nlbp.createTGCalls), "no new TG create on idempotent call") + assert.Equal(t, createListenerCount, len(nlbp.createListenerCalls), "no new listener create on idempotent call") +} + +func TestEnsureClusterNLB_LBCreateErrorSurfaced(t *testing.T) { + nlbp := newFakeNLBProvisioner() + nlbp.createLBErr = errors.New("InsufficientCapacity") + + _, err := EnsureClusterNLB(nlbp, "111122223333", "alpha", []string{"subnet-aaa"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "create NLB eks-alpha") + assert.Empty(t, nlbp.createTGCalls, "TG create should not run when LB create fails") +} + +func TestRegisterClusterTarget_PostsENIIPAndAPIPort(t *testing.T) { + nlbp := newFakeNLBProvisioner() + + err := RegisterClusterTarget(nlbp, "111122223333", "arn:tg/alpha", "10.0.1.42") + require.NoError(t, err) + require.Len(t, nlbp.registerCalls, 1) + + in := nlbp.registerCalls[0] + assert.Equal(t, "arn:tg/alpha", aws.StringValue(in.TargetGroupArn)) + require.Len(t, in.Targets, 1) + assert.Equal(t, "10.0.1.42", aws.StringValue(in.Targets[0].Id)) + assert.Equal(t, k3sAPIServerPort, aws.Int64Value(in.Targets[0].Port)) +} + +func TestRegisterClusterTarget_EmptyInputsRejected(t *testing.T) { + nlbp := newFakeNLBProvisioner() + require.Error(t, RegisterClusterTarget(nlbp, "111122223333", "", "10.0.1.42")) + require.Error(t, RegisterClusterTarget(nlbp, "111122223333", "arn:tg/alpha", "")) + assert.Empty(t, nlbp.registerCalls) +} + +func TestDeregisterClusterTarget_PostsENIIPAndAPIPort(t *testing.T) { + nlbp := newFakeNLBProvisioner() + + err := DeregisterClusterTarget(nlbp, "111122223333", "arn:tg/alpha", "10.0.1.42") + require.NoError(t, err) + require.Len(t, nlbp.deregisterCalls, 1) + + in := nlbp.deregisterCalls[0] + assert.Equal(t, "arn:tg/alpha", aws.StringValue(in.TargetGroupArn)) + require.Len(t, in.Targets, 1) + assert.Equal(t, "10.0.1.42", aws.StringValue(in.Targets[0].Id)) + assert.Equal(t, k3sAPIServerPort, aws.Int64Value(in.Targets[0].Port)) +} + +func TestDeleteClusterNLB_DeletesBoth(t *testing.T) { + nlbp := newFakeNLBProvisioner() + out, err := EnsureClusterNLB(nlbp, "111122223333", "alpha", []string{"subnet-aaa"}) + require.NoError(t, err) + + require.NoError(t, DeleteClusterNLB(nlbp, "111122223333", "alpha")) + require.Len(t, nlbp.deleteLBCalls, 1) + assert.Equal(t, out.LoadBalancerArn, aws.StringValue(nlbp.deleteLBCalls[0].LoadBalancerArn)) + require.Len(t, nlbp.deleteTGCalls, 1) + assert.Equal(t, out.TargetGroupArn, aws.StringValue(nlbp.deleteTGCalls[0].TargetGroupArn)) +} + +func TestDeleteClusterNLB_MissingResourcesNoOp(t *testing.T) { + nlbp := newFakeNLBProvisioner() + + require.NoError(t, DeleteClusterNLB(nlbp, "111122223333", "alpha")) + assert.Empty(t, nlbp.deleteLBCalls) + assert.Empty(t, nlbp.deleteTGCalls) +} + +func TestDeleteClusterNLB_FirstErrorSurfacedSweepContinues(t *testing.T) { + nlbp := newFakeNLBProvisioner() + _, err := EnsureClusterNLB(nlbp, "111122223333", "alpha", []string{"subnet-aaa"}) + require.NoError(t, err) + nlbp.deleteLBErr = errors.New("LoadBalancerInUse") + + err = DeleteClusterNLB(nlbp, "111122223333", "alpha") + require.Error(t, err) + assert.Contains(t, err.Error(), "delete NLB eks-alpha") + assert.Len(t, nlbp.deleteLBCalls, 1) + assert.Len(t, nlbp.deleteTGCalls, 1, "TG delete should still be attempted after LB delete fails") +} + +func assertELBv2TaggedAsEKS(t *testing.T, tgs []*elbv2.Tag, clusterName string) { + t.Helper() + got := map[string]string{} + for _, tg := range tgs { + if tg == nil || tg.Key == nil || tg.Value == nil { + continue + } + got[*tg.Key] = *tg.Value + } + assert.Equal(t, tags.ManagedByEKS, got[tags.ManagedByKey]) + assert.Equal(t, clusterName, got[clusterEKSClusterTagKey]) +} diff --git a/spinifex/handlers/eks/oidc_keypair.go b/spinifex/handlers/eks/oidc_keypair.go new file mode 100644 index 000000000..3d2c8c3d2 --- /dev/null +++ b/spinifex/handlers/eks/oidc_keypair.go @@ -0,0 +1,210 @@ +package handlers_eks + +import ( + "crypto" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" + "github.com/nats-io/nats.go" +) + +// PublicKeyPEMFromPrivate parses a PKCS#8 private-key PEM and returns the PKIX +// public-key PEM. kube-apiserver's --service-account-key-file requires a PUBLIC +// key (ParsePublicKeysPEM rejects private-key blocks), while the matching +// --service-account-signing-key-file takes the private key; the K3s server VM +// needs both, seeded to separate files. +func PublicKeyPEMFromPrivate(privPEM string) (string, error) { + block, _ := pem.Decode([]byte(privPEM)) + if block == nil { + return "", errors.New("eks: PublicKeyPEMFromPrivate: no PEM block in private key") + } + key, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return "", fmt.Errorf("parse pkcs8 private key: %w", err) + } + signer, ok := key.(crypto.Signer) + if !ok { + return "", errors.New("eks: PublicKeyPEMFromPrivate: private key is not a crypto.Signer") + } + pubDER, err := x509.MarshalPKIXPublicKey(signer.Public()) + if err != nil { + return "", fmt.Errorf("marshal pkix public key: %w", err) + } + return string(pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: pubDER})), nil +} + +// oidcJWK and oidcJWKS are the RFC 7517 wire shapes the per-cluster JWKS +// document uses. handlers_sts decodes the same JSON via its own JWK / JWKS +// types — wire compatibility (not Go type identity) is the contract. +type oidcJWK struct { + Kty string `json:"kty"` + Kid string `json:"kid"` + Use string `json:"use,omitempty"` + Alg string `json:"alg,omitempty"` + Crv string `json:"crv,omitempty"` + X string `json:"x,omitempty"` + Y string `json:"y,omitempty"` +} + +type oidcJWKS struct { + Keys []oidcJWK `json:"keys"` +} + +// P-256 coordinate width per SEC1 — JWK requires fixed-length base64url, not +// the variable-length big.Int byte representation. +const p256CoordLen = 32 + +// GenerateClusterOIDCKeypair creates a fresh ECDSA-P256 keypair for the +// cluster, PEM-marshals the private key as PKCS8, envelope-encrypts the PEM +// with masterKey, writes the ciphertext to OIDCSigningKeyKey and the JWKS +// document to OIDCJWKSKey. Returns the plaintext private-key PEM and the JWKS +// bytes so the caller (CreateCluster) can pass them straight to user-data +// without a follow-up KV read + decrypt. +func GenerateClusterOIDCKeypair(kv nats.KeyValue, clusterName string, masterKey []byte) (privPEM string, jwksBytes []byte, err error) { + if clusterName == "" { + return "", nil, errors.New("eks: GenerateClusterOIDCKeypair empty cluster name") + } + if len(masterKey) == 0 { + return "", nil, errors.New("eks: GenerateClusterOIDCKeypair empty master key") + } + + priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return "", nil, fmt.Errorf("generate ECDSA-P256 key: %w", err) + } + + pkcs8, err := x509.MarshalPKCS8PrivateKey(priv) + if err != nil { + return "", nil, fmt.Errorf("marshal pkcs8 private key: %w", err) + } + pemBlock := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pkcs8}) + + ciphertext, err := handlers_iam.EncryptSecret(string(pemBlock), masterKey) + if err != nil { + return "", nil, fmt.Errorf("encrypt OIDC private key: %w", err) + } + if _, err := kv.Put(OIDCSigningKeyKey(clusterName), []byte(ciphertext)); err != nil { + return "", nil, fmt.Errorf("kv put %s: %w", OIDCSigningKeyKey(clusterName), err) + } + + jwksBytes, err = marshalJWKS(&priv.PublicKey) + if err != nil { + return "", nil, err + } + if _, err := kv.Put(OIDCJWKSKey(clusterName), jwksBytes); err != nil { + return "", nil, fmt.Errorf("kv put %s: %w", OIDCJWKSKey(clusterName), err) + } + return string(pemBlock), jwksBytes, nil +} + +// LoadClusterOIDCPrivateKey reads the encrypted PEM from KV, decrypts with +// masterKey, parses it to *ecdsa.PrivateKey. Returns ErrClusterNotFound when +// the key is absent so DeleteCluster idempotency works. +func LoadClusterOIDCPrivateKey(kv nats.KeyValue, clusterName string, masterKey []byte) (*ecdsa.PrivateKey, error) { + if clusterName == "" { + return nil, errors.New("eks: LoadClusterOIDCPrivateKey empty cluster name") + } + if len(masterKey) == 0 { + return nil, errors.New("eks: LoadClusterOIDCPrivateKey empty master key") + } + entry, err := kv.Get(OIDCSigningKeyKey(clusterName)) + if err != nil { + if errors.Is(err, nats.ErrKeyNotFound) { + return nil, ErrClusterNotFound + } + return nil, fmt.Errorf("kv get %s: %w", OIDCSigningKeyKey(clusterName), err) + } + pemStr, err := handlers_iam.DecryptSecret(string(entry.Value()), masterKey) + if err != nil { + return nil, fmt.Errorf("decrypt OIDC private key: %w", err) + } + block, _ := pem.Decode([]byte(pemStr)) + if block == nil { + return nil, errors.New("eks: PEM decode produced no block") + } + keyAny, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return nil, fmt.Errorf("parse pkcs8 private key: %w", err) + } + priv, ok := keyAny.(*ecdsa.PrivateKey) + if !ok { + return nil, fmt.Errorf("eks: unexpected key type %T (want *ecdsa.PrivateKey)", keyAny) + } + if priv.Curve != elliptic.P256() { + return nil, fmt.Errorf("eks: unexpected curve %s (want P-256)", priv.Curve.Params().Name) + } + return priv, nil +} + +// ZeroizeClusterOIDCKey overwrites the encrypted private-key blob with zeros +// of the same length, then deletes the key. DeleteCluster calls this first +// so a crash after this point leaves no recoverable key material on disk. +// Absent key is a no-op — supports idempotent DeleteCluster retries. +func ZeroizeClusterOIDCKey(kv nats.KeyValue, clusterName string) error { + if clusterName == "" { + return errors.New("eks: ZeroizeClusterOIDCKey empty cluster name") + } + key := OIDCSigningKeyKey(clusterName) + entry, err := kv.Get(key) + if err != nil { + if errors.Is(err, nats.ErrKeyNotFound) { + return nil + } + return fmt.Errorf("kv get %s: %w", key, err) + } + zero := make([]byte, len(entry.Value())) + if _, err := kv.Put(key, zero); err != nil { + return fmt.Errorf("kv put zero %s: %w", key, err) + } + if err := kv.Delete(key); err != nil { + return fmt.Errorf("kv delete %s: %w", key, err) + } + return nil +} + +// marshalJWKS builds the single-key JWKS document for an ECDSA P-256 +// public key. kid is the base64url SHA-256 of the SPKI DER. +func marshalJWKS(pub *ecdsa.PublicKey) ([]byte, error) { + pubDER, err := x509.MarshalPKIXPublicKey(pub) + if err != nil { + return nil, fmt.Errorf("marshal public key DER: %w", err) + } + kidHash := sha256.Sum256(pubDER) + kid := base64.RawURLEncoding.EncodeToString(kidHash[:]) + + xb := padCoord(pub.X.Bytes()) + yb := padCoord(pub.Y.Bytes()) + + doc := oidcJWKS{Keys: []oidcJWK{{ + Kty: "EC", + Kid: kid, + Use: "sig", + Alg: "ES256", + Crv: "P-256", + X: base64.RawURLEncoding.EncodeToString(xb), + Y: base64.RawURLEncoding.EncodeToString(yb), + }}} + out, err := json.Marshal(&doc) + if err != nil { + return nil, fmt.Errorf("marshal JWKS: %w", err) + } + return out, nil +} + +func padCoord(b []byte) []byte { + if len(b) >= p256CoordLen { + return b + } + out := make([]byte, p256CoordLen) + copy(out[p256CoordLen-len(b):], b) + return out +} diff --git a/spinifex/handlers/eks/oidc_keypair_test.go b/spinifex/handlers/eks/oidc_keypair_test.go new file mode 100644 index 000000000..341e612f1 --- /dev/null +++ b/spinifex/handlers/eks/oidc_keypair_test.go @@ -0,0 +1,228 @@ +package handlers_eks + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "errors" + "testing" + + "github.com/nats-io/nats.go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var testMasterKey = func() []byte { + k := make([]byte, 32) + if _, err := rand.Read(k); err != nil { + panic(err) + } + return k +}() + +func TestGenerateClusterOIDCKeypair_PersistsJWKSAndEncryptedPrivateKey(t *testing.T) { + kv := newClusterStateTestKV(t) + + _, jwksBytes, err := GenerateClusterOIDCKeypair(kv, "alpha", testMasterKey) + require.NoError(t, err) + require.NotEmpty(t, jwksBytes) + + jwksEntry, err := kv.Get(OIDCJWKSKey("alpha")) + require.NoError(t, err) + assert.Equal(t, jwksBytes, jwksEntry.Value()) + + keyEntry, err := kv.Get(OIDCSigningKeyKey("alpha")) + require.NoError(t, err) + assert.NotEmpty(t, keyEntry.Value()) + assert.NotContains(t, string(keyEntry.Value()), "BEGIN PRIVATE KEY", + "encrypted blob must not contain a PEM header") +} + +// PublicKeyPEMFromPrivate must yield a PKIX PUBLIC KEY PEM matching the private +// key's public part — kube-apiserver's --service-account-key-file requires a +// public key and crash-loops on a private-key PEM. +func TestPublicKeyPEMFromPrivate_RoundTrips(t *testing.T) { + kv := newClusterStateTestKV(t) + privPEM, _, err := GenerateClusterOIDCKeypair(kv, "alpha", testMasterKey) + require.NoError(t, err) + + pubPEM, err := PublicKeyPEMFromPrivate(privPEM) + require.NoError(t, err) + require.Contains(t, pubPEM, "BEGIN PUBLIC KEY") + assert.NotContains(t, pubPEM, "PRIVATE KEY") + + block, _ := pem.Decode([]byte(pubPEM)) + require.NotNil(t, block) + pub, err := x509.ParsePKIXPublicKey(block.Bytes) + require.NoError(t, err) + pubEC, ok := pub.(*ecdsa.PublicKey) + require.True(t, ok) + + stored, err := LoadClusterOIDCPrivateKey(kv, "alpha", testMasterKey) + require.NoError(t, err) + assert.True(t, stored.PublicKey.Equal(pubEC), "derived public key must match the private key") +} + +func TestPublicKeyPEMFromPrivate_RejectsGarbage(t *testing.T) { + _, err := PublicKeyPEMFromPrivate("not a pem") + require.Error(t, err) + + _, err = PublicKeyPEMFromPrivate("-----BEGIN PRIVATE KEY-----\nbm90LWtleQ==\n-----END PRIVATE KEY-----\n") + require.Error(t, err) +} + +// The generator returns the plaintext private-key PEM directly so CreateCluster +// avoids a second KV read + decrypt; it must match the key persisted in KV. +func TestGenerateClusterOIDCKeypair_ReturnsPrivateKeyPEMMatchingStored(t *testing.T) { + kv := newClusterStateTestKV(t) + + privPEM, _, err := GenerateClusterOIDCKeypair(kv, "alpha", testMasterKey) + require.NoError(t, err) + require.Contains(t, privPEM, "BEGIN PRIVATE KEY", "returned PEM must be plaintext") + + block, _ := pem.Decode([]byte(privPEM)) + require.NotNil(t, block) + returned, err := x509.ParsePKCS8PrivateKey(block.Bytes) + require.NoError(t, err) + returnedEC, ok := returned.(*ecdsa.PrivateKey) + require.True(t, ok) + + stored, err := LoadClusterOIDCPrivateKey(kv, "alpha", testMasterKey) + require.NoError(t, err) + assert.True(t, stored.Equal(returnedEC), "returned PEM must match the persisted key") +} + +func TestGenerateClusterOIDCKeypair_JWKSShapeIsRFC7517EC_P256(t *testing.T) { + kv := newClusterStateTestKV(t) + + _, jwksBytes, err := GenerateClusterOIDCKeypair(kv, "alpha", testMasterKey) + require.NoError(t, err) + + var doc oidcJWKS + require.NoError(t, json.Unmarshal(jwksBytes, &doc)) + require.Len(t, doc.Keys, 1) + + k := doc.Keys[0] + assert.Equal(t, "EC", k.Kty) + assert.Equal(t, "P-256", k.Crv) + assert.Equal(t, "ES256", k.Alg) + assert.Equal(t, "sig", k.Use) + assert.NotEmpty(t, k.Kid) + + xBytes, err := base64.RawURLEncoding.DecodeString(k.X) + require.NoError(t, err) + assert.Len(t, xBytes, p256CoordLen, "JWK x must be 32-byte SEC1 coordinate") + yBytes, err := base64.RawURLEncoding.DecodeString(k.Y) + require.NoError(t, err) + assert.Len(t, yBytes, p256CoordLen, "JWK y must be 32-byte SEC1 coordinate") +} + +func TestGenerateClusterOIDCKeypair_TwoCallsProduceDistinctKeys(t *testing.T) { + kv := newClusterStateTestKV(t) + + _, a, err := GenerateClusterOIDCKeypair(kv, "alpha", testMasterKey) + require.NoError(t, err) + _, b, err := GenerateClusterOIDCKeypair(kv, "beta", testMasterKey) + require.NoError(t, err) + assert.NotEqual(t, a, b, "JWKS for distinct clusters must differ") +} + +func TestGenerateClusterOIDCKeypair_EmptyArgsRejected(t *testing.T) { + kv := newClusterStateTestKV(t) + + _, _, err := GenerateClusterOIDCKeypair(kv, "", testMasterKey) + require.Error(t, err) + _, _, err = GenerateClusterOIDCKeypair(kv, "alpha", nil) + require.Error(t, err) +} + +func TestLoadClusterOIDCPrivateKey_RoundTripMatchesJWKSPublic(t *testing.T) { + kv := newClusterStateTestKV(t) + + _, jwksBytes, err := GenerateClusterOIDCKeypair(kv, "alpha", testMasterKey) + require.NoError(t, err) + + priv, err := LoadClusterOIDCPrivateKey(kv, "alpha", testMasterKey) + require.NoError(t, err) + require.NotNil(t, priv) + assert.Equal(t, elliptic.P256(), priv.Curve) + + var doc oidcJWKS + require.NoError(t, json.Unmarshal(jwksBytes, &doc)) + require.Len(t, doc.Keys, 1) + + xExpected, err := base64.RawURLEncoding.DecodeString(doc.Keys[0].X) + require.NoError(t, err) + yExpected, err := base64.RawURLEncoding.DecodeString(doc.Keys[0].Y) + require.NoError(t, err) + + assert.Equal(t, padCoord(priv.X.Bytes()), xExpected) + assert.Equal(t, padCoord(priv.Y.Bytes()), yExpected) +} + +func TestLoadClusterOIDCPrivateKey_WrongMasterKeyFails(t *testing.T) { + kv := newClusterStateTestKV(t) + _, _, err := GenerateClusterOIDCKeypair(kv, "alpha", testMasterKey) + require.NoError(t, err) + + other := make([]byte, 32) + other[0] = 0xff + _, err = LoadClusterOIDCPrivateKey(kv, "alpha", other) + require.Error(t, err) +} + +func TestLoadClusterOIDCPrivateKey_MissingReturnsErrClusterNotFound(t *testing.T) { + kv := newClusterStateTestKV(t) + _, err := LoadClusterOIDCPrivateKey(kv, "ghost", testMasterKey) + require.ErrorIs(t, err, ErrClusterNotFound) +} + +func TestZeroizeClusterOIDCKey_DeletesKeyAndLeavesSiblingsIntact(t *testing.T) { + kv := newClusterStateTestKV(t) + _, _, err := GenerateClusterOIDCKeypair(kv, "alpha", testMasterKey) + require.NoError(t, err) + + require.NoError(t, ZeroizeClusterOIDCKey(kv, "alpha")) + + _, err = kv.Get(OIDCSigningKeyKey("alpha")) + assert.True(t, errors.Is(err, nats.ErrKeyNotFound)) + + jwks, err := kv.Get(OIDCJWKSKey("alpha")) + require.NoError(t, err, "JWKS must survive zeroize (it carries no secret material)") + assert.NotEmpty(t, jwks.Value()) +} + +func TestZeroizeClusterOIDCKey_MissingIsNoop(t *testing.T) { + kv := newClusterStateTestKV(t) + require.NoError(t, ZeroizeClusterOIDCKey(kv, "ghost")) +} + +func TestZeroizeClusterOIDCKey_EmptyNameRejected(t *testing.T) { + kv := newClusterStateTestKV(t) + require.Error(t, ZeroizeClusterOIDCKey(kv, "")) +} + +func TestMarshalJWKS_DeterministicForSameKey(t *testing.T) { + priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + + a, err := marshalJWKS(&priv.PublicKey) + require.NoError(t, err) + b, err := marshalJWKS(&priv.PublicKey) + require.NoError(t, err) + assert.Equal(t, a, b) +} + +func TestPadCoord_LeftPadsShortBigIntBytes(t *testing.T) { + got := padCoord([]byte{0x01, 0x02}) + require.Len(t, got, p256CoordLen) + assert.Equal(t, byte(0x01), got[p256CoordLen-2]) + assert.Equal(t, byte(0x02), got[p256CoordLen-1]) + for _, b := range got[:p256CoordLen-2] { + assert.Zero(t, b) + } +} diff --git a/spinifex/handlers/eks/reconciler_registry.go b/spinifex/handlers/eks/reconciler_registry.go new file mode 100644 index 000000000..06075928a --- /dev/null +++ b/spinifex/handlers/eks/reconciler_registry.go @@ -0,0 +1,166 @@ +package handlers_eks + +import ( + "context" + "errors" + "log/slog" + "sync" + + "github.com/nats-io/nats.go" +) + +// ReconcilerRegistry tracks active per-cluster reconciler goroutines so the +// daemon can spawn, stop, and graceful-shutdown them in unison. One entry per +// (accountID, clusterName) key. +type ReconcilerRegistry struct { + mu sync.Mutex + holders map[string]*reconcilerHandle +} + +type reconcilerHandle struct { + cancel context.CancelFunc + release func() + done chan struct{} +} + +// NewReconcilerRegistry returns an empty registry. +func NewReconcilerRegistry() *ReconcilerRegistry { + return &ReconcilerRegistry{holders: map[string]*reconcilerHandle{}} +} + +// SpawnReconcilerFn is the per-cluster goroutine starter the registry uses. +// Tests inject a fake; production wires a real ClusterReconciler under the +// hood (acquires lease, then loops via Run). +type SpawnReconcilerFn func(ctx context.Context, accountID, clusterName string) (release func(), err error) + +// Spawn launches one reconciler goroutine for (accountID, clusterName). If +// one is already running for that pair, returns nil without spawning a +// duplicate. SpawnFn is responsible for acquiring the leader lease + invoking +// Run; the registry only owns the ctx and the release. +func (r *ReconcilerRegistry) Spawn(parent context.Context, accountID, clusterName string, spawnFn SpawnReconcilerFn) error { + if accountID == "" || clusterName == "" { + return errors.New("eks: ReconcilerRegistry.Spawn empty ids") + } + if spawnFn == nil { + return errors.New("eks: ReconcilerRegistry.Spawn nil SpawnFn") + } + key := registryKey(accountID, clusterName) + + r.mu.Lock() + if _, ok := r.holders[key]; ok { + r.mu.Unlock() + return nil + } + ctx, cancel := context.WithCancel(parent) + done := make(chan struct{}) + h := &reconcilerHandle{cancel: cancel, done: done} + r.holders[key] = h + r.mu.Unlock() + + release, err := spawnFn(ctx, accountID, clusterName) + if err != nil { + r.mu.Lock() + delete(r.holders, key) + r.mu.Unlock() + cancel() + close(done) + return err + } + + r.mu.Lock() + h.release = release + r.mu.Unlock() + + go func() { + <-ctx.Done() + r.mu.Lock() + rel := h.release + r.mu.Unlock() + if rel != nil { + rel() + } + close(done) + }() + + slog.Info("ReconcilerRegistry: spawned", "accountID", accountID, "cluster", clusterName) + return nil +} + +// Stop cancels a single reconciler. Blocks until the goroutine has exited. +// No-op if no reconciler is registered for the key. +func (r *ReconcilerRegistry) Stop(accountID, clusterName string) { + key := registryKey(accountID, clusterName) + r.mu.Lock() + h, ok := r.holders[key] + if ok { + delete(r.holders, key) + } + r.mu.Unlock() + if !ok { + return + } + h.cancel() + <-h.done + slog.Info("ReconcilerRegistry: stopped", "accountID", accountID, "cluster", clusterName) +} + +// StopAll cancels every active reconciler and blocks until all goroutines +// have exited. Called from daemon graceful-shutdown. +func (r *ReconcilerRegistry) StopAll() { + r.mu.Lock() + handles := make([]*reconcilerHandle, 0, len(r.holders)) + for k, h := range r.holders { + handles = append(handles, h) + delete(r.holders, k) + } + r.mu.Unlock() + + for _, h := range handles { + h.cancel() + } + for _, h := range handles { + <-h.done + } +} + +// Has reports whether a reconciler is currently registered for the key. +func (r *ReconcilerRegistry) Has(accountID, clusterName string) bool { + r.mu.Lock() + defer r.mu.Unlock() + _, ok := r.holders[registryKey(accountID, clusterName)] + return ok +} + +func registryKey(accountID, clusterName string) string { + return accountID + "/" + clusterName +} + +// RunClusterReconciler is the production SpawnReconcilerFn: it constructs a +// ClusterReconciler, acquires the leader lease, then drives Run in a +// goroutine. Returns (release, nil) on lease acquired, or (nil, error) on +// constructor failure. If another node holds the lease, returns (nil, nil) +// without spawning — caller can retry on the next reconcile tick. +func RunClusterReconciler( + ctx context.Context, + leaderKV, acctKV nats.KeyValue, + accountID, clusterName, holderID, healthURL string, + opts ...ReconcilerOption, +) (func(), error) { + r, err := NewClusterReconciler(leaderKV, acctKV, accountID, clusterName, holderID, healthURL, opts...) + if err != nil { + return nil, err + } + release, ok := r.AcquireLease() + if !ok { + slog.Info("RunClusterReconciler: lease held elsewhere, skipping spawn", + "accountID", accountID, "cluster", clusterName) + return nil, nil + } + go func() { + if runErr := r.Run(ctx); runErr != nil && !errors.Is(runErr, context.Canceled) { + slog.Info("RunClusterReconciler: Run exited", + "accountID", accountID, "cluster", clusterName, "err", runErr) + } + }() + return release, nil +} diff --git a/spinifex/handlers/eks/reconciler_registry_test.go b/spinifex/handlers/eks/reconciler_registry_test.go new file mode 100644 index 000000000..67e1832ce --- /dev/null +++ b/spinifex/handlers/eks/reconciler_registry_test.go @@ -0,0 +1,95 @@ +package handlers_eks + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestReconcilerRegistry_SpawnAndStop(t *testing.T) { + reg := NewReconcilerRegistry() + + var ( + ran atomic.Bool + released atomic.Bool + ) + fn := func(ctx context.Context, accountID, clusterName string) (func(), error) { + ran.Store(true) + go func() { <-ctx.Done() }() + return func() { released.Store(true) }, nil + } + + require.NoError(t, reg.Spawn(t.Context(), "111122223333", "alpha", fn)) + assert.True(t, reg.Has("111122223333", "alpha")) + assert.True(t, ran.Load()) + + reg.Stop("111122223333", "alpha") + assert.False(t, reg.Has("111122223333", "alpha")) + require.Eventually(t, released.Load, 500*time.Millisecond, 5*time.Millisecond) +} + +func TestReconcilerRegistry_SpawnIdempotent(t *testing.T) { + reg := NewReconcilerRegistry() + var spawnCalls atomic.Int32 + fn := func(ctx context.Context, _, _ string) (func(), error) { + spawnCalls.Add(1) + go func() { <-ctx.Done() }() + return func() {}, nil + } + + require.NoError(t, reg.Spawn(t.Context(), "111122223333", "alpha", fn)) + require.NoError(t, reg.Spawn(t.Context(), "111122223333", "alpha", fn)) + + assert.EqualValues(t, 1, spawnCalls.Load(), "second Spawn must not re-invoke fn") + reg.StopAll() +} + +func TestReconcilerRegistry_SpawnFnErrorRemovesEntry(t *testing.T) { + reg := NewReconcilerRegistry() + fn := func(_ context.Context, _, _ string) (func(), error) { + return nil, errors.New("acquire lease failed") + } + + err := reg.Spawn(t.Context(), "111122223333", "alpha", fn) + require.Error(t, err) + assert.False(t, reg.Has("111122223333", "alpha"), "entry must not linger after spawn failure") +} + +func TestReconcilerRegistry_StopAllCancelsEvery(t *testing.T) { + reg := NewReconcilerRegistry() + var released atomic.Int32 + fn := func(ctx context.Context, _, _ string) (func(), error) { + go func() { <-ctx.Done() }() + return func() { released.Add(1) }, nil + } + + require.NoError(t, reg.Spawn(t.Context(), "111122223333", "alpha", fn)) + require.NoError(t, reg.Spawn(t.Context(), "111122223333", "beta", fn)) + require.NoError(t, reg.Spawn(t.Context(), "444455556666", "gamma", fn)) + + reg.StopAll() + require.Eventually(t, func() bool { return released.Load() == 3 }, 500*time.Millisecond, 5*time.Millisecond) + assert.False(t, reg.Has("111122223333", "alpha")) + assert.False(t, reg.Has("111122223333", "beta")) + assert.False(t, reg.Has("444455556666", "gamma")) +} + +func TestReconcilerRegistry_StopUnknownKeyNoop(t *testing.T) { + reg := NewReconcilerRegistry() + reg.Stop("111122223333", "ghost") + assert.False(t, reg.Has("111122223333", "ghost")) +} + +func TestReconcilerRegistry_SpawnRejectsBadArgs(t *testing.T) { + reg := NewReconcilerRegistry() + fn := func(_ context.Context, _, _ string) (func(), error) { return func() {}, nil } + + require.Error(t, reg.Spawn(t.Context(), "", "alpha", fn)) + require.Error(t, reg.Spawn(t.Context(), "111122223333", "", fn)) + require.Error(t, reg.Spawn(t.Context(), "111122223333", "alpha", nil)) +} diff --git a/spinifex/handlers/eks/security_groups.go b/spinifex/handlers/eks/security_groups.go new file mode 100644 index 000000000..33cf083dc --- /dev/null +++ b/spinifex/handlers/eks/security_groups.go @@ -0,0 +1,162 @@ +package handlers_eks + +import ( + "errors" + "fmt" + "log/slog" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/spinifex/tags" +) + +// sgProvisioner is the subset of handlers_ec2_vpc.VPCService that the cluster +// SG helpers need. Narrow so tests can fake with hand-rolled recorders without +// implementing the full VPCService surface. +type sgProvisioner interface { + CreateSecurityGroup(input *ec2.CreateSecurityGroupInput, accountID string) (*ec2.CreateSecurityGroupOutput, error) + DescribeSecurityGroups(input *ec2.DescribeSecurityGroupsInput, accountID string) (*ec2.DescribeSecurityGroupsOutput, error) + DeleteSecurityGroup(input *ec2.DeleteSecurityGroupInput, accountID string) (*ec2.DeleteSecurityGroupOutput, error) +} + +// clusterEKSClusterTagKey is stamped on every cluster-scoped resource so the +// DeleteCluster sweep + the UI hide-from-customer filter can group them. +const clusterEKSClusterTagKey = "spinifex:eks-cluster" + +// clusterEKSRoleTagKey distinguishes the control-plane SG from the nodegroup +// SG when both live in the same VPC. +const clusterEKSRoleTagKey = "spinifex:eks-role" + +const ( + clusterEKSRoleControlPlane = "control-plane" + clusterEKSRoleNodegroup = "nodegroup" +) + +// ClusterControlPlaneSGName returns the deterministic SG name used for the +// cluster's control-plane SG. Stable so Stage-6 DeleteCluster can locate it +// from clusterName alone. +func ClusterControlPlaneSGName(clusterName string) string { + return fmt.Sprintf("eks-cluster-%s-control-plane-sg", clusterName) +} + +// ClusterNodegroupSGName returns the deterministic SG name used for the +// cluster's nodegroup SG. +func ClusterNodegroupSGName(clusterName string) string { + return fmt.Sprintf("eks-cluster-%s-nodegroup-sg", clusterName) +} + +// EnsureClusterSGs creates (or returns existing) control-plane + nodegroup +// security groups in the customer VPC. Both are tagged +// spinifex:managed-by=eks + spinifex:eks-cluster=. Idempotent: an SG +// whose group-name + vpc-id match the expected pair is reused without a +// second create call. The two creates are independent — a failure on the +// nodegroup SG does not unwind the control-plane SG (DeleteCluster cleans up). +func EnsureClusterSGs(sgp sgProvisioner, accountID, clusterName, vpcID string) (controlPlaneSGID, nodegroupSGID string, err error) { + if clusterName == "" { + return "", "", errors.New("eks: EnsureClusterSGs empty cluster name") + } + if vpcID == "" { + return "", "", errors.New("eks: EnsureClusterSGs empty vpc id") + } + + controlPlaneSGID, err = ensureClusterSG(sgp, accountID, vpcID, clusterName, + ClusterControlPlaneSGName(clusterName), + fmt.Sprintf("EKS control-plane SG for cluster %s", clusterName), + clusterEKSRoleControlPlane) + if err != nil { + return "", "", err + } + + nodegroupSGID, err = ensureClusterSG(sgp, accountID, vpcID, clusterName, + ClusterNodegroupSGName(clusterName), + fmt.Sprintf("EKS nodegroup SG for cluster %s", clusterName), + clusterEKSRoleNodegroup) + if err != nil { + return "", "", err + } + + return controlPlaneSGID, nodegroupSGID, nil +} + +// DeleteClusterSGs removes both cluster SGs in the customer VPC. Missing SGs +// are no-ops so DeleteCluster can retry safely. Delete errors are logged and +// the sweep continues; the first error is returned so the caller surfaces it +// without halting the broader teardown. +func DeleteClusterSGs(sgp sgProvisioner, accountID, clusterName, vpcID string) error { + if clusterName == "" { + return errors.New("eks: DeleteClusterSGs empty cluster name") + } + if vpcID == "" { + return errors.New("eks: DeleteClusterSGs empty vpc id") + } + + var firstErr error + for _, name := range []string{ClusterControlPlaneSGName(clusterName), ClusterNodegroupSGName(clusterName)} { + id, err := lookupSGByName(sgp, accountID, vpcID, name) + if err != nil { + slog.Warn("DeleteClusterSGs: SG lookup failed", "sg", name, "err", err) + if firstErr == nil { + firstErr = err + } + continue + } + if id == "" { + continue + } + if _, err := sgp.DeleteSecurityGroup(&ec2.DeleteSecurityGroupInput{GroupId: aws.String(id)}, accountID); err != nil { + slog.Warn("DeleteClusterSGs: SG delete failed", "sg", id, "name", name, "err", err) + if firstErr == nil { + firstErr = fmt.Errorf("delete SG %s: %w", id, err) + } + } + } + return firstErr +} + +func ensureClusterSG(sgp sgProvisioner, accountID, vpcID, clusterName, name, description, role string) (string, error) { + if id, err := lookupSGByName(sgp, accountID, vpcID, name); err != nil { + return "", err + } else if id != "" { + return id, nil + } + + out, err := sgp.CreateSecurityGroup(&ec2.CreateSecurityGroupInput{ + GroupName: aws.String(name), + Description: aws.String(description), + VpcId: aws.String(vpcID), + TagSpecifications: []*ec2.TagSpecification{{ + ResourceType: aws.String("security-group"), + Tags: []*ec2.Tag{ + {Key: aws.String(tags.ManagedByKey), Value: aws.String(tags.ManagedByEKS)}, + {Key: aws.String(clusterEKSClusterTagKey), Value: aws.String(clusterName)}, + {Key: aws.String(clusterEKSRoleTagKey), Value: aws.String(role)}, + }, + }}, + }, accountID) + if err != nil { + return "", fmt.Errorf("create SG %s: %w", name, err) + } + if out == nil || out.GroupId == nil || *out.GroupId == "" { + return "", fmt.Errorf("eks: CreateSecurityGroup returned empty GroupId for %s", name) + } + return *out.GroupId, nil +} + +func lookupSGByName(sgp sgProvisioner, accountID, vpcID, name string) (string, error) { + out, err := sgp.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{ + Filters: []*ec2.Filter{ + {Name: aws.String("group-name"), Values: aws.StringSlice([]string{name})}, + {Name: aws.String("vpc-id"), Values: aws.StringSlice([]string{vpcID})}, + }, + }, accountID) + if err != nil { + return "", fmt.Errorf("describe SG %s: %w", name, err) + } + for _, g := range out.SecurityGroups { + if g == nil || g.GroupId == nil { + continue + } + return *g.GroupId, nil + } + return "", nil +} diff --git a/spinifex/handlers/eks/security_groups_test.go b/spinifex/handlers/eks/security_groups_test.go new file mode 100644 index 000000000..b84a0e34a --- /dev/null +++ b/spinifex/handlers/eks/security_groups_test.go @@ -0,0 +1,203 @@ +package handlers_eks + +import ( + "errors" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/spinifex/tags" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeSGProvisioner struct { + createCalls []*ec2.CreateSecurityGroupInput + describeCalls []*ec2.DescribeSecurityGroupsInput + deleteCalls []*ec2.DeleteSecurityGroupInput + + // existing maps "name|vpcId" → groupId for DescribeSecurityGroups lookup. + existing map[string]string + + // nextCreateID is returned by the next CreateSecurityGroup call. Tests can + // pre-seed a queue via createIDs to differentiate the control-plane vs + // nodegroup SG IDs. + createIDs []string + + createErr error + describeErr error + deleteErr error +} + +var _ sgProvisioner = (*fakeSGProvisioner)(nil) + +func newFakeSGProvisioner() *fakeSGProvisioner { + return &fakeSGProvisioner{existing: map[string]string{}} +} + +func (f *fakeSGProvisioner) CreateSecurityGroup(input *ec2.CreateSecurityGroupInput, _ string) (*ec2.CreateSecurityGroupOutput, error) { + f.createCalls = append(f.createCalls, input) + if f.createErr != nil { + return nil, f.createErr + } + id := "sg-default" + if len(f.createIDs) > 0 { + id = f.createIDs[0] + f.createIDs = f.createIDs[1:] + } + if input.GroupName != nil && input.VpcId != nil { + f.existing[*input.GroupName+"|"+*input.VpcId] = id + } + return &ec2.CreateSecurityGroupOutput{GroupId: aws.String(id)}, nil +} + +func (f *fakeSGProvisioner) DescribeSecurityGroups(input *ec2.DescribeSecurityGroupsInput, _ string) (*ec2.DescribeSecurityGroupsOutput, error) { + f.describeCalls = append(f.describeCalls, input) + if f.describeErr != nil { + return nil, f.describeErr + } + var name, vpc string + for _, filt := range input.Filters { + if filt == nil || filt.Name == nil || len(filt.Values) == 0 || filt.Values[0] == nil { + continue + } + switch *filt.Name { + case "group-name": + name = *filt.Values[0] + case "vpc-id": + vpc = *filt.Values[0] + } + } + id, ok := f.existing[name+"|"+vpc] + out := &ec2.DescribeSecurityGroupsOutput{} + if ok { + out.SecurityGroups = []*ec2.SecurityGroup{{GroupId: aws.String(id), GroupName: aws.String(name), VpcId: aws.String(vpc)}} + } + return out, nil +} + +func (f *fakeSGProvisioner) DeleteSecurityGroup(input *ec2.DeleteSecurityGroupInput, _ string) (*ec2.DeleteSecurityGroupOutput, error) { + f.deleteCalls = append(f.deleteCalls, input) + if f.deleteErr != nil { + return nil, f.deleteErr + } + return &ec2.DeleteSecurityGroupOutput{}, nil +} + +func TestEnsureClusterSGs_EmptyInputsRejected(t *testing.T) { + sgp := newFakeSGProvisioner() + + _, _, err := EnsureClusterSGs(sgp, "111122223333", "", "vpc-aaa") + require.Error(t, err) + + _, _, err = EnsureClusterSGs(sgp, "111122223333", "alpha", "") + require.Error(t, err) + + assert.Empty(t, sgp.createCalls) + assert.Empty(t, sgp.describeCalls) +} + +func TestEnsureClusterSGs_FreshCreatesBoth(t *testing.T) { + sgp := newFakeSGProvisioner() + sgp.createIDs = []string{"sg-cp-001", "sg-ng-002"} + + cpID, ngID, err := EnsureClusterSGs(sgp, "111122223333", "alpha", "vpc-aaa") + require.NoError(t, err) + assert.Equal(t, "sg-cp-001", cpID) + assert.Equal(t, "sg-ng-002", ngID) + + require.Len(t, sgp.createCalls, 2) + + assertSGCreateTagged(t, sgp.createCalls[0], "eks-cluster-alpha-control-plane-sg", "vpc-aaa", "alpha", clusterEKSRoleControlPlane) + assertSGCreateTagged(t, sgp.createCalls[1], "eks-cluster-alpha-nodegroup-sg", "vpc-aaa", "alpha", clusterEKSRoleNodegroup) +} + +func TestEnsureClusterSGs_IdempotentReusesExisting(t *testing.T) { + sgp := newFakeSGProvisioner() + sgp.existing["eks-cluster-alpha-control-plane-sg|vpc-aaa"] = "sg-existing-cp" + sgp.existing["eks-cluster-alpha-nodegroup-sg|vpc-aaa"] = "sg-existing-ng" + + cpID, ngID, err := EnsureClusterSGs(sgp, "111122223333", "alpha", "vpc-aaa") + require.NoError(t, err) + assert.Equal(t, "sg-existing-cp", cpID) + assert.Equal(t, "sg-existing-ng", ngID) + assert.Empty(t, sgp.createCalls, "no create calls expected when SGs already exist") +} + +func TestEnsureClusterSGs_MixedExistenceCreatesMissing(t *testing.T) { + sgp := newFakeSGProvisioner() + sgp.existing["eks-cluster-alpha-control-plane-sg|vpc-aaa"] = "sg-existing-cp" + sgp.createIDs = []string{"sg-new-ng"} + + cpID, ngID, err := EnsureClusterSGs(sgp, "111122223333", "alpha", "vpc-aaa") + require.NoError(t, err) + assert.Equal(t, "sg-existing-cp", cpID) + assert.Equal(t, "sg-new-ng", ngID) + + require.Len(t, sgp.createCalls, 1, "only nodegroup SG should be created") + assert.Equal(t, "eks-cluster-alpha-nodegroup-sg", aws.StringValue(sgp.createCalls[0].GroupName)) +} + +func TestEnsureClusterSGs_CreateErrorSurfacedFromControlPlane(t *testing.T) { + sgp := newFakeSGProvisioner() + sgp.createErr = errors.New("vpcd unavailable") + + _, _, err := EnsureClusterSGs(sgp, "111122223333", "alpha", "vpc-aaa") + require.Error(t, err) + assert.Contains(t, err.Error(), "create SG eks-cluster-alpha-control-plane-sg") +} + +func TestDeleteClusterSGs_DeletesBothExisting(t *testing.T) { + sgp := newFakeSGProvisioner() + sgp.existing["eks-cluster-alpha-control-plane-sg|vpc-aaa"] = "sg-existing-cp" + sgp.existing["eks-cluster-alpha-nodegroup-sg|vpc-aaa"] = "sg-existing-ng" + + err := DeleteClusterSGs(sgp, "111122223333", "alpha", "vpc-aaa") + require.NoError(t, err) + + require.Len(t, sgp.deleteCalls, 2) + assert.Equal(t, "sg-existing-cp", aws.StringValue(sgp.deleteCalls[0].GroupId)) + assert.Equal(t, "sg-existing-ng", aws.StringValue(sgp.deleteCalls[1].GroupId)) +} + +func TestDeleteClusterSGs_MissingSGsNoOp(t *testing.T) { + sgp := newFakeSGProvisioner() + + err := DeleteClusterSGs(sgp, "111122223333", "alpha", "vpc-aaa") + require.NoError(t, err) + assert.Empty(t, sgp.deleteCalls, "no delete calls expected when SGs already absent") +} + +func TestDeleteClusterSGs_FirstErrorSurfacedSweepContinues(t *testing.T) { + sgp := newFakeSGProvisioner() + sgp.existing["eks-cluster-alpha-control-plane-sg|vpc-aaa"] = "sg-existing-cp" + sgp.existing["eks-cluster-alpha-nodegroup-sg|vpc-aaa"] = "sg-existing-ng" + sgp.deleteErr = errors.New("DependencyViolation") + + err := DeleteClusterSGs(sgp, "111122223333", "alpha", "vpc-aaa") + require.Error(t, err) + assert.Contains(t, err.Error(), "sg-existing-cp") + assert.Len(t, sgp.deleteCalls, 2, "both delete calls should be attempted despite first error") +} + +func assertSGCreateTagged(t *testing.T, in *ec2.CreateSecurityGroupInput, name, vpcID, clusterName, role string) { + t.Helper() + require.NotNil(t, in) + assert.Equal(t, name, aws.StringValue(in.GroupName)) + assert.Equal(t, vpcID, aws.StringValue(in.VpcId)) + require.Len(t, in.TagSpecifications, 1) + spec := in.TagSpecifications[0] + require.NotNil(t, spec) + assert.Equal(t, "security-group", aws.StringValue(spec.ResourceType)) + + got := map[string]string{} + for _, tg := range spec.Tags { + if tg == nil || tg.Key == nil || tg.Value == nil { + continue + } + got[*tg.Key] = *tg.Value + } + assert.Equal(t, tags.ManagedByEKS, got[tags.ManagedByKey]) + assert.Equal(t, clusterName, got[clusterEKSClusterTagKey]) + assert.Equal(t, role, got[clusterEKSRoleTagKey]) +} diff --git a/spinifex/handlers/eks/service_impl.go b/spinifex/handlers/eks/service_impl.go index 1ff9827e6..d6e457c45 100644 --- a/spinifex/handlers/eks/service_impl.go +++ b/spinifex/handlers/eks/service_impl.go @@ -1,35 +1,104 @@ package handlers_eks import ( + "context" "errors" "fmt" + "log/slog" + "net" + "slices" + "sort" + "strconv" + "strings" + "sync" + "time" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/eks" "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/config" + "github.com/mulgadc/spinifex/spinifex/tags" + "github.com/mulgadc/spinifex/spinifex/utils" "github.com/nats-io/nats.go" ) -// EKSServiceImpl is the daemon-side EKSService. All methods currently return -// NotImplemented; bodies land alongside the K3s control-plane work. +// SubnetVPCResolver resolves a subnet ID to its parent VPC ID. Narrow so +// EKSServiceImpl can stay free of the wider handlers/ec2/vpc surface; the +// daemon adapts VPCServiceImpl.GetSubnet onto this contract. +type SubnetVPCResolver interface { + GetSubnetVPC(accountID, subnetID string) (vpcID string, err error) +} + +// EKSServiceDeps wires every external collaborator EKSServiceImpl needs. +// The narrow per-area interfaces (sgProvisioner, nlbProvisioner, etc.) are +// already defined alongside their consumers; the daemon adapts each +// service's concrete type onto the interface. +type EKSServiceDeps struct { + Config *config.Config + NATSConn *nats.Conn + MasterKey []byte + GatewayBaseURL string + Region string + HolderID string + + // NATS auth handed to the K3s server VM so it can publish its one-shot + // bootstrap messages. Shared token + CA (PEM content) + a VM-reachable + // URL — spinifex has no per-principal nkeys hierarchy to scope against. + NATSURL string + NATSToken string + NATSCACert string + + VPCSG sgProvisioner + VPCK3s k3sVPCProvisioner + VPCSubnet SubnetVPCResolver + NLB nlbProvisioner + Instance k3sInstanceLauncher + Image k3sAMIResolver + EIP eipProvisioner +} + +// eipProvisioner is the narrow EIP surface CreateCluster needs to give the +// hidden K3s control-plane VM an egress-only public IP (the distributed-NAT +// dev/edge topology has no shared masquerade address). The daemon adapts the +// concrete EIP service onto this. +type eipProvisioner interface { + AllocateAddress(input *ec2.AllocateAddressInput, accountID string) (*ec2.AllocateAddressOutput, error) + ReleaseAddress(input *ec2.ReleaseAddressInput, accountID string) (*ec2.ReleaseAddressOutput, error) +} + +// EKSServiceImpl is the daemon-side EKSService. CreateCluster / +// DescribeCluster / ListClusters / DeleteCluster have real bodies; every +// other action is still NotImplemented pending follow-up sprints. type EKSServiceImpl struct { - config *config.Config - nc *nats.Conn + deps EKSServiceDeps store *Store leaderKV nats.KeyValue + registry *ReconcilerRegistry + + mu sync.Mutex + bgCtx context.Context + bgCancel context.CancelFunc } var _ EKSService = (*EKSServiceImpl)(nil) -// NewEKSServiceImplWithNATS wires the Store handle and creates the shared -// spinifex-eks-leader bucket so the leader-lease CAS loop has a ready bucket -// to CAS against without a chicken-and-egg bootstrap. -func NewEKSServiceImplWithNATS(cfg *config.Config, nc *nats.Conn) (*EKSServiceImpl, error) { - store, err := NewStore(nc) +const defaultK8sVersion = "1.32" + +// NewEKSServiceImpl wires the Store, leader bucket, reconciler registry, and +// per-cluster background-goroutine context. Validates that MasterKey, +// GatewayBaseURL, Region, and HolderID are set when called from the live +// daemon path; the legacy NewEKSServiceImplWithNATS shim leaves them empty +// and bodies that need them return ServerInternal. +func NewEKSServiceImpl(deps EKSServiceDeps) (*EKSServiceImpl, error) { + if deps.NATSConn == nil { + return nil, errors.New("eks: NewEKSServiceImpl nil NATSConn") + } + store, err := NewStore(deps.NATSConn) if err != nil { return nil, fmt.Errorf("failed to create EKS store: %w", err) } - js, err := nc.JetStream() + js, err := deps.NATSConn.JetStream() if err != nil { return nil, fmt.Errorf("failed to get JetStream context: %w", err) } @@ -37,28 +106,536 @@ func NewEKSServiceImplWithNATS(cfg *config.Config, nc *nats.Conn) (*EKSServiceIm if err != nil { return nil, err } + ctx, cancel := context.WithCancel(context.Background()) return &EKSServiceImpl{ - config: cfg, - nc: nc, + deps: deps, store: store, leaderKV: leaderKV, + registry: NewReconcilerRegistry(), + bgCtx: ctx, + bgCancel: cancel, }, nil } -func notImpl() error { return errors.New(awserrors.ErrorNotImplemented) } +// NewEKSServiceImplWithNATS is a back-compat shim. Bodies that need +// orchestration deps return ServerInternal when invoked through this shim; +// the daemon-handler routing test calls it to verify NATS subject wiring +// without standing up the full dependency graph. +func NewEKSServiceImplWithNATS(cfg *config.Config, nc *nats.Conn) (*EKSServiceImpl, error) { + return NewEKSServiceImpl(EKSServiceDeps{Config: cfg, NATSConn: nc}) +} + +// Shutdown stops the per-cluster reconciler goroutines and cancels every +// background bootstrap goroutine. Safe to call multiple times. +func (s *EKSServiceImpl) Shutdown() { + s.mu.Lock() + cancel := s.bgCancel + s.mu.Unlock() + if cancel != nil { + cancel() + } + s.registry.StopAll() +} + +// SpawnRegisteredReconcilers enumerates every cluster in every per-account +// bucket and registers a reconciler goroutine for each cluster in +// CREATING or ACTIVE state. Called by the daemon on boot so a node restart +// resumes lifecycle reconcile without waiting for the next CreateCluster. +func (s *EKSServiceImpl) SpawnRegisteredReconcilers() error { + if !s.depsReadyForOrchestration() { + slog.Debug("SpawnRegisteredReconcilers: deps not ready, skipping") + return nil + } + js, err := s.deps.NATSConn.JetStream() + if err != nil { + return fmt.Errorf("jetstream: %w", err) + } + buckets := js.KeyValueStoreNames() + for name := range buckets { + if !strings.HasPrefix(name, KVBucketEKSAccountPrefix) { + continue + } + accountID := strings.TrimPrefix(name, KVBucketEKSAccountPrefix) + acctKV, err := js.KeyValue(name) + if err != nil { + slog.Warn("SpawnRegisteredReconcilers: open bucket failed", "bucket", name, "err", err) + continue + } + clusters, err := listClusterNames(acctKV) + if err != nil { + slog.Warn("SpawnRegisteredReconcilers: list clusters failed", "bucket", name, "err", err) + continue + } + for _, cluster := range clusters { + meta, err := GetClusterMeta(acctKV, cluster) + if err != nil { + continue + } + if meta.Status != ClusterStatusCreating && meta.Status != ClusterStatusActive { + continue + } + // A restart mid-CREATING lost the bootstrap subscriptions; the K3s + // VM publishes its one-shot core-NATS messages once and to nobody. + // Re-subscribe to whichever artifacts are still missing so the + // cluster can still reach ACTIVE instead of hanging in CREATING. + if meta.Status == ClusterStatusCreating { + if pending := BootstrapPendingKinds(acctKV, cluster, meta); len(pending) > 0 { + slog.Info("SpawnRegisteredReconcilers: resuming bootstrap", + "cluster", cluster, "pending", pending) + s.spawnBootstrap(accountID, cluster, acctKV, pending) + } + } + s.spawnReconciler(accountID, cluster, meta) + } + } + return nil +} + +func (s *EKSServiceImpl) depsReadyForOrchestration() bool { + return len(s.missingOrchestrationDeps()) == 0 +} + +// requireOrchestrationDeps returns a client-facing ServiceUnavailable (with the +// specific unmet deps logged at ERROR) when the daemon is missing orchestration +// wiring, so the failure is diagnosable instead of a bare ServerInternal. +func (s *EKSServiceImpl) requireOrchestrationDeps(op string) error { + missing := s.missingOrchestrationDeps() + if len(missing) == 0 { + return nil + } + slog.Error("EKS orchestration deps not ready", "op", op, "missing", missing) + return errors.New(awserrors.ErrorServiceUnavailable) +} + +// missingOrchestrationDeps names the deps required for cluster orchestration +// that are unset, so a "deps not ready" rejection can log the precise cause +// instead of an opaque ServerInternal whose only hint is a one-time boot WARN. +func (s *EKSServiceImpl) missingOrchestrationDeps() []string { + var missing []string + if s.deps.VPCSG == nil { + missing = append(missing, "VPCSG") + } + if s.deps.VPCK3s == nil { + missing = append(missing, "VPCK3s") + } + if s.deps.VPCSubnet == nil { + missing = append(missing, "VPCSubnet") + } + if s.deps.NLB == nil { + missing = append(missing, "NLB") + } + if s.deps.Instance == nil { + missing = append(missing, "Instance") + } + if s.deps.Image == nil { + missing = append(missing, "Image") + } + if s.deps.EIP == nil { + missing = append(missing, "EIP") + } + if len(s.deps.MasterKey) == 0 { + missing = append(missing, "MasterKey") + } + if s.deps.GatewayBaseURL == "" { + missing = append(missing, "GatewayBaseURL") + } + if s.deps.Region == "" { + missing = append(missing, "Region") + } + if s.deps.HolderID == "" { + missing = append(missing, "HolderID") + } + return missing +} // --- Cluster --- -func (s *EKSServiceImpl) CreateCluster(_ *eks.CreateClusterInput, _ string) (*eks.CreateClusterOutput, error) { - return nil, notImpl() +func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID string) (*eks.CreateClusterOutput, error) { + if err := s.requireOrchestrationDeps("CreateCluster"); err != nil { + return nil, err + } + if err := validateCreateClusterInput(input); err != nil { + return nil, err + } + name := aws.StringValue(input.Name) + subnetIDs := aws.StringValueSlice(input.ResourcesVpcConfig.SubnetIds) + + js, err := s.deps.NATSConn.JetStream() + if err != nil { + return nil, fmt.Errorf("jetstream: %w", err) + } + acctKV, err := GetOrCreateAccountBucket(js, accountID) + if err != nil { + return nil, fmt.Errorf("get account bucket: %w", err) + } + + if existing, err := GetClusterMeta(acctKV, name); err == nil { + if existing.Status != ClusterStatusFailed { + slog.Info("CreateCluster: cluster already exists", + "name", name, "accountID", accountID, "status", existing.Status) + return nil, errors.New(awserrors.ErrorEKSResourceInUse) + } + // A prior attempt failed. Reclaim whatever it left behind and clear its + // state so this create starts clean — makes a retry (including the + // SDK's own auto-retry) idempotent instead of masking the original + // failure with a spurious "already exists" error. If teardown of the + // failed attempt's resources fails, surface that rather than recreating + // on top of orphans. + slog.Info("CreateCluster: reclaiming FAILED cluster before recreate", + "name", name, "accountID", accountID) + if perr := s.purgeClusterInfra(accountID, name, existing, acctKV); perr != nil { + return nil, fmt.Errorf("reclaim failed cluster %s: %w", name, perr) + } + } else if !errors.Is(err, ErrClusterNotFound) { + return nil, fmt.Errorf("preflight get meta: %w", err) + } + + vpcID, err := s.deps.VPCSubnet.GetSubnetVPC(accountID, subnetIDs[0]) + if err != nil { + return nil, fmt.Errorf("resolve subnet VPC: %w", err) + } + + region := s.deps.Region + arn := fmt.Sprintf("arn:aws:eks:%s:%s:cluster/%s", region, accountID, name) + + meta := &ClusterMeta{ + Name: name, + Arn: arn, + Status: ClusterStatusCreating, + Version: deref(input.Version, defaultK8sVersion), + RoleArn: aws.StringValue(input.RoleArn), + ResourcesVpcConfig: &ClusterVpcConfig{ + SubnetIds: subnetIDs, + VpcId: vpcID, + }, + CreatedAt: time.Now().UTC(), + } + if err := PutClusterMeta(acctKV, meta); err != nil { + return nil, err + } + + cpSG, ngSG, err := EnsureClusterSGs(s.deps.VPCSG, accountID, name, vpcID) + if err != nil { + s.markFailed(acctKV, name) + return nil, fmt.Errorf("ensure cluster SGs: %w", err) + } + meta.ResourcesVpcConfig.SecurityGroupIds = []string{cpSG, ngSG} + + nlb, err := EnsureClusterNLB(s.deps.NLB, accountID, name, subnetIDs) + if err != nil { + s.markFailed(acctKV, name) + return nil, fmt.Errorf("ensure cluster NLB: %w", err) + } + meta.Endpoint = "https://" + net.JoinHostPort(nlb.DNSName, strconv.FormatInt(clusterNLBListenPort, 10)) + meta.NLBArn = nlb.LoadBalancerArn + meta.NLBTargetGroupArn = nlb.TargetGroupArn + + // Persist the NLB ARNs now, before any further fallible step. Without this + // a failure between here and the final PutClusterMeta would leave the NLB + + // target group + listener + backing LB VM created but unrecorded, so + // DeleteCluster (which keys teardown off the persisted ARNs) could not + // reclaim them. + if err := PutClusterMeta(acctKV, meta); err != nil { + s.markFailed(acctKV, name) + return nil, fmt.Errorf("persist NLB arns: %w", err) + } + + oidcIssuer, err := ClusterOIDCIssuer(s.deps.GatewayBaseURL, region, accountID, name) + if err != nil { + s.markFailed(acctKV, name) + return nil, fmt.Errorf("build OIDC issuer: %w", err) + } + meta.OIDCIssuer = oidcIssuer + + privPEM, _, err := GenerateClusterOIDCKeypair(acctKV, name, s.deps.MasterKey) + if err != nil { + s.markFailed(acctKV, name) + return nil, fmt.Errorf("generate OIDC keypair: %w", err) + } + pubPEM, err := PublicKeyPEMFromPrivate(privPEM) + if err != nil { + s.markFailed(acctKV, name) + return nil, fmt.Errorf("derive OIDC public key: %w", err) + } + + k3sOut, err := LaunchK3sServerVM(s.deps.VPCK3s, s.deps.Instance, s.deps.Image, K3sServerInput{ + AccountID: accountID, + ClusterName: name, + Region: region, + SubnetID: subnetIDs[0], + ControlPlaneSGID: cpSG, + NLBDNS: nlb.DNSName, + OIDCIssuer: oidcIssuer, + OIDCPrivateKeyPEM: privPEM, + OIDCPublicKeyPEM: pubPEM, + NATSURL: s.deps.NATSURL, + NATSToken: s.deps.NATSToken, + NATSCACert: s.deps.NATSCACert, + }) + if err != nil { + s.markFailed(acctKV, name) + if errors.Is(err, ErrEKSServerAMINotFound) { + slog.Error("CreateCluster: eks-server AMI not found; cannot launch control plane", + "cluster", name, "accountID", accountID, "err", err) + return nil, errors.New(awserrors.ErrorServiceUnavailable) + } + slog.Error("CreateCluster: K3s control-plane VM launch failed", + "cluster", name, "accountID", accountID, "err", err) + return nil, fmt.Errorf("launch K3s VM: %w", err) + } + meta.ControlPlaneInstanceID = k3sOut.InstanceID + meta.ControlPlaneENIID = k3sOut.ENIID + meta.ControlPlaneENIIP = k3sOut.ENIIP + meta.ControlPlaneMgmtIP = k3sOut.MgmtIP + + // Egress: the control-plane VM must pull container images. Allocate a + // hidden pool address and wire an egress-only SNAT for its /32 (no DNAT — + // the VM stays unreachable inbound; the NLB is its only front door). + egressIP, egressAllocID, err := s.allocateClusterEgressIP(accountID, name) + if err != nil { + s.markFailed(acctKV, name) + return nil, fmt.Errorf("allocate cluster egress IP: %w", err) + } + meta.EgressEIPAllocationID = egressAllocID + meta.EgressEIPPublicIP = egressIP + utils.PublishEvent(s.deps.NATSConn, "vpc.add-system-egress", systemEgressEvent{ + VpcId: vpcID, + SubnetId: subnetIDs[0], + InstanceIp: k3sOut.ENIIP, + ExternalIp: egressIP, + }) + + // Record the VM + ENI + egress IP before registering the target so a failed + // register (or anything after) leaves them recoverable by DeleteCluster. + if err := PutClusterMeta(acctKV, meta); err != nil { + s.markFailed(acctKV, name) + return nil, fmt.Errorf("persist control-plane ids: %w", err) + } + + if err := RegisterClusterTarget(s.deps.NLB, accountID, nlb.TargetGroupArn, k3sOut.ENIIP); err != nil { + s.markFailed(acctKV, name) + return nil, fmt.Errorf("register NLB target: %w", err) + } + + if err := PutClusterMeta(acctKV, meta); err != nil { + return nil, err + } + + s.spawnBootstrap(accountID, name, acctKV, nil) + s.spawnReconciler(accountID, name, meta) + + return &eks.CreateClusterOutput{Cluster: clusterMetaToAWS(meta)}, nil } -func (s *EKSServiceImpl) DescribeCluster(_ *eks.DescribeClusterInput, _ string) (*eks.DescribeClusterOutput, error) { - return nil, notImpl() +// systemEgressEvent is the wire shape for vpc.add-system-egress / +// vpc.delete-system-egress (mirrors network/subscribers.SystemEgressEvent; +// kept local to avoid importing the network layer from a handler). +type systemEgressEvent struct { + VpcId string `json:"vpc_id"` + SubnetId string `json:"subnet_id"` + InstanceIp string `json:"instance_ip"` + ExternalIp string `json:"external_ip"` } -func (s *EKSServiceImpl) ListClusters(_ *eks.ListClustersInput, _ string) (*eks.ListClustersOutput, error) { - return nil, notImpl() +// allocateClusterEgressIP allocates a hidden pool address for the cluster's +// control-plane VM egress, tagged ManagedBy=eks so it stays out of customer +// DescribeAddresses listings. +func (s *EKSServiceImpl) allocateClusterEgressIP(accountID, name string) (publicIP, allocationID string, err error) { + out, err := s.deps.EIP.AllocateAddress(&ec2.AllocateAddressInput{ + Domain: aws.String("vpc"), + TagSpecifications: []*ec2.TagSpecification{{ + ResourceType: aws.String("elastic-ip"), + Tags: []*ec2.Tag{{Key: aws.String(tags.ManagedByKey), Value: aws.String(tags.ManagedByEKS)}}, + }}, + }, accountID) + if err != nil { + return "", "", err + } + if out == nil || aws.StringValue(out.PublicIp) == "" || aws.StringValue(out.AllocationId) == "" { + return "", "", errors.New("eks: AllocateAddress returned incomplete EIP") + } + return aws.StringValue(out.PublicIp), aws.StringValue(out.AllocationId), nil +} + +func (s *EKSServiceImpl) DescribeCluster(input *eks.DescribeClusterInput, accountID string) (*eks.DescribeClusterOutput, error) { + name := aws.StringValue(input.Name) + if name == "" { + return nil, errors.New(awserrors.ErrorInvalidParameterValue) + } + js, err := s.deps.NATSConn.JetStream() + if err != nil { + return nil, fmt.Errorf("jetstream: %w", err) + } + acctKV, err := GetOrCreateAccountBucket(js, accountID) + if err != nil { + return nil, fmt.Errorf("get account bucket: %w", err) + } + meta, err := GetClusterMeta(acctKV, name) + if err != nil { + if errors.Is(err, ErrClusterNotFound) { + return nil, errors.New(awserrors.ErrorEKSResourceNotFound) + } + return nil, err + } + return &eks.DescribeClusterOutput{Cluster: clusterMetaToAWS(meta)}, nil +} + +func (s *EKSServiceImpl) ListClusters(input *eks.ListClustersInput, accountID string) (*eks.ListClustersOutput, error) { + js, err := s.deps.NATSConn.JetStream() + if err != nil { + return nil, fmt.Errorf("jetstream: %w", err) + } + acctKV, err := GetOrCreateAccountBucket(js, accountID) + if err != nil { + return nil, fmt.Errorf("get account bucket: %w", err) + } + names, err := listClusterNames(acctKV) + if err != nil { + return nil, err + } + sort.Strings(names) + + maxResults := int(aws.Int64Value(input.MaxResults)) + if maxResults <= 0 || maxResults > 100 { + maxResults = 100 + } + startToken := aws.StringValue(input.NextToken) + + page := make([]string, 0, maxResults) + var nextToken string + startIdx := 0 + if startToken != "" { + startIdx = sort.SearchStrings(names, startToken) + } + endIdx := min(startIdx+maxResults, len(names)) + page = append(page, names[startIdx:endIdx]...) + if endIdx < len(names) { + nextToken = names[endIdx] + } + + out := &eks.ListClustersOutput{Clusters: aws.StringSlice(page)} + if nextToken != "" { + out.NextToken = aws.String(nextToken) + } + return out, nil +} + +func (s *EKSServiceImpl) DeleteCluster(input *eks.DeleteClusterInput, accountID string) (*eks.DeleteClusterOutput, error) { + if err := s.requireOrchestrationDeps("DeleteCluster"); err != nil { + return nil, err + } + name := aws.StringValue(input.Name) + if name == "" { + return nil, errors.New(awserrors.ErrorInvalidParameterValue) + } + js, err := s.deps.NATSConn.JetStream() + if err != nil { + return nil, fmt.Errorf("jetstream: %w", err) + } + acctKV, err := GetOrCreateAccountBucket(js, accountID) + if err != nil { + return nil, fmt.Errorf("get account bucket: %w", err) + } + + meta, err := GetClusterMeta(acctKV, name) + if err != nil { + if errors.Is(err, ErrClusterNotFound) { + return nil, errors.New(awserrors.ErrorEKSResourceNotFound) + } + return nil, err + } + + if err := SetClusterStatus(acctKV, name, ClusterStatusDeleting); err != nil { + return nil, fmt.Errorf("set DELETING: %w", err) + } + meta.Status = ClusterStatusDeleting + + if err := s.purgeClusterInfra(accountID, name, meta, acctKV); err != nil { + slog.Error("DeleteCluster: teardown incomplete; leaving cluster DELETING for retry", + "cluster", name, "err", err) + return nil, fmt.Errorf("eks: DeleteCluster %s: %w", name, err) + } + + return &eks.DeleteClusterOutput{Cluster: clusterMetaToAWS(meta)}, nil +} + +// purgeClusterInfra tears down a cluster's live AWS resources and erases its +// per-cluster KV state. Zeroize (a security guarantee — no recoverable key +// material) plus NLB and VM teardown are blocking: their failures are joined +// and returned BEFORE the KV sweep, so a billable resource is never orphaned +// without the meta record that owns its ARNs/IDs. SG cleanup is best-effort +// (a leaked SG strands no owning record). Shared by DeleteCluster and the +// FAILED-cluster reclaim path in CreateCluster. +func (s *EKSServiceImpl) purgeClusterInfra(accountID, name string, meta *ClusterMeta, acctKV nats.KeyValue) error { + s.registry.Stop(accountID, name) + + var teardownErrs []error + + if err := ZeroizeClusterOIDCKey(acctKV, name); err != nil { + teardownErrs = append(teardownErrs, fmt.Errorf("zeroize OIDC key: %w", err)) + } + + if meta.NLBArn != "" { + // Deregister is best-effort: DeleteClusterNLB tears down the whole NLB + // + target group, so a stale target registration cannot leak past it. + if meta.NLBTargetGroupArn != "" && meta.ControlPlaneENIIP != "" { + if err := DeregisterClusterTarget(s.deps.NLB, accountID, meta.NLBTargetGroupArn, meta.ControlPlaneENIIP); err != nil { + slog.Warn("purgeClusterInfra: deregister NLB target failed", "cluster", name, "err", err) + } + } + if err := DeleteClusterNLB(s.deps.NLB, accountID, name); err != nil { + teardownErrs = append(teardownErrs, fmt.Errorf("delete NLB: %w", err)) + } + } + + if meta.ControlPlaneInstanceID != "" || meta.ControlPlaneENIID != "" { + if err := TerminateK3sServerVM(s.deps.VPCK3s, s.deps.Instance, + accountID, meta.ControlPlaneInstanceID, meta.ControlPlaneENIID); err != nil { + teardownErrs = append(teardownErrs, fmt.Errorf("terminate K3s VM: %w", err)) + } + } + + // Egress: drop the OVN reroute+snat (fire-and-forget event, prune-safe) and + // release the hidden pool address. ReleaseAddress is blocking — the EIP is + // billable and its allocation ID lives in the meta we are about to erase, so + // a release failure must not let the KV sweep orphan it. + if meta.EgressEIPPublicIP != "" || meta.EgressEIPAllocationID != "" { + if meta.ResourcesVpcConfig != nil && meta.ResourcesVpcConfig.VpcId != "" && meta.ControlPlaneENIIP != "" { + subnetID := "" + if len(meta.ResourcesVpcConfig.SubnetIds) > 0 { + subnetID = meta.ResourcesVpcConfig.SubnetIds[0] + } + utils.PublishEvent(s.deps.NATSConn, "vpc.delete-system-egress", systemEgressEvent{ + VpcId: meta.ResourcesVpcConfig.VpcId, + SubnetId: subnetID, + InstanceIp: meta.ControlPlaneENIIP, + ExternalIp: meta.EgressEIPPublicIP, + }) + } + if meta.EgressEIPAllocationID != "" { + if _, err := s.deps.EIP.ReleaseAddress(&ec2.ReleaseAddressInput{ + AllocationId: aws.String(meta.EgressEIPAllocationID), + }, accountID); err != nil { + teardownErrs = append(teardownErrs, fmt.Errorf("release egress EIP: %w", err)) + } + } + } + + if len(teardownErrs) > 0 { + return errors.Join(teardownErrs...) + } + + if meta.ResourcesVpcConfig != nil && meta.ResourcesVpcConfig.VpcId != "" { + if err := DeleteClusterSGs(s.deps.VPCSG, accountID, name, meta.ResourcesVpcConfig.VpcId); err != nil { + slog.Warn("purgeClusterInfra: DeleteClusterSGs failed", "cluster", name, "err", err) + } + } + + // Only now, with NLB + VM confirmed gone, erase the meta + per-cluster KV. + if err := DeleteClusterPrefix(acctKV, name); err != nil { + return fmt.Errorf("delete cluster prefix: %w", err) + } + return nil } func (s *EKSServiceImpl) UpdateClusterConfig(_ *eks.UpdateClusterConfigInput, _ string) (*eks.UpdateClusterConfigOutput, error) { @@ -69,10 +646,6 @@ func (s *EKSServiceImpl) UpdateClusterVersion(_ *eks.UpdateClusterVersionInput, return nil, notImpl() } -func (s *EKSServiceImpl) DeleteCluster(_ *eks.DeleteClusterInput, _ string) (*eks.DeleteClusterOutput, error) { - return nil, notImpl() -} - // --- Nodegroup --- func (s *EKSServiceImpl) CreateNodegroup(_ *eks.CreateNodegroupInput, _ string) (*eks.CreateNodegroupOutput, error) { @@ -194,3 +767,167 @@ func (s *EKSServiceImpl) UntagResource(_ *eks.UntagResourceInput, _ string) (*ek func (s *EKSServiceImpl) ListTagsForResource(_ *eks.ListTagsForResourceInput, _ string) (*eks.ListTagsForResourceOutput, error) { return nil, notImpl() } + +// --- helpers --- + +func notImpl() error { return errors.New(awserrors.ErrorNotImplemented) } + +func validateCreateClusterInput(input *eks.CreateClusterInput) error { + if input == nil || input.Name == nil || *input.Name == "" { + return errors.New(awserrors.ErrorInvalidParameterValue) + } + if input.RoleArn == nil || !strings.HasPrefix(*input.RoleArn, "arn:aws:iam:") { + return errors.New(awserrors.ErrorInvalidParameterValue) + } + if input.ResourcesVpcConfig == nil || len(input.ResourcesVpcConfig.SubnetIds) == 0 { + return errors.New(awserrors.ErrorInvalidParameterValue) + } + for _, sn := range input.ResourcesVpcConfig.SubnetIds { + if sn == nil || *sn == "" { + return errors.New(awserrors.ErrorInvalidParameterValue) + } + } + if input.AccessConfig != nil && input.AccessConfig.AuthenticationMode != nil { + mode := *input.AccessConfig.AuthenticationMode + if mode != "" && mode != eks.AuthenticationModeApi { + return errors.New(awserrors.ErrorInvalidParameter) + } + } + return nil +} + +func (s *EKSServiceImpl) markFailed(kv nats.KeyValue, name string) { + if err := SetClusterStatus(kv, name, ClusterStatusFailed); err != nil { + slog.Warn("CreateCluster: SetClusterStatus(FAILED) failed", "cluster", name, "err", err) + } +} + +// spawnBootstrap launches the one-shot NATS bootstrap subscriber for a cluster. +// kinds nil means a fresh CreateCluster (wait on all four subjects); a non-nil +// subset is a daemon-restart resume that waits only on the missing artifacts. +func (s *EKSServiceImpl) spawnBootstrap(accountID, clusterName string, kv nats.KeyValue, kinds []string) { + boot, err := NewNATSBootstrap(s.deps.NATSConn, kv, s.deps.MasterKey, accountID, clusterName) + if err != nil { + slog.Error("spawnBootstrap: NewNATSBootstrap failed", "cluster", clusterName, "err", err) + return + } + go func() { + err := boot.RunForKinds(s.bgCtx, kinds) + if err == nil || errors.Is(err, context.Canceled) { + return + } + // Bootstrap died without collecting all requested artifacts. Without + // this the cluster sits in CREATING forever — fail it so + // DescribeCluster surfaces the cause and DeleteCluster can reclaim the + // resources. + slog.Warn("NATSBootstrap exited; marking cluster FAILED", "cluster", clusterName, "err", err) + if markErr := MarkClusterFailed(kv, clusterName, "bootstrap failed: "+err.Error()); markErr != nil && + !errors.Is(markErr, ErrClusterNotFound) { + slog.Error("spawnBootstrap: MarkClusterFailed", "cluster", clusterName, "err", markErr) + } + }() +} + +func (s *EKSServiceImpl) spawnReconciler(accountID, clusterName string, meta *ClusterMeta) { + healthURL := "" + switch { + case meta.ControlPlaneMgmtIP != "": + // Workaround until authoritative DNS (Eclipso/Route53) lands: the + // reconciler runs on the host, outside the VPC overlay, and can neither + // resolve the NLB DNS name nor route to the VPC-internal NLB front-end. + // The control-plane VM's br-mgmt NIC is the one apiserver address + // reachable from the daemon, so probe :6443/healthz there directly. The + // probe client sets InsecureSkipVerify, so the mgmt IP need not be in the + // apiserver cert SAN. + healthURL = "https://" + net.JoinHostPort(meta.ControlPlaneMgmtIP, "6443") + "/healthz" + case meta.Endpoint != "": + healthURL = strings.TrimRight(meta.Endpoint, "/") + "/healthz" + } + js, err := s.deps.NATSConn.JetStream() + if err != nil { + slog.Error("spawnReconciler: jetstream", "err", err) + return + } + acctKV, err := GetOrCreateAccountBucket(js, accountID) + if err != nil { + slog.Error("spawnReconciler: account bucket", "err", err) + return + } + spawn := func(ctx context.Context, _, _ string) (func(), error) { + return RunClusterReconciler(ctx, s.leaderKV, acctKV, accountID, clusterName, s.deps.HolderID, healthURL) + } + if err := s.registry.Spawn(s.bgCtx, accountID, clusterName, spawn); err != nil { + slog.Error("spawnReconciler: registry spawn", "cluster", clusterName, "err", err) + } +} + +func clusterMetaToAWS(meta *ClusterMeta) *eks.Cluster { + if meta == nil { + return nil + } + out := &eks.Cluster{ + Name: aws.String(meta.Name), + Arn: aws.String(meta.Arn), + Status: aws.String(string(meta.Status)), + Version: aws.String(meta.Version), + RoleArn: aws.String(meta.RoleArn), + CreatedAt: aws.Time(meta.CreatedAt), + } + if meta.Endpoint != "" { + out.Endpoint = aws.String(meta.Endpoint) + } + if meta.OIDCIssuer != "" { + out.Identity = &eks.Identity{Oidc: &eks.OIDC{Issuer: aws.String(meta.OIDCIssuer)}} + } + if meta.CertificateAuthorityB64 != "" { + out.CertificateAuthority = &eks.Certificate{Data: aws.String(meta.CertificateAuthorityB64)} + } + if meta.ResourcesVpcConfig != nil { + out.ResourcesVpcConfig = &eks.VpcConfigResponse{ + SubnetIds: aws.StringSlice(meta.ResourcesVpcConfig.SubnetIds), + SecurityGroupIds: aws.StringSlice(meta.ResourcesVpcConfig.SecurityGroupIds), + VpcId: aws.String(meta.ResourcesVpcConfig.VpcId), + } + } + if meta.HealthIssue != "" { + out.Health = &eks.ClusterHealth{ + Issues: []*eks.ClusterIssue{{ + Code: aws.String(eks.ClusterIssueCodeClusterUnreachable), + Message: aws.String(meta.HealthIssue), + }}, + } + } + return out +} + +func listClusterNames(kv nats.KeyValue) ([]string, error) { + keys, err := kv.Keys() + if err != nil { + if errors.Is(err, nats.ErrNoKeysFound) { + return nil, nil + } + return nil, fmt.Errorf("kv keys: %w", err) + } + suffix := "/meta" + names := make([]string, 0, len(keys)) + for _, k := range keys { + if !strings.HasPrefix(k, "clusters/") || !strings.HasSuffix(k, suffix) { + continue + } + name := strings.TrimSuffix(strings.TrimPrefix(k, "clusters/"), suffix) + if name == "" || strings.Contains(name, "/") { + continue + } + if !slices.Contains(names, name) { + names = append(names, name) + } + } + return names, nil +} + +func deref(p *string, fallback string) string { + if p == nil || *p == "" { + return fallback + } + return *p +} diff --git a/spinifex/handlers/eks/service_impl_test.go b/spinifex/handlers/eks/service_impl_test.go index fce46a0a8..5b293d557 100644 --- a/spinifex/handlers/eks/service_impl_test.go +++ b/spinifex/handlers/eks/service_impl_test.go @@ -1,12 +1,15 @@ package handlers_eks import ( + "errors" "testing" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/eks" "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/nats-io/nats.go" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -18,17 +21,27 @@ func setupTestService(t *testing.T) *EKSServiceImpl { return svc } -func TestEKSServiceImpl_ClusterMethodsReturnNotImplemented(t *testing.T) { +// TestEKSServiceImpl_ClusterLifecycleShimMode covers the four lifecycle +// methods when the service is constructed via NewEKSServiceImplWithNATS +// (the shim path the daemon-handler routing test uses): orchestration deps +// are absent so CreateCluster/DeleteCluster short-circuit to ServiceUnavailable +// (the missing deps are logged at ERROR), DescribeCluster hits an empty +// per-account bucket and surfaces ResourceNotFoundException, and ListClusters +// returns an empty list. The UpdateClusterConfig + UpdateClusterVersion paths +// stay NotImplemented. +func TestEKSServiceImpl_ClusterLifecycleShimMode(t *testing.T) { svc := setupTestService(t) _, err := svc.CreateCluster(&eks.CreateClusterInput{Name: aws.String("c1")}, testAccountID) - require.EqualError(t, err, awserrors.ErrorNotImplemented) + require.EqualError(t, err, awserrors.ErrorServiceUnavailable) _, err = svc.DescribeCluster(&eks.DescribeClusterInput{Name: aws.String("c1")}, testAccountID) - require.EqualError(t, err, awserrors.ErrorNotImplemented) + require.EqualError(t, err, awserrors.ErrorEKSResourceNotFound) - _, err = svc.ListClusters(&eks.ListClustersInput{}, testAccountID) - require.EqualError(t, err, awserrors.ErrorNotImplemented) + out, err := svc.ListClusters(&eks.ListClustersInput{}, testAccountID) + require.NoError(t, err) + require.NotNil(t, out) + require.Empty(t, out.Clusters) _, err = svc.UpdateClusterConfig(&eks.UpdateClusterConfigInput{Name: aws.String("c1")}, testAccountID) require.EqualError(t, err, awserrors.ErrorNotImplemented) @@ -37,7 +50,65 @@ func TestEKSServiceImpl_ClusterMethodsReturnNotImplemented(t *testing.T) { require.EqualError(t, err, awserrors.ErrorNotImplemented) _, err = svc.DeleteCluster(&eks.DeleteClusterInput{Name: aws.String("c1")}, testAccountID) - require.EqualError(t, err, awserrors.ErrorNotImplemented) + require.EqualError(t, err, awserrors.ErrorServiceUnavailable) +} + +// EKS only supports the API authentication mode here; CONFIG_MAP (and the +// API_AND_CONFIG_MAP hybrid) must be rejected with InvalidParameterException. +func TestValidateCreateClusterInput_RejectsConfigMapAuthMode(t *testing.T) { + in := createInput("alpha") + in.AccessConfig = &eks.CreateAccessConfigRequest{ + AuthenticationMode: aws.String(eks.AuthenticationModeConfigMap), + } + err := validateCreateClusterInput(in) + require.EqualError(t, err, awserrors.ErrorInvalidParameter) +} + +func TestValidateCreateClusterInput_AcceptsAPIAuthMode(t *testing.T) { + in := createInput("alpha") + in.AccessConfig = &eks.CreateAccessConfigRequest{ + AuthenticationMode: aws.String(eks.AuthenticationModeApi), + } + require.NoError(t, validateCreateClusterInput(in)) +} + +// A create request with no subnets is malformed and must be rejected before any +// orchestration work (InvalidParameterValue → 400). +func TestValidateCreateClusterInput_RejectsMissingSubnetIds(t *testing.T) { + in := createInput("alpha") + in.ResourcesVpcConfig = &eks.VpcConfigRequest{} + require.EqualError(t, validateCreateClusterInput(in), awserrors.ErrorInvalidParameterValue) + + in.ResourcesVpcConfig = nil + require.EqualError(t, validateCreateClusterInput(in), awserrors.ErrorInvalidParameterValue) +} + +// DescribeCluster on an absent cluster must reach the KV lookup (full deps +// wired, unlike the shim short-circuit) and surface ResourceNotFoundException. +func TestDescribeCluster_NotFoundWithFullDeps(t *testing.T) { + f := newEKSServiceFixture(t) + + _, err := f.svc.DescribeCluster(&eks.DescribeClusterInput{Name: aws.String("ghost")}, testAccountID) + require.EqualError(t, err, awserrors.ErrorEKSResourceNotFound) +} + +// Security guarantee: the OIDC signing key must be zeroized BEFORE infra +// teardown, so a teardown failure (which leaves the meta for retry) can never +// leave recoverable key material behind. Force VM terminate to fail and assert +// the key is already gone while the meta survives. +func TestDeleteCluster_ZeroizesOIDCKeyBeforeTeardown(t *testing.T) { + f := newDeleteClusterFixture(t, "alpha") + f.inst.terminateErr = errors.New("hypervisor unreachable") + + _, err := f.svc.DeleteCluster(deleteInput("alpha"), testAccountID) + require.Error(t, err, "teardown failure must surface") + + _, getErr := f.kv.Get(OIDCSigningKeyKey("alpha")) + assert.ErrorIs(t, getErr, nats.ErrKeyNotFound, "OIDC key must be zeroized before teardown, even when teardown fails") + + meta, metaErr := GetClusterMeta(f.kv, "alpha") + require.NoError(t, metaErr, "meta must survive a failed teardown for retry") + assert.Equal(t, ClusterStatusDeleting, meta.Status) } func TestEKSServiceImpl_NodegroupMethodsReturnNotImplemented(t *testing.T) { diff --git a/spinifex/handlers/eks/store.go b/spinifex/handlers/eks/store.go index b67aad680..2e4048e78 100644 --- a/spinifex/handlers/eks/store.go +++ b/spinifex/handlers/eks/store.go @@ -74,6 +74,15 @@ func OIDCJWKSKey(cluster string) string { return fmt.Sprintf("clusters/%s/oidc-jwks.json", cluster) } +// OIDCJWKSVerifiedKey returns the KV key marking that the K3s-VM-published JWKS +// passed the controller cross-check (kid + kty match the controller-generated +// keypair). The reconciler gates the ACTIVE transition on this marker, NOT on +// OIDCJWKSKey — the controller pre-seeds OIDCJWKSKey at create time, so its +// presence proves nothing about the running cluster's actual signing key. +func OIDCJWKSVerifiedKey(cluster string) string { + return fmt.Sprintf("clusters/%s/oidc-jwks-verified", cluster) +} + // AdminKubeconfigKey returns the KV key for a cluster's encrypted admin // kubeconfig (used by the spinifex-side nodegroup reconciler). func AdminKubeconfigKey(cluster string) string { diff --git a/spinifex/handlers/elbv2/launcher.go b/spinifex/handlers/elbv2/launcher.go index 3ca9b6e8b..26d65c184 100644 --- a/spinifex/handlers/elbv2/launcher.go +++ b/spinifex/handlers/elbv2/launcher.go @@ -1,112 +1,17 @@ package handlers_elbv2 -// SystemInstanceLauncher is the subset of daemon functionality needed by -// ELBv2 to launch and terminate system-managed VMs (e.g. ALB VMs). -// Defined here to avoid a circular import between handlers/elbv2 and daemon. -// -// System instances are owned by the system account and hidden from -// customer DescribeInstances calls. -type SystemInstanceLauncher interface { - // LaunchSystemInstance creates and starts a system-managed VM. - // The VM is assigned to the system account and not visible to customers. - // The caller provides the instance type, AMI, subnet, user data, and - // optionally a pre-created ENI to attach. - // - // Returns the instance ID and private IP once the VM is running. - LaunchSystemInstance(input *SystemInstanceInput) (*SystemInstanceOutput, error) +import "github.com/mulgadc/spinifex/spinifex/handlers/sysinstance" - // TerminateSystemInstance stops and cleans up a system-managed VM. - TerminateSystemInstance(instanceID string) error -} - -// SystemInstanceInput describes the direct-boot microVM to launch for an -// ELBv2 load balancer. There is no AMI or cloud-init path — kernel and -// initrd come from the bundled spinifex package and per-VM config is -// delivered via QEMU fw_cfg blobs. -// -// JSON tags are explicit because this struct crosses the -// system.LaunchInstance.* NATS subject root when ELBv2 hands a launch off -// to the cluster — see daemon/daemon_system_dispatch.go. -type SystemInstanceInput struct { - InstanceType string `json:"instance_type"` // e.g. "sys.micro" - SubnetID string `json:"subnet_id"` // VPC subnet for networking - - // ENI fields — the VM always uses a pre-created ENI (the ALB's primary ENI). - ENIID string `json:"eni_id"` - ENIMac string `json:"eni_mac"` - ENIIP string `json:"eni_ip"` - - // ExtraENIs lists additional pre-created ENIs that should be attached to - // the VM alongside the primary ENI. Used for multi-subnet ALBs so the VM - // has a NIC (and tap on br-int) in every subnet the LB spans. - ExtraENIs []ExtraENIInput `json:"extra_enis,omitempty"` - - // Scheme is the ALB scheme ("internet-facing" or "internal"). - // Internet-facing ALBs get a public IP and NAT rules; internal ALBs do not. - Scheme string `json:"scheme"` - - // AccountID is the owner account of the ALB's ENI. Required so the daemon - // can look up and update the ENI record (which is keyed by account). - AccountID string `json:"account_id"` - - // HostfwdPorts specifies additional guest ports to forward from the host - // via the QEMU user-mode dev NIC (dev_networking mode only). - // Each entry is a guest port (e.g. 80, 443). The host port is auto-assigned. - HostfwdPorts []int `json:"hostfwd_ports,omitempty"` - - // NICs defines the network interfaces for the microVM. - // Index 0 is the primary VPC ENI, index 1 is the management NIC, index 2+ are extra ENIs. - NICs []NICConfig `json:"nics,omitempty"` - - // LBAgentEnv is a KEY=value blob written to /etc/conf.d/lb-agent inside - // the guest via fw_cfg. - LBAgentEnv string `json:"lb_agent_env"` - - // CACert holds PEM-encoded CA certificate bytes delivered to the guest via fw_cfg. - CACert string `json:"ca_cert"` -} - -// ExtraENIInput describes an additional pre-created ENI to attach to a -// system instance. The primary ENI is still passed via ENIID/ENIMac/ENIIP. -type ExtraENIInput struct { - ENIID string `json:"eni_id"` - ENIMac string `json:"eni_mac"` - ENIIP string `json:"eni_ip"` - SubnetID string `json:"subnet_id"` -} - -// NICConfig describes a single network interface for a microVM direct-boot launch. -// Index 0 is the primary VPC ENI, index 1 is the management NIC, index 2+ are extra ENIs. -type NICConfig struct { - MAC string `json:"mac"` // e.g. "02:0a:01:23:45:67" - CIDR string `json:"cidr"` // e.g. "10.0.1.5/24" - Gateway string `json:"gateway,omitempty"` // e.g. "10.0.1.1"; empty for mgmt NIC - IsDefault bool `json:"is_default"` // true for exactly one NIC (primary VPC ENI) - RouteDst string `json:"route_dst,omitempty"` // specific host route dst, e.g. "10.20.0.5/32" - RouteVia string `json:"route_via,omitempty"` // next-hop for RouteDst -} - -// RecoveryContext carries the per-VM fields the ELBv2 service needs to -// reconstruct a SystemInstanceInput during host-reboot recovery, without -// importing the vm package (which would cycle back through daemon). -// MgmtMAC / MgmtIP are the values the daemon already allocated for this -// instance on a prior launch; the service injects them back into NIC[1] -// rather than re-allocating from the mgmt IPAM. -type RecoveryContext struct { - InstanceID string - InstanceType string - ENIMac string - MgmtMAC string - MgmtIP string -} - -// SystemInstanceOutput contains the result of a successful launch. -type SystemInstanceOutput struct { - InstanceID string `json:"instance_id"` // e.g. "i-xxxxx" - PrivateIP string `json:"private_ip"` // VPC private IP - PublicIP string `json:"public_ip,omitempty"` // Public IP (only for internet-facing scheme) - - // HostfwdMap maps guest port → host port for any forwarded ports. - // Only populated when dev_networking is enabled and HostfwdPorts were requested. - HostfwdMap map[int]int `json:"hostfwd_map,omitempty"` -} +// The system-instance launch contract now lives in the boot-agnostic +// handlers/sysinstance package so both ELBv2 and EKS can launch system VMs +// without an eks→elbv2 dependency. These aliases keep existing ELBv2 source +// (which builds direct-boot microVM inputs) compiling unchanged. ELBv2 always +// launches with BootMode=BootDirect (the zero value). +type ( + SystemInstanceLauncher = sysinstance.SystemInstanceLauncher + SystemInstanceInput = sysinstance.SystemInstanceInput + ExtraENIInput = sysinstance.ExtraENIInput + NICConfig = sysinstance.NICConfig + RecoveryContext = sysinstance.RecoveryContext + SystemInstanceOutput = sysinstance.SystemInstanceOutput +) diff --git a/spinifex/handlers/sts/assume_role_with_web_identity_test.go b/spinifex/handlers/sts/assume_role_with_web_identity_test.go index 367aec23e..5669d1257 100644 --- a/spinifex/handlers/sts/assume_role_with_web_identity_test.go +++ b/spinifex/handlers/sts/assume_role_with_web_identity_test.go @@ -56,8 +56,8 @@ func newWebIdentityFixture(t *testing.T, svc *STSServiceImpl, accountID string) Y: base64.RawURLEncoding.EncodeToString(priv.Y.Bytes()), }}} - issuer := fmt.Sprintf("https://oidc.eks.%s.%s/%s/%s", - testWebRegion, testWebSuffix, accountID, testWebClusterName) + issuer := fmt.Sprintf("https://gw.%s/oidc/eks/%s/%s/%s", + testWebSuffix, testWebRegion, accountID, testWebClusterName) issuerHostPath := strings.TrimPrefix(issuer, "https://") federatedARN := handlers_iam.OIDCProviderARN(accountID, issuerHostPath) @@ -346,8 +346,8 @@ func TestAssumeRoleWithWebIdentity_ProviderNotRegistered(t *testing.T) { priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) require.NoError(t, err) const kid = "unregistered" - issuer := fmt.Sprintf("https://oidc.eks.%s.%s/%s/%s", - testWebRegion, testWebSuffix, testCallerAccountID, "unregistered-cluster") + issuer := fmt.Sprintf("https://gw.%s/oidc/eks/%s/%s/%s", + testWebSuffix, testWebRegion, testCallerAccountID, "unregistered-cluster") issuerHostPath := strings.TrimPrefix(issuer, "https://") federatedARN := handlers_iam.OIDCProviderARN(testCallerAccountID, issuerHostPath) diff --git a/spinifex/handlers/sts/oidc_jwks.go b/spinifex/handlers/sts/oidc_jwks.go index 32bddb6a5..175b745cd 100644 --- a/spinifex/handlers/sts/oidc_jwks.go +++ b/spinifex/handlers/sts/oidc_jwks.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/url" + "slices" "strings" handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks" @@ -47,15 +48,34 @@ func (s *JWKS) FindByKID(kid string) *JWK { return nil } -// ParseEKSIssuerURL extracts the (accountID, clusterName) pair encoded in an -// EKS OIDC issuer URL of the form: +// eksIssuerPathSegments is the fixed path layout of a spinifex EKS OIDC issuer: // -// https://oidc.eks.{region}.{suffix}/{accountID}/{clusterName} +// oidc / eks / {region} / {accountID} / {clusterName} +const ( + eksIssuerPathSegments = 5 + eksIssuerSegOIDC = 0 + eksIssuerSegEKS = 1 + eksIssuerSegRegion = 2 + eksIssuerSegAccountID = 3 + eksIssuerSegCluster = 4 +) + +// ParseEKSIssuerURL extracts the (accountID, clusterName) pair encoded in a +// spinifex EKS OIDC issuer URL of the form: // -// Returns an error for any URL that does not match that exact two-segment -// path structure under an `oidc.eks.` host prefix — defensive parsing prevents -// a maliciously crafted `iss` claim from steering the JWKS lookup at an -// attacker-controlled cluster bucket. +// https://{gateway-host}/oidc/eks/{region}/{accountID}/{clusterName} +// +// This matches the URL ClusterOIDCIssuer (handlers/eks) emits and that awsgw +// serves /.well-known/openid-configuration + /keys under. The host is the +// awsgw gateway (an IP:port on bare-metal/on-prem, not an `oidc.eks.*` vhost), +// so host trust is established by the IAM OIDC-provider registration — the full +// issuer string is the registry key in verifyOIDCProviderRegistered — and by +// the JWKS signature check, not by a host name pattern. +// +// Returns an error for any URL that does not match this exact 5-segment path +// structure: defensive parsing pins accountID and clusterName at fixed +// positions so a maliciously crafted `iss` claim cannot steer the JWKS lookup +// at a different cluster bucket. func ParseEKSIssuerURL(issuer string) (accountID, clusterName string, err error) { u, err := url.Parse(issuer) if err != nil { @@ -64,14 +84,18 @@ func ParseEKSIssuerURL(issuer string) (accountID, clusterName string, err error) if u.Scheme != "https" { return "", "", errors.New("issuer URL must use https scheme") } - if !strings.HasPrefix(u.Host, "oidc.eks.") { - return "", "", errors.New("issuer host must start with oidc.eks") + if u.Host == "" { + return "", "", errors.New("issuer URL missing host") } segments := strings.Split(strings.Trim(u.Path, "/"), "/") - if len(segments) != 2 || segments[0] == "" || segments[1] == "" { - return "", "", errors.New("issuer path must be /{accountID}/{clusterName}") + if len(segments) != eksIssuerPathSegments || + segments[eksIssuerSegOIDC] != "oidc" || segments[eksIssuerSegEKS] != "eks" { + return "", "", errors.New("issuer path must be /oidc/eks/{region}/{accountID}/{clusterName}") } - return segments[0], segments[1], nil + if slices.Contains(segments, "") { + return "", "", errors.New("issuer path has empty segment") + } + return segments[eksIssuerSegAccountID], segments[eksIssuerSegCluster], nil } // FetchClusterJWKS reads the JWKS document for an EKS cluster from the diff --git a/spinifex/handlers/sts/oidc_jwks_test.go b/spinifex/handlers/sts/oidc_jwks_test.go index d3a465861..c4ec3b5e4 100644 --- a/spinifex/handlers/sts/oidc_jwks_test.go +++ b/spinifex/handlers/sts/oidc_jwks_test.go @@ -12,7 +12,20 @@ import ( ) func TestParseEKSIssuerURL_Valid(t *testing.T) { - accountID, clusterName, err := ParseEKSIssuerURL("https://oidc.eks.au-mel-1.mulga.local/123456789012/my-cluster") + accountID, clusterName, err := ParseEKSIssuerURL("https://10.0.0.1:9999/oidc/eks/au-mel-1/123456789012/my-cluster") + require.NoError(t, err) + assert.Equal(t, "123456789012", accountID) + assert.Equal(t, "my-cluster", clusterName) +} + +// Cross-component contract: a token whose iss is built by ClusterOIDCIssuer +// must parse cleanly via ParseEKSIssuerURL and resolve to the same +// (accountID, clusterName). This is the collision that bead 165.11 fixes. +func TestParseEKSIssuerURL_RoundTripsClusterOIDCIssuer(t *testing.T) { + issuer, err := handlers_eks.ClusterOIDCIssuer("https://10.0.0.1:9999", "au-mel-1", "123456789012", "my-cluster") + require.NoError(t, err) + + accountID, clusterName, err := ParseEKSIssuerURL(issuer) require.NoError(t, err) assert.Equal(t, "123456789012", accountID) assert.Equal(t, "my-cluster", clusterName) @@ -23,13 +36,15 @@ func TestParseEKSIssuerURL_Invalid(t *testing.T) { name string issuer string }{ - {"http scheme", "http://oidc.eks.au-mel-1.mulga.local/123456789012/c1"}, - {"wrong host prefix", "https://attacker.example.com/123456789012/c1"}, - {"single segment", "https://oidc.eks.au-mel-1.mulga.local/123456789012"}, - {"three segments", "https://oidc.eks.au-mel-1.mulga.local/123/c/extra"}, - {"empty cluster segment", "https://oidc.eks.au-mel-1.mulga.local/123456789012/"}, - {"empty account segment", "https://oidc.eks.au-mel-1.mulga.local//c1"}, - {"empty path", "https://oidc.eks.au-mel-1.mulga.local"}, + {"http scheme", "http://10.0.0.1:9999/oidc/eks/au-mel-1/123456789012/c1"}, + {"missing host", "https:///oidc/eks/au-mel-1/123456789012/c1"}, + {"wrong path prefix", "https://10.0.0.1:9999/foo/eks/au-mel-1/123456789012/c1"}, + {"too few segments", "https://10.0.0.1:9999/oidc/eks/au-mel-1/123456789012"}, + {"too many segments", "https://10.0.0.1:9999/oidc/eks/au-mel-1/123456789012/c1/extra"}, + {"empty account segment", "https://10.0.0.1:9999/oidc/eks/au-mel-1//c1"}, + {"empty cluster segment", "https://10.0.0.1:9999/oidc/eks/au-mel-1/123456789012/"}, + {"empty path", "https://10.0.0.1:9999"}, + {"legacy aws two-segment shape", "https://oidc.eks.au-mel-1.mulga.local/123456789012/c1"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { diff --git a/spinifex/handlers/sysinstance/launcher.go b/spinifex/handlers/sysinstance/launcher.go new file mode 100644 index 000000000..7b20ee677 --- /dev/null +++ b/spinifex/handlers/sysinstance/launcher.go @@ -0,0 +1,155 @@ +// Package sysinstance holds the boot-agnostic contract for launching +// system-managed VMs. A system instance is owned by the system account and +// hidden from customer DescribeInstances calls. Two boot styles are supported: +// +// - BootDirect: a direct-boot microVM (bundled vmlinuz+initramfs, per-VM +// config delivered via QEMU fw_cfg blobs; no AMI, volume, or cloud-init). +// Used by the ELBv2 LB agent. +// - BootAMI: a full AMI boot (root volume cloned from an AMI, cloud-init +// user-data + network-config seed, q35 machine). Used by the EKS K3s +// control-plane VM. +// +// The types live here (not in handlers/elbv2) so both ELBv2 and EKS can launch +// system instances without an eks→elbv2 dependency. handlers/elbv2 re-exports +// them as type aliases for source compatibility. +package sysinstance + +import "errors" + +// ErrSystemInstanceNotFound is returned by TerminateSystemInstance when the +// target instance is unknown to the daemon — already terminated, or never +// present on this node. Teardown callers treat it as idempotent success so a +// retried delete (e.g. EKS DeleteCluster after the VM already drained) does not +// wedge forever on a VM that is legitimately gone. +var ErrSystemInstanceNotFound = errors.New("sysinstance: instance not found") + +// BootMode selects how a system instance boots. +type BootMode string + +const ( + // BootDirect is the direct-boot microVM path (fw_cfg-delivered config). + BootDirect BootMode = "direct" + // BootAMI is the AMI + cloud-init path (root volume + seed ISO). + BootAMI BootMode = "ami" +) + +// SystemInstanceLauncher is the subset of daemon functionality needed to launch +// and terminate system-managed VMs. Defined here to avoid a circular import +// between the launching handlers (elbv2, eks) and daemon. +type SystemInstanceLauncher interface { + // LaunchSystemInstance creates and starts a system-managed VM. The VM is + // assigned to the system account and not visible to customers. Returns the + // instance ID and private IP once the VM is running. + LaunchSystemInstance(input *SystemInstanceInput) (*SystemInstanceOutput, error) + + // TerminateSystemInstance stops and cleans up a system-managed VM. + TerminateSystemInstance(instanceID string) error +} + +// SystemInstanceInput describes a system VM to launch. The BootMode field +// selects which subset of fields is consumed: +// +// - BootDirect uses NICs, LBAgentEnv, CACert (fw_cfg blobs); ImageID/UserData +// are ignored. +// - BootAMI uses ImageID and UserData (cloud-init); NICs/LBAgentEnv/CACert are +// ignored. The mgmt NIC is allocated by the daemon and rendered into the +// cloud-init network-config from the VM record. +// +// JSON tags are explicit because this struct crosses the +// system.LaunchInstance.* NATS subject root when a launch is handed off to the +// cluster — see daemon/daemon_system_dispatch.go. +type SystemInstanceInput struct { + // BootMode selects the boot style. Empty defaults to BootDirect for + // backward compatibility with the original ELBv2-only contract. + BootMode BootMode `json:"boot_mode,omitempty"` + + // ManagedBy is the spinifex:managed-by tag value stamped on the instance so + // the UI/customer listings hide it (e.g. "elbv2", "eks"). + ManagedBy string `json:"managed_by,omitempty"` + + InstanceType string `json:"instance_type"` // e.g. "sys.micro" + SubnetID string `json:"subnet_id"` // VPC subnet for networking + + // ENI fields — the VM always uses a pre-created ENI (the primary ENI). + ENIID string `json:"eni_id"` + ENIMac string `json:"eni_mac"` + ENIIP string `json:"eni_ip"` + + // ExtraENIs lists additional pre-created ENIs to attach alongside the + // primary ENI (e.g. multi-subnet ALBs). + ExtraENIs []ExtraENIInput `json:"extra_enis,omitempty"` + + // Scheme is the ALB scheme ("internet-facing" or "internal"). BootDirect + // only; ignored for BootAMI. + Scheme string `json:"scheme,omitempty"` + + // AccountID is the owner account of the primary ENI. Required so the daemon + // can look up and update the ENI record (which is keyed by account). + AccountID string `json:"account_id"` + + // HostfwdPorts forwards additional guest ports from the host via the QEMU + // user-mode dev NIC (dev_networking mode only). + HostfwdPorts []int `json:"hostfwd_ports,omitempty"` + + // NICs defines the network interfaces for a BootDirect microVM. Index 0 is + // the primary VPC ENI, index 1 the mgmt NIC, index 2+ extra ENIs. + NICs []NICConfig `json:"nics,omitempty"` + + // LBAgentEnv is a KEY=value blob written to /etc/conf.d/lb-agent inside a + // BootDirect guest via fw_cfg. + LBAgentEnv string `json:"lb_agent_env,omitempty"` + + // CACert holds PEM CA bytes delivered to a BootDirect guest via fw_cfg. + CACert string `json:"ca_cert,omitempty"` + + // ImageID is the AMI to clone the root volume from (BootAMI only). + ImageID string `json:"image_id,omitempty"` + + // UserData is the raw (un-encoded) cloud-init user-data for a BootAMI guest. + // The launcher base64-encodes it for the RunInstances path. + UserData string `json:"user_data,omitempty"` +} + +// ExtraENIInput describes an additional pre-created ENI to attach to a system +// instance. The primary ENI is still passed via ENIID/ENIMac/ENIIP. +type ExtraENIInput struct { + ENIID string `json:"eni_id"` + ENIMac string `json:"eni_mac"` + ENIIP string `json:"eni_ip"` + SubnetID string `json:"subnet_id"` +} + +// NICConfig describes a single network interface for a BootDirect microVM. +// Index 0 is the primary VPC ENI, index 1 the management NIC, index 2+ extra. +type NICConfig struct { + MAC string `json:"mac"` // e.g. "02:0a:01:23:45:67" + CIDR string `json:"cidr"` // e.g. "10.0.1.5/24" + Gateway string `json:"gateway,omitempty"` // e.g. "10.0.1.1"; empty for mgmt NIC + IsDefault bool `json:"is_default"` // true for exactly one NIC (primary VPC ENI) + RouteDst string `json:"route_dst,omitempty"` // specific host route dst, e.g. "10.20.0.5/32" + RouteVia string `json:"route_via,omitempty"` // next-hop for RouteDst +} + +// RecoveryContext carries the per-VM fields needed to reconstruct a +// SystemInstanceInput during host-reboot recovery without importing the vm +// package. MgmtMAC / MgmtIP are the values the daemon already allocated on a +// prior launch; the service injects them back into NIC[1] rather than +// re-allocating from the mgmt IPAM. +type RecoveryContext struct { + InstanceID string + InstanceType string + ENIMac string + MgmtMAC string + MgmtIP string +} + +// SystemInstanceOutput contains the result of a successful launch. +type SystemInstanceOutput struct { + InstanceID string `json:"instance_id"` // e.g. "i-xxxxx" + PrivateIP string `json:"private_ip"` // VPC private IP + PublicIP string `json:"public_ip,omitempty"` // only for internet-facing scheme + MgmtIP string `json:"mgmt_ip,omitempty"` // br-mgmt NIC IP, reachable from the host + + // HostfwdMap maps guest port → host port for any forwarded ports. + HostfwdMap map[int]int `json:"hostfwd_map,omitempty"` +} diff --git a/spinifex/network/external/igw.go b/spinifex/network/external/igw.go index 7c9375ae9..1f2d5c351 100644 --- a/spinifex/network/external/igw.go +++ b/spinifex/network/external/igw.go @@ -47,6 +47,15 @@ type IGWManager interface { EnsureSubnetEgressDrop(ctx context.Context, vpcID, subnetID string, prefix netip.Prefix) error // RemoveSubnetEgressDrop is the inverse. Idempotent. RemoveSubnetEgressDrop(ctx context.Context, vpcID, subnetID string, prefix netip.Prefix) error + // EnsureSystemInstanceEgress wires egress-only internet access for a + // single system instance (e.g. an EKS K3s server VM) sourced from + // instanceIP in subnetID, SNAT'd to externalIP. Installs a /32 reroute + // policy at SystemInstanceEgressPriority (above the drop gate so it works + // even on a drop-gated subnet) plus a plain snat (no DNAT, no inbound). + // Requires AttachIGW to have run first. Idempotent. + EnsureSystemInstanceEgress(ctx context.Context, vpcID, subnetID, instanceIP, externalIP string) error + // RemoveSystemInstanceEgress is the inverse. Idempotent. + RemoveSystemInstanceEgress(ctx context.Context, vpcID, subnetID, instanceIP, externalIP string) error } // IGWManagerConfig is the construction-time bag for igwManager. @@ -356,12 +365,75 @@ func (m *igwManager) ensureSubnetEgressAtPriority(ctx context.Context, vpcID, su }) } +// EnsureSystemInstanceEgress installs an egress-only path for one system +// instance: a /32 reroute policy out the gateway port plus a plain snat +// (instanceIP -> externalIP). The reroute sits above the subnet drop gate so +// it works even where the instance's subnet is otherwise private; where the +// subnet already has a 1000 reroute it is harmless (same nexthop). The snat is +// not dnat_and_snat, so the instance is never reachable inbound. +func (m *igwManager) EnsureSystemInstanceEgress(ctx context.Context, vpcID, subnetID, instanceIP, externalIP string) error { + if vpcID == "" || subnetID == "" { + return errors.New("EnsureSystemInstanceEgress: vpcID and subnetID required") + } + srcIP, err := netip.ParseAddr(instanceIP) + if err != nil { + return fmt.Errorf("EnsureSystemInstanceEgress: parse instance IP %q: %w", instanceIP, err) + } + if externalIP == "" { + return errors.New("EnsureSystemInstanceEgress: externalIP required") + } + + _, nexthop, _, err := m.resolveGatewayNetwork(ctx, vpcID) + if err != nil { + return fmt.Errorf("resolve gateway network for %s: %w", vpcID, err) + } + if nexthop == "" { + return fmt.Errorf("EnsureSystemInstanceEgress: no gateway nexthop for %s (IGW not attached?)", vpcID) + } + + if err := m.routes.AddSystemInstanceEgress(ctx, vpcID, policy.SystemInstanceEgressSpec{ + SubnetID: subnetID, + SrcIP: srcIP, + Prefix: defaultRoutePrefix, + Nexthop: nexthop, + OutputPort: topology.GatewayRouterPort(vpcID), + ExcludeCIDRs: m.vpcExcludeCIDRs(ctx, vpcID), + }); err != nil { + return fmt.Errorf("add system instance egress reroute: %w", err) + } + if err := m.nat.AddSystemInstanceSNAT(ctx, vpcID, instanceIP+"/32", externalIP); err != nil { + return fmt.Errorf("add system instance egress snat: %w", err) + } + return nil +} + +// RemoveSystemInstanceEgress deletes the policy + snat installed by +// EnsureSystemInstanceEgress. Idempotent. +func (m *igwManager) RemoveSystemInstanceEgress(ctx context.Context, vpcID, subnetID, instanceIP, externalIP string) error { + if vpcID == "" || subnetID == "" { + return errors.New("RemoveSystemInstanceEgress: vpcID and subnetID required") + } + srcIP, err := netip.ParseAddr(instanceIP) + if err != nil { + return fmt.Errorf("RemoveSystemInstanceEgress: parse instance IP %q: %w", instanceIP, err) + } + var firstErr error + if err := m.routes.DeleteSystemInstanceEgress(ctx, vpcID, subnetID, srcIP, defaultRoutePrefix, m.vpcExcludeCIDRs(ctx, vpcID)); err != nil { + firstErr = fmt.Errorf("delete system instance egress reroute: %w", err) + } + if err := m.nat.DeleteSystemInstanceSNAT(ctx, vpcID, instanceIP+"/32"); err != nil && firstErr == nil { + firstErr = fmt.Errorf("delete system instance egress snat: %w", err) + } + return firstErr +} + // linkLocalCIDR / multicastCIDR are appended to drop-policy excludes so // link-local (DHCP, metadata, 169.254.169.254) and multicast destinations // are not killed by the per-subnet gate. var ( - linkLocalCIDR = netip.MustParsePrefix("169.254.0.0/16") - multicastCIDR = netip.MustParsePrefix("224.0.0.0/4") + linkLocalCIDR = netip.MustParsePrefix("169.254.0.0/16") + multicastCIDR = netip.MustParsePrefix("224.0.0.0/4") + defaultRoutePrefix = netip.MustParsePrefix("0.0.0.0/0") ) // dropExcludeCIDRs returns the exclusion list for a per-subnet drop policy: diff --git a/spinifex/network/external/igw_test.go b/spinifex/network/external/igw_test.go index 1a6802ded..1e46e85d9 100644 --- a/spinifex/network/external/igw_test.go +++ b/spinifex/network/external/igw_test.go @@ -375,6 +375,114 @@ func TestRemoveNATGatewaySubnetEgress_PropagatesDeleteError(t *testing.T) { require.Error(t, err) } +// TestEnsureSystemInstanceEgress installs the /32 reroute plus the snat-only +// NAT, and asserts no inbound dnat_and_snat row exists (egress-only). +func TestEnsureSystemInstanceEgress(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedVPCRouter(t, m, "vpc-1", "10.0.0.0/16") + pool := &ExternalPoolConfig{Name: "p", Gateway: "192.168.1.1", PrefixLen: 24} + mgr, _ := newTestIGWManager(t, m, policy.NATModeDistributed, pool, LinkLocalAllocator{}, []string{"chassis-a"}) + + require.NoError(t, mgr.AttachIGW(ctx, IGWSpec{VPCID: "vpc-1", InternetGatewayID: "igw-1"})) + require.NoError(t, mgr.EnsureSystemInstanceEgress(ctx, "vpc-1", "subnet-k3s", "10.0.4.10", "203.0.113.7")) + + policies, err := m.ListLogicalRouterPolicies(ctx, topology.VPCRouter("vpc-1")) + require.NoError(t, err) + require.Len(t, policies, 1) + assert.Equal(t, policy.SystemInstanceEgressPriority, policies[0].Priority) + assert.Contains(t, policies[0].Match, "ip4.src == 10.0.4.10/32") + + var snat, dnat *nbdb.NAT + for _, n := range m.NATs { + switch n.Type { + case "snat": + snat = n + case "dnat_and_snat": + dnat = n + } + } + require.NotNil(t, snat, "egress snat must be installed") + assert.Equal(t, "203.0.113.7", snat.ExternalIP) + assert.Equal(t, "10.0.4.10/32", snat.LogicalIP) + assert.Nil(t, dnat, "egress-only: no inbound dnat_and_snat may be installed") +} + +// TestEnsureSystemInstanceEgress_IsIdempotent re-runs install; row counts stay 1. +func TestEnsureSystemInstanceEgress_IsIdempotent(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedVPCRouter(t, m, "vpc-1", "10.0.0.0/16") + pool := &ExternalPoolConfig{Name: "p", Gateway: "192.168.1.1", PrefixLen: 24} + mgr, _ := newTestIGWManager(t, m, policy.NATModeDistributed, pool, LinkLocalAllocator{}, nil) + + require.NoError(t, mgr.AttachIGW(ctx, IGWSpec{VPCID: "vpc-1", InternetGatewayID: "igw-1"})) + require.NoError(t, mgr.EnsureSystemInstanceEgress(ctx, "vpc-1", "subnet-k3s", "10.0.4.10", "203.0.113.7")) + require.NoError(t, mgr.EnsureSystemInstanceEgress(ctx, "vpc-1", "subnet-k3s", "10.0.4.10", "203.0.113.7")) + + policies, err := m.ListLogicalRouterPolicies(ctx, topology.VPCRouter("vpc-1")) + require.NoError(t, err) + assert.Len(t, policies, 1) + snatCount := 0 + for _, n := range m.NATs { + if n.Type == "snat" { + snatCount++ + } + } + assert.Equal(t, 1, snatCount) +} + +// TestRemoveSystemInstanceEgress tears down both the reroute and the snat and +// is idempotent on a second call. +func TestRemoveSystemInstanceEgress(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedVPCRouter(t, m, "vpc-1", "10.0.0.0/16") + pool := &ExternalPoolConfig{Name: "p", Gateway: "192.168.1.1", PrefixLen: 24} + mgr, _ := newTestIGWManager(t, m, policy.NATModeDistributed, pool, LinkLocalAllocator{}, nil) + + require.NoError(t, mgr.AttachIGW(ctx, IGWSpec{VPCID: "vpc-1", InternetGatewayID: "igw-1"})) + require.NoError(t, mgr.EnsureSystemInstanceEgress(ctx, "vpc-1", "subnet-k3s", "10.0.4.10", "203.0.113.7")) + require.NoError(t, mgr.RemoveSystemInstanceEgress(ctx, "vpc-1", "subnet-k3s", "10.0.4.10", "203.0.113.7")) + + policies, err := m.ListLogicalRouterPolicies(ctx, topology.VPCRouter("vpc-1")) + require.NoError(t, err) + assert.Empty(t, policies) + for _, n := range m.NATs { + assert.NotEqual(t, "snat", n.Type, "snat must be removed") + } + + require.NoError(t, mgr.RemoveSystemInstanceEgress(ctx, "vpc-1", "subnet-k3s", "10.0.4.10", "203.0.113.7")) +} + +// TestSystemInstanceEgress_RejectsBadArgs guards validation for malformed events. +func TestSystemInstanceEgress_RejectsBadArgs(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedVPCRouter(t, m, "vpc-1", "10.0.0.0/16") + pool := &ExternalPoolConfig{Name: "p", Gateway: "192.168.1.1", PrefixLen: 24} + mgr, _ := newTestIGWManager(t, m, policy.NATModeDistributed, pool, LinkLocalAllocator{}, nil) + require.NoError(t, mgr.AttachIGW(ctx, IGWSpec{VPCID: "vpc-1", InternetGatewayID: "igw-1"})) + + require.Error(t, mgr.EnsureSystemInstanceEgress(ctx, "", "subnet-k3s", "10.0.4.10", "203.0.113.7")) + require.Error(t, mgr.EnsureSystemInstanceEgress(ctx, "vpc-1", "", "10.0.4.10", "203.0.113.7")) + require.Error(t, mgr.EnsureSystemInstanceEgress(ctx, "vpc-1", "subnet-k3s", "not-an-ip", "203.0.113.7")) + require.Error(t, mgr.EnsureSystemInstanceEgress(ctx, "vpc-1", "subnet-k3s", "10.0.4.10", "")) +} + +// TestEnsureSystemInstanceEgress_RequiresAttachedIGW asserts a clear error when +// no gateway nexthop exists — without it the reroute would be unreachable. +func TestEnsureSystemInstanceEgress_RequiresAttachedIGW(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedVPCRouter(t, m, "vpc-1", "10.0.0.0/16") + pool := &ExternalPoolConfig{Name: "p", PrefixLen: 24} // no Gateway → no centralized nexthop + mgr, _ := newTestIGWManager(t, m, policy.NATModeCentralized, pool, failingAllocator{}, nil) + + err := mgr.EnsureSystemInstanceEgress(ctx, "vpc-1", "subnet-k3s", "10.0.4.10", "203.0.113.7") + require.Error(t, err) +} + // TestRemoveSubnetEgress_RejectsEmptyArgs covers the IGW-priority sibling of // the NATGW empty-arg guard. Both vpcID and subnetID are mandatory. func TestRemoveSubnetEgress_RejectsEmptyArgs(t *testing.T) { diff --git a/spinifex/network/policy/nat.go b/spinifex/network/policy/nat.go index dd6d82993..3d7a71110 100644 --- a/spinifex/network/policy/nat.go +++ b/spinifex/network/policy/nat.go @@ -51,6 +51,16 @@ type NATManager interface { // DeleteSNAT removes the IGW default-outbound snat for vpcCIDR. DeleteSNAT(ctx context.Context, vpcID, vpcCIDR string) error + + // AddSystemInstanceSNAT installs an egress-only snat rewriting a single + // system instance's /32 (logicalIP) to externalIP. Unlike AddEIP it is a + // plain snat, not dnat_and_snat, so there is no inbound path to the + // instance. Idempotent; tagged role=system-instance-egress. + AddSystemInstanceSNAT(ctx context.Context, vpcID, logicalIP, externalIP string) error + + // DeleteSystemInstanceSNAT removes the egress-only snat by logicalIP; + // idempotent. + DeleteSystemInstanceSNAT(ctx context.Context, vpcID, logicalIP string) error } // FlowsBarrier blocks until ovn-northd has compiled NB → SB and every @@ -246,3 +256,48 @@ func (m *natManager) DeleteSNAT(ctx context.Context, vpcID, vpcCIDR string) erro } return nil } + +func (m *natManager) AddSystemInstanceSNAT(ctx context.Context, vpcID, logicalIP, externalIP string) error { + router := topology.VPCRouter(vpcID) + snatRule := &nbdb.NAT{ + Type: "snat", + ExternalIP: externalIP, + LogicalIP: logicalIP, + ExternalIDs: map[string]string{ + "spinifex:vpc_id": vpcID, + "spinifex:public_ip": externalIP, + "spinifex:role": "system-instance-egress", + }, + } + // Skip when the existing row already matches; avoids the duplicate-append + // on a re-published add (the snat is keyed by its unique pool external IP). + if existing, err := m.ovn.FindNATByExternalIP(ctx, "snat", externalIP); err != nil { + slog.Warn("policy: AddSystemInstanceSNAT idempotency lookup failed", "external_ip", externalIP, "err", err) + } else if existing != nil && existing.LogicalIP == logicalIP { + slog.Info("policy: AddSystemInstanceSNAT idempotent skip — rule already current", + "router", router, "external_ip", externalIP, "logical_ip", logicalIP) + return nil + } + + if err := m.ovn.AddNAT(ctx, router, snatRule); err != nil { + return fmt.Errorf("add system-instance snat %s -> %s on %s: %w", logicalIP, externalIP, router, err) + } + // Block until SB + chassis have the SNAT flow; otherwise first egress + // packets from the instance drop. + if err := m.barrier(); err != nil { + slog.Warn("policy: AddSystemInstanceSNAT flows barrier failed", + "logical_ip", logicalIP, "external_ip", externalIP, "err", err) + } + return nil +} + +func (m *natManager) DeleteSystemInstanceSNAT(ctx context.Context, vpcID, logicalIP string) error { + router := topology.VPCRouter(vpcID) + if err := m.ovn.DeleteNAT(ctx, router, "snat", logicalIP); err != nil { + if errors.Is(err, ovn.ErrNATNotFound) { + return nil + } + return fmt.Errorf("delete system-instance snat %s on %s: %w", logicalIP, router, err) + } + return nil +} diff --git a/spinifex/network/policy/nat_test.go b/spinifex/network/policy/nat_test.go index 9b065d832..866aea2a3 100644 --- a/spinifex/network/policy/nat_test.go +++ b/spinifex/network/policy/nat_test.go @@ -370,3 +370,23 @@ func TestNATManager_AddSNAT_AndDelete(t *testing.T) { assert.Nil(t, findNAT(m, "snat", "10.0.0.0/16")) require.NoError(t, nm.DeleteSNAT(ctx, "vpc-1", "10.0.0.0/16")) } + +func TestNATManager_AddSystemInstanceSNAT_AndDelete(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedRouter(t, m, "vpc-1") + nm, _ := NewNATManager(m, NATModeDistributed) + + require.NoError(t, nm.AddSystemInstanceSNAT(ctx, "vpc-1", "172.31.4.10/32", "1.2.3.4")) + got := findNAT(m, "snat", "172.31.4.10/32") + require.NotNil(t, got) + assert.Equal(t, "1.2.3.4", got.ExternalIP) + assert.Equal(t, "system-instance-egress", got.ExternalIDs["spinifex:role"]) + // snat-only: no dnat_and_snat row, so the instance stays unreachable inbound. + assert.Nil(t, findNAT(m, "dnat_and_snat", "172.31.4.10/32")) + + require.NoError(t, nm.DeleteSystemInstanceSNAT(ctx, "vpc-1", "172.31.4.10/32")) + assert.Nil(t, findNAT(m, "snat", "172.31.4.10/32")) + // Idempotent on a missing rule. + require.NoError(t, nm.DeleteSystemInstanceSNAT(ctx, "vpc-1", "172.31.4.10/32")) +} diff --git a/spinifex/network/policy/route.go b/spinifex/network/policy/route.go index 7893e8595..fcf5a0fac 100644 --- a/spinifex/network/policy/route.go +++ b/spinifex/network/policy/route.go @@ -24,6 +24,13 @@ const ( SubnetEgressPriorityDrop = 1100 SubnetEgressPriorityIGW = 1000 SubnetEgressPriorityNATGW = 900 + + // SystemInstanceEgressPriority sits above the drop gate (1100) so a single + // system instance (e.g. an EKS K3s server VM) egresses even when its subnet + // is otherwise drop-gated. The policy is scoped to the instance's /32 + // source, so peer instances in the same subnet are unaffected — they still + // hit the 1100 drop (or the subnet's own 1000 reroute where one exists). + SystemInstanceEgressPriority = 1200 ) // RouteSpec is a static route on a VPC's LogicalRouter. OutputPort is @@ -52,6 +59,20 @@ type SubnetEgressSpec struct { ExcludeCIDRs []netip.Prefix } +// SystemInstanceEgressSpec describes a per-instance egress override installed +// as an OVN Logical_Router_Policy at SystemInstanceEgressPriority. The match +// adds an `ip4.src == /32` clause to the subnet-egress shape so the +// reroute is confined to a single system instance; peer instances in the same +// subnet are untouched. Action is "reroute" via Nexthop out OutputPort. +type SystemInstanceEgressSpec struct { + SubnetID string + SrcIP netip.Addr + Prefix netip.Prefix + Nexthop string + OutputPort string + ExcludeCIDRs []netip.Prefix +} + // RouteManager owns static routes on VPC LogicalRouters. Adds are // idempotent (no-op on match, delete-then-add on drift) so the reconciler // can replay safely. @@ -64,6 +85,8 @@ type RouteManager interface { DeleteSubnetEgress(ctx context.Context, vpcID, subnetID string, prefix netip.Prefix, excludeCIDRs []netip.Prefix) error AddSubnetEgressDrop(ctx context.Context, vpcID string, spec SubnetEgressDropSpec) error DeleteSubnetEgressDrop(ctx context.Context, vpcID, subnetID string, prefix netip.Prefix, excludeCIDRs []netip.Prefix) error + AddSystemInstanceEgress(ctx context.Context, vpcID string, spec SystemInstanceEgressSpec) error + DeleteSystemInstanceEgress(ctx context.Context, vpcID, subnetID string, srcIP netip.Addr, prefix netip.Prefix, excludeCIDRs []netip.Prefix) error } // SubnetEgressDropSpec describes a per-subnet egress DROP override installed @@ -263,6 +286,62 @@ func (m *routeManager) DeleteSubnetEgressDrop(ctx context.Context, vpcID, subnet return nil } +func (m *routeManager) AddSystemInstanceEgress(ctx context.Context, vpcID string, spec SystemInstanceEgressSpec) error { + if spec.SubnetID == "" { + return fmt.Errorf("system instance egress: SubnetID required") + } + if !spec.SrcIP.IsValid() { + return fmt.Errorf("system instance egress: SrcIP required") + } + if spec.OutputPort == "" { + return fmt.Errorf("system instance egress: OutputPort required (ovn-northd drops policy reroute otherwise)") + } + router := topology.VPCRouter(vpcID) + match := systemInstanceEgressMatch(spec.SubnetID, spec.SrcIP, spec.Prefix, spec.ExcludeCIDRs) + + existing, err := m.ovn.FindLogicalRouterPolicy(ctx, router, SystemInstanceEgressPriority, match) + if err != nil { + return fmt.Errorf("find LR policy %q on %s: %w", match, router, err) + } + if existing != nil { + if existing.Action == "reroute" && existing.Nexthop != nil && *existing.Nexthop == spec.Nexthop && + existing.ExternalIDs["spinifex:output_port"] == spec.OutputPort { + return nil + } + if err := m.ovn.DeleteLogicalRouterPolicy(ctx, router, SystemInstanceEgressPriority, match); err != nil { + return fmt.Errorf("delete drifted LR policy %q on %s: %w", match, router, err) + } + } + + nexthop := spec.Nexthop + row := &nbdb.LogicalRouterPolicy{ + Priority: SystemInstanceEgressPriority, + Match: match, + Action: "reroute", + Nexthop: &nexthop, + Options: map[string]string{}, + ExternalIDs: map[string]string{ + "spinifex:subnet": spec.SubnetID, + "spinifex:src_ip": spec.SrcIP.String(), + "spinifex:output_port": spec.OutputPort, + "spinifex:role": "system-instance-egress", + }, + } + if err := m.ovn.AddLogicalRouterPolicy(ctx, router, row); err != nil { + return fmt.Errorf("add LR policy %q -> %s on %s: %w", match, spec.Nexthop, router, err) + } + return nil +} + +func (m *routeManager) DeleteSystemInstanceEgress(ctx context.Context, vpcID, subnetID string, srcIP netip.Addr, prefix netip.Prefix, excludeCIDRs []netip.Prefix) error { + router := topology.VPCRouter(vpcID) + match := systemInstanceEgressMatch(subnetID, srcIP, prefix, excludeCIDRs) + if err := m.ovn.DeleteLogicalRouterPolicy(ctx, router, SystemInstanceEgressPriority, match); err != nil { + return fmt.Errorf("delete system instance egress LR policy %q on %s: %w", match, router, err) + } + return nil +} + func subnetEgressMatch(subnetID string, prefix netip.Prefix, excludeCIDRs []netip.Prefix) string { match := fmt.Sprintf(`inport == %q && ip4.dst == %s`, topology.SubnetRouterPort(subnetID), prefix.String()) for _, ex := range excludeCIDRs { @@ -274,6 +353,18 @@ func subnetEgressMatch(subnetID string, prefix netip.Prefix, excludeCIDRs []neti return match } +func systemInstanceEgressMatch(subnetID string, srcIP netip.Addr, prefix netip.Prefix, excludeCIDRs []netip.Prefix) string { + match := fmt.Sprintf(`inport == %q && ip4.src == %s/32 && ip4.dst == %s`, + topology.SubnetRouterPort(subnetID), srcIP.String(), prefix.String()) + for _, ex := range excludeCIDRs { + if !ex.IsValid() { + continue + } + match += fmt.Sprintf(` && ip4.dst != %s`, ex.String()) + } + return match +} + func policyMatches(existing *nbdb.LogicalRouterPolicy, want SubnetEgressSpec) bool { if existing.Action != "reroute" { return false diff --git a/spinifex/network/policy/route_test.go b/spinifex/network/policy/route_test.go index 6fa5a4a33..0ba25dfe0 100644 --- a/spinifex/network/policy/route_test.go +++ b/spinifex/network/policy/route_test.go @@ -131,6 +131,128 @@ func TestRouteManager_AddSubnetEgress_InstallsScopedPolicy(t *testing.T) { assert.Equal(t, "gw-vpc-1", p.ExternalIDs["spinifex:output_port"]) } +func TestRouteManager_AddSystemInstanceEgress_InstallsScopedPolicy(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedRouter(t, m, "vpc-1") + rm := NewRouteManager(m) + + require.NoError(t, rm.AddSystemInstanceEgress(ctx, "vpc-1", SystemInstanceEgressSpec{ + SubnetID: "subnet-k3s", + SrcIP: netip.MustParseAddr("172.31.4.10"), + Prefix: netip.MustParsePrefix("0.0.0.0/0"), + Nexthop: "192.168.1.1", + OutputPort: "gw-vpc-1", + })) + + policies, err := m.ListLogicalRouterPolicies(ctx, topology.VPCRouter("vpc-1")) + require.NoError(t, err) + require.Len(t, policies, 1) + p := policies[0] + assert.Equal(t, "reroute", p.Action) + assert.Equal(t, SystemInstanceEgressPriority, p.Priority) + require.NotNil(t, p.Nexthop) + assert.Equal(t, "192.168.1.1", *p.Nexthop) + // Scoped to the single instance /32 source so peers in the subnet are untouched. + assert.Contains(t, p.Match, "ip4.src == 172.31.4.10/32") + assert.Contains(t, p.Match, topology.SubnetRouterPort("subnet-k3s")) + assert.Equal(t, "172.31.4.10", p.ExternalIDs["spinifex:src_ip"]) + assert.Equal(t, "system-instance-egress", p.ExternalIDs["spinifex:role"]) +} + +func TestRouteManager_AddSystemInstanceEgress_IsIdempotent(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedRouter(t, m, "vpc-1") + rm := NewRouteManager(m) + + spec := SystemInstanceEgressSpec{ + SubnetID: "subnet-k3s", + SrcIP: netip.MustParseAddr("172.31.4.10"), + Prefix: netip.MustParsePrefix("0.0.0.0/0"), + Nexthop: "192.168.1.1", + OutputPort: "gw-vpc-1", + } + require.NoError(t, rm.AddSystemInstanceEgress(ctx, "vpc-1", spec)) + require.NoError(t, rm.AddSystemInstanceEgress(ctx, "vpc-1", spec)) + require.NoError(t, rm.AddSystemInstanceEgress(ctx, "vpc-1", spec)) + + policies, err := m.ListLogicalRouterPolicies(ctx, topology.VPCRouter("vpc-1")) + require.NoError(t, err) + assert.Len(t, policies, 1) +} + +func TestRouteManager_AddSystemInstanceEgress_DriftReplacesNexthop(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedRouter(t, m, "vpc-1") + rm := NewRouteManager(m) + + spec := SystemInstanceEgressSpec{ + SubnetID: "subnet-k3s", + SrcIP: netip.MustParseAddr("172.31.4.10"), + Prefix: netip.MustParsePrefix("0.0.0.0/0"), + Nexthop: "192.168.1.1", + OutputPort: "gw-vpc-1", + } + require.NoError(t, rm.AddSystemInstanceEgress(ctx, "vpc-1", spec)) + spec.Nexthop = "192.168.1.254" + require.NoError(t, rm.AddSystemInstanceEgress(ctx, "vpc-1", spec)) + + policies, err := m.ListLogicalRouterPolicies(ctx, topology.VPCRouter("vpc-1")) + require.NoError(t, err) + require.Len(t, policies, 1) + require.NotNil(t, policies[0].Nexthop) + assert.Equal(t, "192.168.1.254", *policies[0].Nexthop) +} + +func TestRouteManager_AddSystemInstanceEgress_RejectsMissingFields(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedRouter(t, m, "vpc-1") + rm := NewRouteManager(m) + + valid := SystemInstanceEgressSpec{ + SubnetID: "subnet-k3s", + SrcIP: netip.MustParseAddr("172.31.4.10"), + Prefix: netip.MustParsePrefix("0.0.0.0/0"), + Nexthop: "192.168.1.1", + OutputPort: "gw-vpc-1", + } + + noSubnet := valid + noSubnet.SubnetID = "" + assert.Error(t, rm.AddSystemInstanceEgress(ctx, "vpc-1", noSubnet)) + + noSrc := valid + noSrc.SrcIP = netip.Addr{} + assert.Error(t, rm.AddSystemInstanceEgress(ctx, "vpc-1", noSrc)) + + noPort := valid + noPort.OutputPort = "" + assert.Error(t, rm.AddSystemInstanceEgress(ctx, "vpc-1", noPort)) +} + +func TestRouteManager_DeleteSystemInstanceEgress_RemovesAndIdempotent(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedRouter(t, m, "vpc-1") + rm := NewRouteManager(m) + + src := netip.MustParseAddr("172.31.4.10") + prefix := netip.MustParsePrefix("0.0.0.0/0") + require.NoError(t, rm.AddSystemInstanceEgress(ctx, "vpc-1", SystemInstanceEgressSpec{ + SubnetID: "subnet-k3s", SrcIP: src, Prefix: prefix, Nexthop: "192.168.1.1", OutputPort: "gw-vpc-1", + })) + + require.NoError(t, rm.DeleteSystemInstanceEgress(ctx, "vpc-1", "subnet-k3s", src, prefix, nil)) + policies, err := m.ListLogicalRouterPolicies(ctx, topology.VPCRouter("vpc-1")) + require.NoError(t, err) + assert.Empty(t, policies) + // Idempotent on a missing policy. + require.NoError(t, rm.DeleteSystemInstanceEgress(ctx, "vpc-1", "subnet-k3s", src, prefix, nil)) +} + func TestRouteManager_AddSubnetEgress_IsIdempotent(t *testing.T) { ctx := context.Background() m := mock.New() diff --git a/spinifex/network/subscribers/events.go b/spinifex/network/subscribers/events.go index de3bb0442..6e12c8198 100644 --- a/spinifex/network/subscribers/events.go +++ b/spinifex/network/subscribers/events.go @@ -19,6 +19,8 @@ const ( TopicDeleteIGWRoute = "vpc.delete-igw-route" TopicGateSubnetEgress = "vpc.gate-subnet-egress" TopicUngateSubnetEgress = "vpc.ungate-subnet-egress" + TopicAddSystemEgress = "vpc.add-system-egress" + TopicDeleteSystemEgress = "vpc.delete-system-egress" TopicCreateSG = "vpc.create-sg" TopicDeleteSG = "vpc.delete-sg" TopicUpdateSG = "vpc.update-sg" @@ -113,6 +115,19 @@ type SubnetEgressUngateEvent struct { DestinationCidr string `json:"destination_cidr"` } +// SystemEgressEvent is published on vpc.add-system-egress / +// vpc.delete-system-egress to give a single hidden system instance (e.g. an +// EKS K3s server VM) egress-only internet access. The subscriber installs (or +// removes) a /32 reroute policy at SystemInstanceEgressPriority plus a plain +// snat (InstanceIp -> ExternalIp) — no DNAT, so the instance stays unreachable +// inbound. +type SystemEgressEvent struct { + VpcId string `json:"vpc_id"` + SubnetId string `json:"subnet_id"` + InstanceIp string `json:"instance_ip"` + ExternalIp string `json:"external_ip"` +} + // SGRule mirrors handlers/ec2/vpc.SGRule on the wire (kept local to avoid // the handler import). type SGRule struct { diff --git a/spinifex/network/subscribers/subscribers.go b/spinifex/network/subscribers/subscribers.go index fc3368bc5..69fdd8ca6 100644 --- a/spinifex/network/subscribers/subscribers.go +++ b/spinifex/network/subscribers/subscribers.go @@ -82,6 +82,8 @@ func (s *Subscriber) Subscribe(nc *nats.Conn) ([]*nats.Subscription, error) { {TopicDeleteIGWRoute, s.handleDeleteIGWRoute}, {TopicGateSubnetEgress, s.handleGateSubnetEgress}, {TopicUngateSubnetEgress, s.handleUngateSubnetEgress}, + {TopicAddSystemEgress, s.handleAddSystemEgress}, + {TopicDeleteSystemEgress, s.handleDeleteSystemEgress}, {TopicCreateSG, s.handleCreateSG}, {TopicDeleteSG, s.handleDeleteSG}, {TopicUpdateSG, s.handleUpdateSG}, diff --git a/spinifex/network/subscribers/subscribers_test.go b/spinifex/network/subscribers/subscribers_test.go index 1d2f3c106..e528cb285 100644 --- a/spinifex/network/subscribers/subscribers_test.go +++ b/spinifex/network/subscribers/subscribers_test.go @@ -98,6 +98,7 @@ func TestSubscribe_RegistersAllTopics(t *testing.T) { TopicAddNATGateway, TopicDeleteNATGateway, TopicAddIGWRoute, TopicDeleteIGWRoute, TopicGateSubnetEgress, TopicUngateSubnetEgress, + TopicAddSystemEgress, TopicDeleteSystemEgress, TopicCreateSG, TopicDeleteSG, TopicUpdateSG, } if len(subs) != len(wantTopics) { @@ -173,6 +174,66 @@ func TestHandleAddNATGateway_InstallsPerSubnetEgress(t *testing.T) { }, 2*time.Second, 20*time.Millisecond, "NATGW egress policy must be removed on delete event") } +// TestHandleSystemEgress_InstallsAndRemoves drives the full event path: a +// vpc.add-system-egress event installs the /32 reroute + snat-only NAT, and +// vpc.delete-system-egress tears both down. +func TestHandleSystemEgress_InstallsAndRemoves(t *testing.T) { + ctx := context.Background() + _, nc := testutil.StartTestNATS(t) + sub, m := newTestSubscriber(t) + subs, err := sub.Subscribe(nc) + require.NoError(t, err) + defer func() { + for _, s := range subs { + _ = s.Unsubscribe() + } + }() + + require.NoError(t, m.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{ + Name: topology.VPCRouter("vpc-1"), + ExternalIDs: map[string]string{"spinifex:vpc_id": "vpc-1"}, + })) + require.NoError(t, sub.igw.AttachIGW(ctx, external.IGWSpec{VPCID: "vpc-1", InternetGatewayID: "igw-1"})) + + evt := SystemEgressEvent{ + VpcId: "vpc-1", + SubnetId: "subnet-k3s", + InstanceIp: "10.0.4.10", + ExternalIp: "192.168.1.60", + } + data, err := json.Marshal(evt) + require.NoError(t, err) + require.NoError(t, nc.Publish(TopicAddSystemEgress, data)) + + require.Eventually(t, func() bool { + policies, perr := m.ListLogicalRouterPolicies(ctx, topology.VPCRouter("vpc-1")) + if perr != nil || len(policies) != 1 { + return false + } + return policies[0].Priority == policy.SystemInstanceEgressPriority + }, 2*time.Second, 20*time.Millisecond, "system egress reroute must be installed") + + policies, err := m.ListLogicalRouterPolicies(ctx, topology.VPCRouter("vpc-1")) + require.NoError(t, err) + require.Len(t, policies, 1) + assert.Contains(t, policies[0].Match, "ip4.src == 10.0.4.10/32") + + snat, err := m.FindNATByExternalIP(ctx, "snat", "192.168.1.60") + require.NoError(t, err) + require.NotNil(t, snat) + assert.Equal(t, "10.0.4.10/32", snat.LogicalIP) + + require.NoError(t, nc.Publish(TopicDeleteSystemEgress, data)) + require.Eventually(t, func() bool { + policies, perr := m.ListLogicalRouterPolicies(ctx, topology.VPCRouter("vpc-1")) + if perr != nil || len(policies) != 0 { + return false + } + got, gerr := m.FindNATByExternalIP(ctx, "snat", "192.168.1.60") + return gerr == nil && got == nil + }, 2*time.Second, 20*time.Millisecond, "system egress reroute + snat must be removed on delete") +} + // TestHandleAddNATGateway_InvalidCIDRSkipsPolicy: the SNAT install must still // happen, but a malformed DestinationCidr must not install any LR policy. func TestHandleAddNATGateway_InvalidCIDRSkipsPolicy(t *testing.T) { diff --git a/spinifex/network/subscribers/topology.go b/spinifex/network/subscribers/topology.go index 0db002c9d..745bb46eb 100644 --- a/spinifex/network/subscribers/topology.go +++ b/spinifex/network/subscribers/topology.go @@ -310,6 +310,41 @@ func (s *Subscriber) handleAddNATGateway(msg *nats.Msg) { "natgw_id", evt.NatGatewayId) } +func (s *Subscriber) handleAddSystemEgress(msg *nats.Msg) { + var evt SystemEgressEvent + if err := json.Unmarshal(msg.Data, &evt); err != nil { + slog.Error("subscribers: failed to unmarshal vpc.add-system-egress event", "err", err) + return + } + ctx := context.Background() + if err := s.igw.EnsureSystemInstanceEgress(ctx, evt.VpcId, evt.SubnetId, evt.InstanceIp, evt.ExternalIp); err != nil { + slog.Error("subscribers: EnsureSystemInstanceEgress failed", + "vpc_id", evt.VpcId, "subnet_id", evt.SubnetId, + "instance_ip", evt.InstanceIp, "external_ip", evt.ExternalIp, "err", err) + return + } + slog.Info("subscribers: system instance egress installed", + "vpc_id", evt.VpcId, "subnet_id", evt.SubnetId, + "instance_ip", evt.InstanceIp, "external_ip", evt.ExternalIp) +} + +func (s *Subscriber) handleDeleteSystemEgress(msg *nats.Msg) { + var evt SystemEgressEvent + if err := json.Unmarshal(msg.Data, &evt); err != nil { + slog.Error("subscribers: failed to unmarshal vpc.delete-system-egress event", "err", err) + return + } + ctx := context.Background() + if err := s.igw.RemoveSystemInstanceEgress(ctx, evt.VpcId, evt.SubnetId, evt.InstanceIp, evt.ExternalIp); err != nil { + slog.Error("subscribers: RemoveSystemInstanceEgress failed", + "vpc_id", evt.VpcId, "subnet_id", evt.SubnetId, + "instance_ip", evt.InstanceIp, "external_ip", evt.ExternalIp, "err", err) + return + } + slog.Info("subscribers: system instance egress removed", + "vpc_id", evt.VpcId, "subnet_id", evt.SubnetId, "instance_ip", evt.InstanceIp) +} + func (s *Subscriber) handleDeleteNATGateway(msg *nats.Msg) { var evt NATGatewayEvent if err := json.Unmarshal(msg.Data, &evt); err != nil {