mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-21 03:36:35 +00:00
Merge branch 'dev' of github.com:mulgadc/spinifex into dev
This commit is contained in:
tomnewton-mulga
@@ -2,6 +2,13 @@ name: E2E Bare-Metal
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
workflow_call:
|
||||
inputs:
|
||||
ref:
|
||||
description: "spinifex ref to checkout/test (default: triggering ref)"
|
||||
required: false
|
||||
type: string
|
||||
default: ""
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -12,7 +19,7 @@ env:
|
||||
|
||||
jobs:
|
||||
e2e_baremetal:
|
||||
name: Bare-Metal Multi-Node E2E
|
||||
name: Bare-Metal Single-Node E2E
|
||||
runs-on: [ self-hosted, ci-baremetal ]
|
||||
timeout-minutes: 120
|
||||
concurrency:
|
||||
@@ -20,11 +27,17 @@ jobs:
|
||||
cancel-in-progress: false
|
||||
env:
|
||||
BM_ISO_CACHE: /var/cache/spinifex-iso-builder
|
||||
ARTIFACT_DIR: /tmp/spinifex-e2e-baremetal-artifacts-${{ github.run_id }}
|
||||
steps:
|
||||
- name: Clean Workspace
|
||||
run: sudo rm -rf "$GITHUB_WORKSPACE"/*
|
||||
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
# When called (nightly/release) the caller picks the ref (dev on
|
||||
# schedule, the tag on release); standalone dispatch defaults to the
|
||||
# triggering ref.
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
|
||||
- uses: actions/create-github-app-token@v3
|
||||
id: app-token
|
||||
@@ -92,6 +105,9 @@ jobs:
|
||||
fi
|
||||
|
||||
|
||||
- name: Install go-junit-report on runner
|
||||
run: go install github.com/jstemmer/go-junit-report/v2@latest
|
||||
|
||||
- name: Run Bare-Metal E2E
|
||||
env:
|
||||
BM_CI_PASSWORD: ${{ secrets.BM_CI_PASSWORD }}
|
||||
@@ -100,8 +116,38 @@ jobs:
|
||||
BM_UBUNTU_IMAGE: ${{ env.BM_ISO_CACHE }}/ubuntu-26.04-cloudimg-amd64.img
|
||||
SPX_EXTERNAL_POOL: ${{ env.SPINIFEX_CI_POOL }}
|
||||
run: |
|
||||
mkdir -p "${ARTIFACT_DIR}"
|
||||
# Capture combined bootstrap + e2e output so failures produce a
|
||||
# downloadable, analyzable artifact (the suites run go test -v on the
|
||||
# init node and stream back over SSH). pipefail keeps the script's
|
||||
# real exit code through the tee.
|
||||
set -o pipefail
|
||||
sudo -E mulga/scripts/baremetal/run-baremetal-e2e.sh \
|
||||
--vmlinuz "${BM_ISO_CACHE}/vmlinuz" \
|
||||
--initrd "${BM_ISO_CACHE}/initrd.img" \
|
||||
--squashfs "${BM_ISO_CACHE}/spinifex.squashfs" \
|
||||
--tests-tar "${RUNNER_TEMP}/spinifex-tests.tar"
|
||||
--tests-tar "${RUNNER_TEMP}/spinifex-tests.tar" \
|
||||
2>&1 | tee "${ARTIFACT_DIR}/test-baremetal.log"
|
||||
|
||||
- name: Generate JUnit report
|
||||
if: always()
|
||||
run: |
|
||||
LOG="${ARTIFACT_DIR}/test-baremetal.log"
|
||||
[ -f "$LOG" ] || exit 0
|
||||
go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-baremetal.xml" || true
|
||||
|
||||
- name: Analyze e2e failures
|
||||
if: always()
|
||||
uses: ./.github/actions/e2e-analyze
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
log-dir: ${{ env.ARTIFACT_DIR }}
|
||||
title: "Bare-Metal E2E failure analysis"
|
||||
|
||||
- uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: e2e-baremetal-artifacts-${{ github.run_id }}
|
||||
path: ${{ env.ARTIFACT_DIR }}
|
||||
if-no-files-found: warn
|
||||
retention-days: 14
|
||||
|
||||
@@ -216,3 +216,22 @@ jobs:
|
||||
tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
|
||||
tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1
|
||||
./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
|
||||
|
||||
# Bare-metal single-node suite (PXE-reimaged physical hardware). Doesn't fit
|
||||
# the tofu matrix shape (different runner/provisioning/120m timeout), so it
|
||||
# runs as a sibling job via the reusable e2e-baremetal workflow. Gated like a
|
||||
# matrix cell ("cell 20"): every weeknight cron, or dispatch with 20 selected
|
||||
# (blank cells = all). Tests `dev` on schedule to match GIT_BRANCH above.
|
||||
baremetal:
|
||||
name: cell-20 (baremetal/pxe/real-hw)
|
||||
if: >-
|
||||
github.event_name == 'schedule' ||
|
||||
inputs.cells == '' ||
|
||||
contains(format(',{0},', inputs.cells), ',20,')
|
||||
permissions:
|
||||
contents: read
|
||||
statuses: write
|
||||
uses: ./.github/workflows/e2e-baremetal.yml
|
||||
with:
|
||||
ref: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
|
||||
secrets: inherit
|
||||
|
||||
+114
-9
@@ -18,9 +18,9 @@ on:
|
||||
description: "PR head SHA to report status on"
|
||||
required: false
|
||||
suites:
|
||||
description: "Space-separated Make targets to run (e.g. 'e2e-single e2e-multinode e2e-cert e2e-lb')"
|
||||
description: "Space-separated Make targets to run (e.g. 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-eks')"
|
||||
required: false
|
||||
default: "e2e-single e2e-multinode e2e-cert e2e-lb"
|
||||
default: "e2e-single e2e-multinode e2e-cert e2e-lb e2e-eks"
|
||||
test_run:
|
||||
description: "Optional regex for -test.run (e.g. '^TestMultinodeIPSec$'). Empty = run all tests in suite."
|
||||
required: false
|
||||
@@ -32,7 +32,7 @@ permissions:
|
||||
|
||||
env:
|
||||
GIT_BRANCH: ${{ github.ref_name }}
|
||||
SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-lb' }}
|
||||
SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-eks' }}
|
||||
TEST_RUN: ${{ inputs.test_run || '' }}
|
||||
|
||||
jobs:
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
# suites='e2e-reboot' only). Push triggers and the default SUITES list
|
||||
# both include single/cert/lb so the inputs.suites null-check keeps the
|
||||
# job running on plain PR pushes.
|
||||
if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-lb')
|
||||
if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-lb') || contains(github.event.inputs.suites, 'e2e-eks')
|
||||
runs-on: [ self-hosted, ci-single ]
|
||||
timeout-minutes: 45
|
||||
env:
|
||||
@@ -70,7 +70,15 @@ jobs:
|
||||
true
|
||||
|
||||
- name: Clean Workspace
|
||||
run: sudo rm -rf "$GITHUB_WORKSPACE"/*
|
||||
run: |
|
||||
# Detach any leaked mounts under the workspace before deleting. A
|
||||
# killed image build can leave a `mount --bind /dev` under its build
|
||||
# dir; if `rm -rf` traverses it as root it wipes the host's /dev
|
||||
# (deletes /dev/null and wedges the box). umount deepest-first, then
|
||||
# forbid rm from crossing any mount boundary as a hard backstop.
|
||||
awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
|
||||
| sort -r | xargs -r -I{} sudo umount -l -R {} || true
|
||||
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
|
||||
|
||||
- uses: actions/create-github-app-token@v3
|
||||
id: app-token
|
||||
@@ -146,6 +154,82 @@ jobs:
|
||||
echo "wan_ip=${WAN_IP}" >> "$GITHUB_OUTPUT"
|
||||
echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# The eks suite's CreateCluster boots a control-plane VM from the
|
||||
# eks-node AMI, so that AMI must exist in the VM's image store before the
|
||||
# suite runs. build-system-image.sh customizes the image with libguestfs
|
||||
# (an isolated userspace appliance) — it touches no host block device, so
|
||||
# it is safe on the shared runner. The build needs the spinifex source +
|
||||
# network (runner only); the import needs predastore/NATS (VM only), so
|
||||
# build on the runner then ship the raw and import on the VM.
|
||||
- name: Build eks-node AMI (libguestfs, runner)
|
||||
if: contains(env.SUITES, 'e2e-eks')
|
||||
working-directory: spinifex
|
||||
run: |
|
||||
set -euo pipefail
|
||||
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
|
||||
banner "build eks-node AMI (libguestfs — no host devices)"
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y --no-install-recommends libguestfs-tools qemu-utils
|
||||
# Debian/Ubuntu ship /boot/vmlinuz-* mode 0600; the libguestfs
|
||||
# appliance needs to read a host kernel. direct backend = no libvirt.
|
||||
export LIBGUESTFS_BACKEND=direct
|
||||
sudo chmod 0644 /boot/vmlinuz-* || true
|
||||
group_open "make build-eks-node-image"
|
||||
make build-eks-node-image
|
||||
group_close
|
||||
ok "raw: $(du -h /tmp/eks-node-image-build/eks-node-alpine.raw | cut -f1)"
|
||||
|
||||
- name: Import eks-node AMI on VM
|
||||
if: contains(env.SUITES, 'e2e-eks')
|
||||
run: |
|
||||
set -euo pipefail
|
||||
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
|
||||
# Pull import args straight from the manifest so they never drift.
|
||||
eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
|
||||
spinifex/scripts/images/eks-node/manifest.conf)"
|
||||
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
|
||||
VM=tf-user@${{ steps.vm.outputs.wan_ip }}
|
||||
banner "import eks-node AMI to VM image store"
|
||||
scp "${SSH_OPTS[@]}" \
|
||||
/tmp/eks-node-image-build/eks-node-alpine.raw \
|
||||
"$VM":/tmp/eks-node-alpine.raw
|
||||
ssh "${SSH_OPTS[@]}" "$VM" \
|
||||
"spx admin images import --file /tmp/eks-node-alpine.raw \
|
||||
--distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
|
||||
--boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}"
|
||||
ok "eks-node AMI imported"
|
||||
|
||||
# The eks data-plane subtest (TestEKS/KubectlGetNodes) drives the full
|
||||
# `aws eks get-token` → kubectl → apiserver → token-webhook → gateway
|
||||
# broker → STS verify path. The e2e binaries run ON the VM, so kubectl
|
||||
# must be on the VM's PATH or harness.NewKubectl t.Skips the subtest.
|
||||
# Pin to the k3s server version (setup.sh K3S_VERSION) — kubectl is
|
||||
# ±1-minor skew tolerant; matching is simplest. Checksum-verified.
|
||||
- name: Install kubectl on VM
|
||||
if: contains(env.SUITES, 'e2e-eks')
|
||||
run: |
|
||||
set -euo pipefail
|
||||
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
|
||||
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
|
||||
VM=tf-user@${{ steps.vm.outputs.wan_ip }}
|
||||
banner "install kubectl on VM (eks data-plane subtest)"
|
||||
ssh "${SSH_OPTS[@]}" "$VM" 'bash -s' <<'EOF'
|
||||
set -euo pipefail
|
||||
KVER=v1.32.5
|
||||
case "$(uname -m)" in
|
||||
x86_64) ARCH=amd64 ;;
|
||||
aarch64) ARCH=arm64 ;;
|
||||
*) echo "unsupported arch $(uname -m)" >&2; exit 1 ;;
|
||||
esac
|
||||
cd /tmp
|
||||
curl -fsSLO "https://dl.k8s.io/release/${KVER}/bin/linux/${ARCH}/kubectl"
|
||||
curl -fsSLO "https://dl.k8s.io/release/${KVER}/bin/linux/${ARCH}/kubectl.sha256"
|
||||
echo "$(cat kubectl.sha256) kubectl" | sha256sum -c -
|
||||
sudo install -m 0755 kubectl /usr/local/bin/kubectl
|
||||
kubectl version --client
|
||||
EOF
|
||||
ok "kubectl installed on VM"
|
||||
|
||||
- name: Build E2E test binaries on runner
|
||||
working-directory: spinifex/tests/e2e
|
||||
run: |
|
||||
@@ -385,7 +469,15 @@ jobs:
|
||||
true
|
||||
|
||||
- name: Clean Workspace
|
||||
run: sudo rm -rf "$GITHUB_WORKSPACE"/*
|
||||
run: |
|
||||
# Detach any leaked mounts under the workspace before deleting. A
|
||||
# killed image build can leave a `mount --bind /dev` under its build
|
||||
# dir; if `rm -rf` traverses it as root it wipes the host's /dev
|
||||
# (deletes /dev/null and wedges the box). umount deepest-first, then
|
||||
# forbid rm from crossing any mount boundary as a hard backstop.
|
||||
awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
|
||||
| sort -r | xargs -r -I{} sudo umount -l -R {} || true
|
||||
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
|
||||
|
||||
- uses: actions/create-github-app-token@v3
|
||||
id: app-token
|
||||
@@ -473,7 +565,12 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
|
||||
SUITES_BUILD=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(reboot|single)$' | tr '\n' ' ')
|
||||
# eks is excluded on multinode: EKS KV is a clustered (Raft) JetStream
|
||||
# stream and create-cluster needs an elected leader, but multinode
|
||||
# JetStream clustering isn't solid yet (mulga-omq/z2z/5yg). EKS is
|
||||
# validated single-node only for now; re-enable here once clustered
|
||||
# JetStream + multinode EKS are a deliberate target.
|
||||
SUITES_BUILD=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(reboot|single|eks)$' | tr '\n' ' ')
|
||||
if [ -z "$(echo "${SUITES_BUILD}" | tr -d ' ')" ]; then
|
||||
echo "no multinode-eligible suites requested (SUITES=${SUITES}); nothing to build"
|
||||
exit 0
|
||||
@@ -556,7 +653,7 @@ jobs:
|
||||
id: e2e
|
||||
run: |
|
||||
set +e
|
||||
SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(reboot|single)$' | tr '\n' ' ')
|
||||
SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(reboot|single|eks)$' | tr '\n' ' ')
|
||||
if [ -z "$(echo "${SUITES_RUN}" | tr -d ' ')" ]; then
|
||||
echo "no multinode-eligible suites requested (SUITES=${SUITES}); skipping"
|
||||
echo "rc=0" >> "$GITHUB_OUTPUT"
|
||||
@@ -793,7 +890,15 @@ jobs:
|
||||
true
|
||||
|
||||
- name: Clean Workspace
|
||||
run: sudo rm -rf "$GITHUB_WORKSPACE"/*
|
||||
run: |
|
||||
# Detach any leaked mounts under the workspace before deleting. A
|
||||
# killed image build can leave a `mount --bind /dev` under its build
|
||||
# dir; if `rm -rf` traverses it as root it wipes the host's /dev
|
||||
# (deletes /dev/null and wedges the box). umount deepest-first, then
|
||||
# forbid rm from crossing any mount boundary as a hard backstop.
|
||||
awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
|
||||
| sort -r | xargs -r -I{} sudo umount -l -R {} || true
|
||||
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
|
||||
|
||||
- uses: actions/create-github-app-token@v3
|
||||
id: app-token
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
name: Publish EKS node AMI
|
||||
|
||||
# Builds the eks-node system AMI (libguestfs, on the runner) and publishes the
|
||||
# compressed qcow2 + .sha256 sidecar to the Cloudflare R2 bucket that backs
|
||||
# iso.mulgadc.com/system-ami/. Operators then install it on any cluster with:
|
||||
# spx admin images import --name spinifex-eks-node
|
||||
#
|
||||
# Manual-only: publishing overwrites the mirror artifact, so it is a deliberate
|
||||
# action, not something every push should trigger.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: "Build + convert only; skip the R2 upload"
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: publish-eks-ami
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
name: Build + publish eks-node AMI
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
steps:
|
||||
- name: Checkout Spinifex
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
- name: Checkout Viperblock
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
repository: mulgadc/viperblock
|
||||
ref: dev
|
||||
path: viperblock
|
||||
|
||||
- name: Checkout Predastore
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
repository: mulgadc/predastore
|
||||
ref: dev
|
||||
path: predastore
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
cache-dependency-path: "spinifex/go.sum"
|
||||
go-version-file: "spinifex/go.mod"
|
||||
|
||||
- name: Setup Go Workspace
|
||||
run: go work init ./spinifex ./viperblock ./predastore
|
||||
|
||||
- name: Install build + upload tooling
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y --no-install-recommends libguestfs-tools qemu-utils
|
||||
if ! command -v aws &>/dev/null; then
|
||||
curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip
|
||||
unzip -q /tmp/awscliv2.zip -d /tmp/aws-install
|
||||
sudo /tmp/aws-install/aws/install
|
||||
rm -rf /tmp/awscliv2.zip /tmp/aws-install
|
||||
fi
|
||||
|
||||
- name: Build + publish to R2
|
||||
working-directory: spinifex
|
||||
env:
|
||||
# Map the repo's R2 secrets onto the names publish-system-image.sh reads.
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_SECRET_ACCESS_KEY }}
|
||||
R2_ENDPOINT: ${{ secrets.CF_R2_ENDPOINT_URL }}
|
||||
R2_BUCKET: spinifex-iso
|
||||
DRY_RUN: ${{ inputs.dry_run }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# libguestfs needs to read a host kernel; Ubuntu ships it mode 0600.
|
||||
# direct backend = no libvirt on the runner.
|
||||
export LIBGUESTFS_BACKEND=direct
|
||||
sudo chmod 0644 /boot/vmlinuz-* || true
|
||||
FLAGS=(--build)
|
||||
if [[ "${DRY_RUN}" == "true" ]]; then
|
||||
FLAGS+=(--dry-run)
|
||||
fi
|
||||
./scripts/publish-system-image.sh \
|
||||
scripts/images/eks-node/manifest.conf "${FLAGS[@]}"
|
||||
@@ -131,7 +131,16 @@ jobs:
|
||||
REPO: ${{ github.repository }}
|
||||
steps:
|
||||
- name: Clean Workspace
|
||||
run: rm -rf "$GITHUB_WORKSPACE"/*
|
||||
run: |
|
||||
# This runner (ci-single) is shared with the iso job, which mounts a
|
||||
# kernel-global devtmpfs at <workspace>/mulga/scripts/iso-builder/.cache/rootfs/dev.
|
||||
# If that job is killed its EXIT-trap unmount never runs and the bind
|
||||
# leaks; a plain rm -rf then traverses it and deletes the host's /dev
|
||||
# (wipes /dev/null, wedges the box). Detach any leaked mounts under the
|
||||
# workspace first, then forbid rm from crossing a mount boundary.
|
||||
awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
|
||||
| sort -r | xargs -r -I{} sudo umount -l -R {} || true
|
||||
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
|
||||
|
||||
- uses: actions/create-github-app-token@v3
|
||||
id: app-token
|
||||
@@ -169,8 +178,36 @@ jobs:
|
||||
TARBALL=$(find "${GITHUB_WORKSPACE}/release-assets" -name "*-linux-amd64.tar.gz" | head -1)
|
||||
./bootstrap-install.sh "${SPINIFEX_CI_ENV}" "$SETUP_SH" "$TARBALL"
|
||||
|
||||
- name: Teardown
|
||||
if: always()
|
||||
working-directory: mulga/scripts/tofu-cluster
|
||||
run: |
|
||||
tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
|
||||
tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
|
||||
./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
|
||||
|
||||
# Bare-metal single-node suite on the real ISO/squashfs + PXE-boot path —
|
||||
# the only release check that exercises hardware install. Runs in parallel
|
||||
# with `verify` (both need: release). Promotion and ISO upload both hard-gate
|
||||
# on this passing.
|
||||
baremetal:
|
||||
needs: release
|
||||
permissions:
|
||||
contents: read
|
||||
statuses: write
|
||||
uses: ./.github/workflows/e2e-baremetal.yml
|
||||
secrets: inherit
|
||||
|
||||
# Promote the draft only after BOTH the tofu install (`verify`) and the
|
||||
# bare-metal hardware boot (`baremetal`) pass — runs only if both succeed.
|
||||
promote:
|
||||
needs: [ verify, baremetal ]
|
||||
runs-on: ubuntu-24.04
|
||||
env:
|
||||
TAG: ${{ github.ref_name }}
|
||||
REPO: ${{ github.repository }}
|
||||
steps:
|
||||
- name: Promote Release
|
||||
if: success()
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
@@ -182,29 +219,31 @@ jobs:
|
||||
echo "Release ${TAG} promoted to latest"
|
||||
fi
|
||||
|
||||
# Annotate the (still-draft) release if either gate failed.
|
||||
mark-failed:
|
||||
needs: [ verify, baremetal ]
|
||||
if: failure()
|
||||
runs-on: ubuntu-24.04
|
||||
env:
|
||||
TAG: ${{ github.ref_name }}
|
||||
REPO: ${{ github.repository }}
|
||||
steps:
|
||||
- name: Mark Release Failed
|
||||
if: failure()
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
EXISTING_BODY=$(gh release view "$TAG" --repo "$REPO" --json body -q .body)
|
||||
NOTES_FILE=$(mktemp)
|
||||
printf 'Verification failed — see workflow run for details.\n\n%s' "$EXISTING_BODY" > "$NOTES_FILE"
|
||||
printf 'Verification failed (tofu install or bare-metal boot) — see workflow run for details.\n\n%s' "$EXISTING_BODY" > "$NOTES_FILE"
|
||||
gh release edit "$TAG" --repo "$REPO" --notes-file "$NOTES_FILE"
|
||||
rm -f "$NOTES_FILE"
|
||||
|
||||
- name: Teardown
|
||||
if: always()
|
||||
working-directory: mulga/scripts/tofu-cluster
|
||||
run: |
|
||||
tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
|
||||
tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
|
||||
./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
|
||||
|
||||
iso:
|
||||
needs: release
|
||||
runs-on: [ self-hosted, ci-single ]
|
||||
timeout-minutes: 90
|
||||
# ISO build (~60-90m) overlaps the parallel baremetal run (~120m); the
|
||||
# upload gate then waits for baremetal to finish, so the job must outlast it.
|
||||
timeout-minutes: 180
|
||||
steps:
|
||||
- name: Clean Workspace
|
||||
run: |
|
||||
@@ -216,7 +255,7 @@ jobs:
|
||||
sudo umount -l "${ROOTFS}/dev" 2>/dev/null || true
|
||||
sudo umount -l "${ROOTFS}/sys" 2>/dev/null || true
|
||||
sudo umount -l "${ROOTFS}/proc" 2>/dev/null || true
|
||||
sudo rm -rf "$GITHUB_WORKSPACE"/*
|
||||
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
|
||||
|
||||
- uses: actions/create-github-app-token@v3
|
||||
id: app-token
|
||||
@@ -289,24 +328,28 @@ jobs:
|
||||
SPINIFEX_BIN="${GITHUB_WORKSPACE}/spinifex/bin/spx" \
|
||||
INSTALLER_BIN="${GITHUB_WORKSPACE}/spinifex/bin/spinifex-installer"
|
||||
|
||||
- name: Wait for verify
|
||||
- name: Wait for verify + baremetal
|
||||
id: wait-verify
|
||||
if: steps.build-iso.outcome == 'success'
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
echo "Waiting for verify job to complete..."
|
||||
# Gate the R2 upload on BOTH the tofu install (`verify`) and the
|
||||
# real-hardware boot (`baremetal`, a reusable workflow whose job shows
|
||||
# as "baremetal / ..."). Upload only if both conclude success.
|
||||
echo "Waiting for verify + baremetal jobs to complete..."
|
||||
while true; do
|
||||
CONCLUSION=$(gh api \
|
||||
"repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs" \
|
||||
--jq '.jobs[] | select(.name == "verify") | .conclusion // empty')
|
||||
if [ -n "$CONCLUSION" ]; then
|
||||
echo "Verify conclusion: $CONCLUSION"
|
||||
[ "$CONCLUSION" = "success" ] && exit 0
|
||||
JOBS=$(gh api \
|
||||
"repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs" --paginate)
|
||||
VERIFY=$(echo "$JOBS" | jq -r '.jobs[] | select(.name == "verify") | .conclusion // empty')
|
||||
BAREMETAL=$(echo "$JOBS" | jq -r '.jobs[] | select(.name | startswith("baremetal")) | .conclusion // empty')
|
||||
if [ -n "$VERIFY" ] && [ -n "$BAREMETAL" ]; then
|
||||
echo "verify=$VERIFY baremetal=$BAREMETAL"
|
||||
{ [ "$VERIFY" = "success" ] && [ "$BAREMETAL" = "success" ]; } && exit 0
|
||||
exit 1
|
||||
fi
|
||||
echo "Verify still running, retrying in 30s..."
|
||||
echo "Still waiting (verify=${VERIFY:-pending} baremetal=${BAREMETAL:-pending}), retry 30s..."
|
||||
sleep 30
|
||||
done
|
||||
|
||||
|
||||
@@ -92,6 +92,9 @@ build-eks-node-image: ## Build the unified eks-node AMI (K3s server+agent; role
|
||||
import-eks-node-image: ## Build + register the eks-node AMI (requires a running cluster)
|
||||
$(MAKE) build-system-image IMAGE=eks-node IMPORT=1
|
||||
|
||||
publish-eks-node-image: ## Build + publish the eks-node AMI to Cloudflare R2 (needs R2_ENDPOINT + AWS_* env)
|
||||
./scripts/publish-system-image.sh scripts/images/eks-node/manifest.conf --build
|
||||
|
||||
MICROVM_OUT_DIR := build/microvm
|
||||
MICROVM_ARTIFACTS := $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz
|
||||
MICROVM_INPUTS := scripts/build-microvm-image.sh $(MICROVM_OUT_DIR)/init.sh $(MICROVM_OUT_DIR)/inittab bin/lb-agent
|
||||
@@ -350,7 +353,7 @@ ansible-cluster-bootstrap:
|
||||
$(if $(POOL),-e cluster_external_pool=$(POOL),) \
|
||||
$(_ANSIBLE_EXTRA)
|
||||
|
||||
.PHONY: build build-ui build-installer build-lb-agent build-system-image build-eks-node-image import-eks-node-image build-microvm-image install-microvm go_build go_run preflight test test-cover test-race diff-coverage bench run test-actions test-harness manifest-check diff-coverage bench run \
|
||||
.PHONY: build build-ui build-installer build-lb-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-microvm-image install-microvm go_build go_run preflight test test-cover test-race diff-coverage bench run test-actions test-harness manifest-check diff-coverage bench run \
|
||||
deploy reinstall clean \
|
||||
install-system install-go install-aws quickinstall \
|
||||
lint fix govulncheck \
|
||||
|
||||
+45
-7
@@ -6,20 +6,58 @@ mount -t devtmpfs dev /dev
|
||||
# virtio_mmio is built-in (CONFIG_VIRTIO_MMIO=y) so virtio-net devices are
|
||||
# already enumerated by the time PID 1 runs — no modprobe needed.
|
||||
# qemu_fw_cfg and virtio_net are =m in stock Alpine linux-virt, so load them.
|
||||
modprobe qemu_fw_cfg 2>/dev/null || modprobe fw_cfg_sysfs 2>/dev/null || true
|
||||
modprobe virtio_net 2>/dev/null || true
|
||||
#
|
||||
# modprobe needs a usable modules.dep; when the image is built on a host whose
|
||||
# depmod can't produce one, busybox modprobe silently no-ops and the module
|
||||
# never loads — fw_cfg then never appears and init panics. insmod by explicit
|
||||
# path needs no modules.dep, so fall back to it. Handles .ko and .ko.gz.
|
||||
load_mod() {
|
||||
modprobe "$1" 2>/dev/null && return 0
|
||||
for ko in $(find /lib/modules -name "$1.ko" -o -name "$1.ko.gz" 2>/dev/null); do
|
||||
insmod "$ko" 2>/dev/null && return 0
|
||||
case "$ko" in
|
||||
*.gz) gunzip -c "$ko" >/tmp/mod.ko 2>/dev/null \
|
||||
&& insmod /tmp/mod.ko 2>/dev/null && return 0 ;;
|
||||
esac
|
||||
done
|
||||
return 1
|
||||
}
|
||||
load_mod qemu_fw_cfg || load_mod fw_cfg_sysfs || true
|
||||
# virtio_net depends on net_failover, which depends on failover. The load_mod
|
||||
# insmod fallback loads a single .ko with no dependency resolution, and
|
||||
# modules.dep is absent when the image is built on a kmod-less host (depmod
|
||||
# skipped), so modprobe can't resolve the chain either. Load it explicitly
|
||||
# bottom-up so the NIC comes up regardless of how the image was built —
|
||||
# without this the agent has no network and the LB hangs in provisioning.
|
||||
load_mod failover || true
|
||||
load_mod net_failover || true
|
||||
load_mod virtio_net || true
|
||||
|
||||
for i in 1 2 3 4 5; do
|
||||
[ -f /sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw ] && break
|
||||
sleep 0.02
|
||||
NETCFG=/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw
|
||||
i=0
|
||||
while [ "$i" -lt 25 ]; do
|
||||
[ -f "$NETCFG" ] && break
|
||||
sleep 0.04
|
||||
i=$((i + 1))
|
||||
done
|
||||
|
||||
if [ ! -f /sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw ]; then
|
||||
if [ ! -f "$NETCFG" ]; then
|
||||
# Dump host/guest fw_cfg state to the serial console before init exits so a
|
||||
# "netcfg not available" panic is diagnosable from the captured console log
|
||||
# (module-not-loaded vs qemu-not-exposing-by_name vs wrong machine type).
|
||||
echo "[init] FATAL: fw_cfg netcfg not available" >&2
|
||||
echo "[init] dmi: vendor=$(cat /sys/class/dmi/id/sys_vendor 2>/dev/null) product=$(cat /sys/class/dmi/id/product_name 2>/dev/null) bios=$(cat /sys/class/dmi/id/bios_version 2>/dev/null)" >&2
|
||||
echo "[init] qemu_fw_cfg dir present: $([ -d /sys/firmware/qemu_fw_cfg ] && echo yes || echo NO)" >&2
|
||||
echo "[init] modules: $(lsmod 2>/dev/null | awk 'NR>1{print $1}' | tr '\n' ',')" >&2
|
||||
echo "[init] by_name listing:" >&2
|
||||
ls -la /sys/firmware/qemu_fw_cfg/by_name/ 2>&1 | sed 's/^/[init] /' >&2
|
||||
ls -la /sys/firmware/qemu_fw_cfg/by_name/opt/ 2>&1 | sed 's/^/[init] /' >&2
|
||||
echo "[init] dmesg fw_cfg:" >&2
|
||||
dmesg 2>/dev/null | grep -i fw_cfg | sed 's/^/[init] /' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
. /sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw
|
||||
. "$NETCFG"
|
||||
|
||||
for n in 0 1 2 3 4 5; do
|
||||
eval mac=\$NIC${n}_MAC
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
// eks-gateway-publish runs inside the EKS K3s control-plane VM and relays a
|
||||
// single publication to the host through the AWS gateway, instead of dialing
|
||||
// core NATS directly.
|
||||
//
|
||||
// It reads a JSON payload on stdin, wraps it as
|
||||
// {accountId, channel, kind, payload}, SigV4-signs (service "eks") an HTTPS
|
||||
// POST to {gateway}/clusters/{cluster}/internal-publish, and retries with
|
||||
// backoff until the gateway returns 2xx or the attempt budget is exhausted —
|
||||
// so a degraded link surfaces as a non-zero exit rather than a silently
|
||||
// dropped message (the failure mode of fire-and-forget `nats pub`).
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
// echo '{"token":"..."}' | eks-gateway-publish -channel bootstrap -kind k3s-bootstrap-token
|
||||
// kubectl ... | eks-gateway-publish -channel state
|
||||
//
|
||||
// Flags default to environment variables seeded by cloud-init:
|
||||
// EKS_GATEWAY_URL, EKS_GATEWAY_CA, EKS_ACCESS_KEY, EKS_SECRET_KEY, EKS_REGION,
|
||||
// EKS_ACCOUNT_ID, EKS_CLUSTER_NAME.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/eksgw"
|
||||
|
||||
_ "github.com/mulgadc/spinifex/internal/fipsboot"
|
||||
)
|
||||
|
||||
const (
|
||||
// Retry budget: the control plane reaches readiness before this runs, so a
|
||||
// failing POST means a degraded link, not a cold start. Bounded so a stuck
|
||||
// boot still terminates the OpenRC service.
|
||||
maxAttempts = 30
|
||||
retryDelay = 5 * time.Second
|
||||
)
|
||||
|
||||
type publishBody struct {
|
||||
AccountID string `json:"accountId"`
|
||||
Channel string `json:"channel"`
|
||||
Kind string `json:"kind"`
|
||||
Payload json.RawMessage `json:"payload"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
var (
|
||||
gatewayURL string
|
||||
gatewayCA string
|
||||
accessKey string
|
||||
secretKey string
|
||||
region string
|
||||
accountID string
|
||||
clusterName string
|
||||
channel string
|
||||
kind string
|
||||
)
|
||||
|
||||
flag.StringVar(&gatewayURL, "gateway", os.Getenv("EKS_GATEWAY_URL"), "Gateway URL (e.g. https://10.15.8.1:9999)")
|
||||
flag.StringVar(&gatewayCA, "gateway-ca", os.Getenv("EKS_GATEWAY_CA"), "Path to gateway TLS CA PEM (optional; falls back to system trust)")
|
||||
flag.StringVar(&accessKey, "access-key", os.Getenv("EKS_ACCESS_KEY"), "AWS access key ID")
|
||||
flag.StringVar(&secretKey, "secret-key", os.Getenv("EKS_SECRET_KEY"), "AWS secret access key")
|
||||
flag.StringVar(®ion, "region", os.Getenv("EKS_REGION"), "AWS region for SigV4 signing")
|
||||
flag.StringVar(&accountID, "account-id", os.Getenv("EKS_ACCOUNT_ID"), "Cluster account ID")
|
||||
flag.StringVar(&clusterName, "cluster", os.Getenv("EKS_CLUSTER_NAME"), "Cluster name")
|
||||
flag.StringVar(&channel, "channel", "", "Publish channel: bootstrap|state")
|
||||
flag.StringVar(&kind, "kind", "", "Bootstrap subject kind (bootstrap channel only)")
|
||||
flag.Parse()
|
||||
|
||||
switch {
|
||||
case accountID == "":
|
||||
fatal("--account-id is required (or set EKS_ACCOUNT_ID)")
|
||||
case clusterName == "":
|
||||
fatal("--cluster is required (or set EKS_CLUSTER_NAME)")
|
||||
case channel != "bootstrap" && channel != "state":
|
||||
fatal("--channel must be bootstrap or state")
|
||||
case channel == "bootstrap" && kind == "":
|
||||
fatal("--kind is required for the bootstrap channel")
|
||||
}
|
||||
|
||||
payload, err := io.ReadAll(os.Stdin)
|
||||
if err != nil {
|
||||
fatal(fmt.Sprintf("read stdin payload: %v", err))
|
||||
}
|
||||
if len(bytes.TrimSpace(payload)) == 0 {
|
||||
fatal("empty stdin payload")
|
||||
}
|
||||
if !json.Valid(payload) {
|
||||
fatal("stdin payload is not valid JSON")
|
||||
}
|
||||
|
||||
body, err := json.Marshal(publishBody{
|
||||
AccountID: accountID,
|
||||
Channel: channel,
|
||||
Kind: kind,
|
||||
Payload: json.RawMessage(payload),
|
||||
})
|
||||
if err != nil {
|
||||
fatal(fmt.Sprintf("marshal request body: %v", err))
|
||||
}
|
||||
|
||||
client, err := eksgw.New(gatewayURL, gatewayCA, accessKey, secretKey, region)
|
||||
if err != nil {
|
||||
fatal(fmt.Sprintf("build gateway client: %v", err))
|
||||
}
|
||||
path := "/clusters/" + clusterName + "/internal-publish"
|
||||
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= maxAttempts; attempt++ {
|
||||
if _, err := client.Post(path, body); err != nil {
|
||||
lastErr = err
|
||||
slog.Warn("eks-gateway-publish: attempt failed",
|
||||
"channel", channel, "kind", kind, "attempt", attempt, "err", err)
|
||||
if attempt < maxAttempts {
|
||||
time.Sleep(retryDelay)
|
||||
}
|
||||
continue
|
||||
}
|
||||
slog.Info("eks-gateway-publish: published", "channel", channel, "kind", kind)
|
||||
return
|
||||
}
|
||||
fatal(fmt.Sprintf("publish failed after %d attempts: %v", maxAttempts, lastErr))
|
||||
}
|
||||
|
||||
func fatal(msg string) {
|
||||
slog.Error("eks-gateway-publish: " + msg)
|
||||
os.Exit(1)
|
||||
}
|
||||
+79
-109
@@ -1,8 +1,10 @@
|
||||
// eks-token-webhook is the K8s TokenReview webhook authenticator baked into
|
||||
// the eks-server AMI. It decodes the SigV4-presigned GetCallerIdentity URL
|
||||
// produced by `aws eks get-token`, calls Mulga STS over NATS (which enforces
|
||||
// the x-k8s-aws-id cross-cluster pin against this cluster's name), and resolves
|
||||
// the principal ARN to a TokenReview response via the cluster's AccessEntry KV.
|
||||
// the eks-server AMI. Per kubectl call the kube-apiserver POSTs the bearer
|
||||
// token (produced by `aws eks get-token`); the webhook relays it through the
|
||||
// AWS gateway broker (SigV4 HTTPS POST to /clusters/{name}/token-review), which
|
||||
// host-side runs the STS verify (cross-cluster x-k8s-aws-id pin) + AccessEntry
|
||||
// KV lookup and returns the resolved identity. The webhook never speaks core
|
||||
// NATS — same gateway-broker model as ELBv2's lb-agent and eks-gateway-publish.
|
||||
//
|
||||
// It binds 127.0.0.1 only; the kube-apiserver is the sole client (loopback).
|
||||
// On startup it writes the apiserver webhook kubeconfig (with its self-signed
|
||||
@@ -20,46 +22,48 @@ import (
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/eksgw"
|
||||
handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
|
||||
"github.com/mulgadc/spinifex/spinifex/utils"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// config is the webhook's runtime configuration, sourced from the first-boot
|
||||
// env (/etc/spinifex-eks/first-boot.env) cloud-init seeds onto the VM.
|
||||
type config struct {
|
||||
addr string
|
||||
natsURL string
|
||||
natsToken string
|
||||
natsCA string
|
||||
gatewayURL string
|
||||
gatewayCA string
|
||||
accessKey string
|
||||
secretKey string
|
||||
region string
|
||||
accountID string
|
||||
clusterName string
|
||||
certPath string
|
||||
keyPath string
|
||||
kubeconfig string
|
||||
verifyTO time.Duration
|
||||
}
|
||||
|
||||
func loadConfig() (config, error) {
|
||||
c := config{
|
||||
addr: flagAddr,
|
||||
natsURL: os.Getenv("SPINIFEX_NATS_URL"),
|
||||
natsToken: os.Getenv("SPINIFEX_NATS_TOKEN"),
|
||||
natsCA: os.Getenv("SPINIFEX_NATS_CA"),
|
||||
gatewayURL: os.Getenv("EKS_GATEWAY_URL"),
|
||||
gatewayCA: os.Getenv("EKS_GATEWAY_CA"),
|
||||
accessKey: os.Getenv("EKS_ACCESS_KEY"),
|
||||
secretKey: os.Getenv("EKS_SECRET_KEY"),
|
||||
region: os.Getenv("EKS_REGION"),
|
||||
accountID: os.Getenv("EKS_ACCOUNT_ID"),
|
||||
clusterName: os.Getenv("EKS_CLUSTER_NAME"),
|
||||
certPath: envOr("EKS_WEBHOOK_CERT", "/etc/spinifex-eks/token-webhook.crt"),
|
||||
keyPath: envOr("EKS_WEBHOOK_KEY", "/etc/spinifex-eks/token-webhook.key"),
|
||||
kubeconfig: envOr("EKS_WEBHOOK_KUBECONFIG", "/etc/spinifex-eks/token-webhook.kubeconfig"),
|
||||
verifyTO: 5 * time.Second,
|
||||
}
|
||||
if c.natsURL == "" {
|
||||
return c, errors.New("SPINIFEX_NATS_URL not set")
|
||||
}
|
||||
if c.accountID == "" {
|
||||
switch {
|
||||
case c.gatewayURL == "":
|
||||
return c, errors.New("EKS_GATEWAY_URL not set")
|
||||
case c.accessKey == "" || c.secretKey == "":
|
||||
return c, errors.New("EKS_ACCESS_KEY / EKS_SECRET_KEY not set")
|
||||
case c.accountID == "":
|
||||
return c, errors.New("EKS_ACCOUNT_ID not set")
|
||||
}
|
||||
if c.clusterName == "" {
|
||||
case c.clusterName == "":
|
||||
return c, errors.New("EKS_CLUSTER_NAME not set")
|
||||
}
|
||||
return c, nil
|
||||
@@ -103,22 +107,15 @@ func run(cfg config) error {
|
||||
return fmt.Errorf("write apiserver kubeconfig: %w", err)
|
||||
}
|
||||
|
||||
nc, err := utils.ConnectNATSWithRetry(cfg.natsURL, cfg.natsToken, cfg.natsCA)
|
||||
client, err := eksgw.New(cfg.gatewayURL, cfg.gatewayCA, cfg.accessKey, cfg.secretKey, cfg.region)
|
||||
if err != nil {
|
||||
return fmt.Errorf("connect NATS: %w", err)
|
||||
}
|
||||
defer nc.Close()
|
||||
|
||||
kv, err := openAccountKV(nc, cfg.accountID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open account KV: %w", err)
|
||||
return fmt.Errorf("build gateway client: %w", err)
|
||||
}
|
||||
|
||||
authr := &authenticator{
|
||||
nc: nc,
|
||||
kv: kv,
|
||||
accountID: cfg.accountID,
|
||||
clusterName: cfg.clusterName,
|
||||
verifyTO: cfg.verifyTO,
|
||||
review: gatewayReviewer(client, cfg.clusterName, cfg.accountID),
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
@@ -136,50 +133,42 @@ func run(cfg config) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// openAccountKV attaches to the per-account EKS KV bucket. The bucket already
|
||||
// exists by the time a control-plane VM is running (CreateCluster created it),
|
||||
// but JetStream may briefly lag at boot, so retry a few times.
|
||||
func openAccountKV(nc *nats.Conn, accountID string) (nats.KeyValue, error) {
|
||||
js, err := nc.JetStream()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("jetstream: %w", err)
|
||||
}
|
||||
bucket := handlers_eks.AccountBucketName(accountID)
|
||||
var lastErr error
|
||||
for range 30 {
|
||||
kv, kvErr := js.KeyValue(bucket)
|
||||
if kvErr == nil {
|
||||
return kv, nil
|
||||
// tokenReviewRequest is the body POSTed to /clusters/{name}/token-review. The
|
||||
// webhook holds system SigV4 creds (system account, not the cluster account),
|
||||
// so accountId names the cluster account explicitly — same as PublishInternal.
|
||||
type tokenReviewRequest struct {
|
||||
AccountID string `json:"accountId"`
|
||||
Token string `json:"token"`
|
||||
}
|
||||
|
||||
// gatewayReviewer returns the per-call relay: SigV4-POST {accountId,token} to
|
||||
// the gateway and decode the resolved identity. A non-2xx / transport error is
|
||||
// surfaced (handle() turns it into a 5xx so the apiserver retries) rather than a
|
||||
// silent deny, distinguishing "webhook broker down" from "token rejected".
|
||||
func gatewayReviewer(client *eksgw.Client, clusterName, accountID string) func(string) (handlers_eks.WebhookTokenReviewResult, error) {
|
||||
path := "/clusters/" + clusterName + "/token-review"
|
||||
return func(token string) (handlers_eks.WebhookTokenReviewResult, error) {
|
||||
body, err := json.Marshal(tokenReviewRequest{AccountID: accountID, Token: token})
|
||||
if err != nil {
|
||||
return handlers_eks.WebhookTokenReviewResult{}, fmt.Errorf("marshal request: %w", err)
|
||||
}
|
||||
lastErr = kvErr
|
||||
time.Sleep(2 * time.Second)
|
||||
respBody, err := client.Post(path, body)
|
||||
if err != nil {
|
||||
return handlers_eks.WebhookTokenReviewResult{}, err
|
||||
}
|
||||
var res handlers_eks.WebhookTokenReviewResult
|
||||
if err := json.Unmarshal(respBody, &res); err != nil {
|
||||
return handlers_eks.WebhookTokenReviewResult{}, fmt.Errorf("decode response: %w", err)
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
return nil, fmt.Errorf("kv bucket %s not available: %w", bucket, lastErr)
|
||||
}
|
||||
|
||||
// authenticator resolves a TokenReview against STS (over NATS) + AccessEntry KV.
|
||||
// authenticator resolves a TokenReview by relaying to the gateway broker.
|
||||
type authenticator struct {
|
||||
nc *nats.Conn
|
||||
kv nats.KeyValue
|
||||
accountID string
|
||||
clusterName string
|
||||
verifyTO time.Duration
|
||||
}
|
||||
|
||||
// verify resolves a presigned get-token URL into the caller principal via the
|
||||
// awsgw-hosted STS verify subject, binding the signature to this cluster name.
|
||||
func (a *authenticator) verify(presignedURL string) (*handlers_eks.TokenVerifyResponse, error) {
|
||||
req := handlers_eks.TokenVerifyRequest{
|
||||
PresignedURL: presignedURL,
|
||||
ClusterName: a.clusterName,
|
||||
}
|
||||
// accountID header is unused by the verify responder; pass empty.
|
||||
return utils.NATSRequest[handlers_eks.TokenVerifyResponse](
|
||||
a.nc, handlers_eks.TokenVerifySubject, req, a.verifyTO, "")
|
||||
}
|
||||
|
||||
// lookup reads the AccessEntry for a principal ARN from the cluster KV.
|
||||
func (a *authenticator) lookup(principalARN string) (*handlers_eks.AccessEntryRecord, error) {
|
||||
return handlers_eks.GetAccessEntryRecord(a.kv, a.clusterName, principalARN)
|
||||
review func(token string) (handlers_eks.WebhookTokenReviewResult, error)
|
||||
}
|
||||
|
||||
func (a *authenticator) handle(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -193,7 +182,27 @@ func (a *authenticator) handle(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
status := authenticate(review.Spec.Token, a.verify, a.lookup)
|
||||
res, err := a.review(review.Spec.Token)
|
||||
if err != nil {
|
||||
// Broker fault (gateway down, transport error): a 5xx tells the apiserver
|
||||
// to retry rather than treating a transient outage as a hard deny.
|
||||
slog.Error("TokenReview relay failed", "cluster", a.clusterName, "err", err)
|
||||
http.Error(w, "token review unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
status := tokenReviewStatus{Authenticated: res.Authenticated}
|
||||
if res.Authenticated {
|
||||
status.User = userInfo{
|
||||
Username: res.Username,
|
||||
UID: res.UID,
|
||||
Groups: res.Groups,
|
||||
}
|
||||
// Confirm the token for the audiences the apiserver asked about; an empty
|
||||
// status.audiences is rejected when --api-audiences is configured.
|
||||
status.Audiences = review.Spec.Audiences
|
||||
}
|
||||
slog.Info("TokenReview decision", "authenticated", status.Authenticated, "username", status.User.Username, "audiences", status.Audiences)
|
||||
|
||||
resp := tokenReview{
|
||||
APIVersion: "authentication.k8s.io/v1",
|
||||
@@ -206,45 +215,6 @@ func (a *authenticator) handle(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
// authenticate is the pure decision core: decode token → STS verify → KV
|
||||
// AccessEntry lookup → TokenReview status. Any failure yields an unauthenticated
|
||||
// status (never a 5xx) so a forged/unknown token is a clean K8s 401, not a
|
||||
// webhook outage. verify and lookup are injected so the logic is unit-testable.
|
||||
func authenticate(
|
||||
token string,
|
||||
verify func(presignedURL string) (*handlers_eks.TokenVerifyResponse, error),
|
||||
lookup func(principalARN string) (*handlers_eks.AccessEntryRecord, error),
|
||||
) tokenReviewStatus {
|
||||
presignedURL, err := handlers_eks.DecodeGetToken(token)
|
||||
if err != nil {
|
||||
return tokenReviewStatus{Authenticated: false}
|
||||
}
|
||||
ident, err := verify(presignedURL)
|
||||
if err != nil {
|
||||
slog.Debug("token verify rejected", "err", err)
|
||||
return tokenReviewStatus{Authenticated: false}
|
||||
}
|
||||
rec, err := lookup(ident.ARN)
|
||||
if err != nil {
|
||||
// No AccessEntry for an otherwise-valid IAM principal: authenticated to
|
||||
// AWS, but not granted any K8s identity on this cluster.
|
||||
slog.Debug("no access entry for principal", "arn", ident.ARN, "err", err)
|
||||
return tokenReviewStatus{Authenticated: false}
|
||||
}
|
||||
uid := ident.UserID
|
||||
if uid == "" {
|
||||
uid = ident.ARN
|
||||
}
|
||||
return tokenReviewStatus{
|
||||
Authenticated: true,
|
||||
User: userInfo{
|
||||
Username: rec.KubernetesUsername,
|
||||
UID: uid,
|
||||
Groups: rec.KubernetesGroups,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// newServer builds the loopback TLS server. Split out so tests can construct it
|
||||
// without binding a port.
|
||||
func newServer(addr string, handler http.Handler, cert tlsCertificate) *http.Server {
|
||||
|
||||
@@ -2,9 +2,13 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
|
||||
@@ -12,83 +16,79 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func validToken(url string) string {
|
||||
return "k8s-aws-v1." + base64.RawURLEncoding.EncodeToString([]byte(url))
|
||||
// postReview drives authenticator.handle with a TokenReview carrying token and
|
||||
// audiences, returning the decoded response status and HTTP code.
|
||||
func postReview(t *testing.T, a *authenticator, token string, audiences []string) (tokenReviewStatus, int) {
|
||||
t.Helper()
|
||||
body, err := json.Marshal(tokenReview{
|
||||
APIVersion: "authentication.k8s.io/v1",
|
||||
Kind: "TokenReview",
|
||||
Spec: tokenReviewSpec{Token: token, Audiences: audiences},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/authenticate", strings.NewReader(string(body)))
|
||||
w := httptest.NewRecorder()
|
||||
a.handle(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
return tokenReviewStatus{}, w.Code
|
||||
}
|
||||
var resp tokenReview
|
||||
require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp))
|
||||
return resp.Status, w.Code
|
||||
}
|
||||
|
||||
const testARN = "arn:aws:iam::111122223333:role/admin"
|
||||
func TestHandle_GrantsAndStampsAudiences(t *testing.T) {
|
||||
a := &authenticator{
|
||||
accountID: "111122223333",
|
||||
clusterName: "alpha",
|
||||
review: func(string) (handlers_eks.WebhookTokenReviewResult, error) {
|
||||
return handlers_eks.WebhookTokenReviewResult{
|
||||
Authenticated: true,
|
||||
Username: "arn:aws:iam::111122223333:role/admin",
|
||||
UID: "AROAEXAMPLE:session",
|
||||
Groups: []string{"system:masters"},
|
||||
}, nil
|
||||
},
|
||||
}
|
||||
|
||||
func okVerify(_ string) (*handlers_eks.TokenVerifyResponse, error) {
|
||||
return &handlers_eks.TokenVerifyResponse{
|
||||
AccountID: "111122223333",
|
||||
ARN: testARN,
|
||||
UserID: "AROAEXAMPLE:session",
|
||||
PrincipalType: "AssumedRole",
|
||||
}, nil
|
||||
status, code := postReview(t, a, "tok", []string{"https://kubernetes.default.svc"})
|
||||
require.Equal(t, http.StatusOK, code)
|
||||
require.True(t, status.Authenticated)
|
||||
assert.Equal(t, "arn:aws:iam::111122223333:role/admin", status.User.Username)
|
||||
assert.Equal(t, "AROAEXAMPLE:session", status.User.UID)
|
||||
assert.Equal(t, []string{"system:masters"}, status.User.Groups)
|
||||
assert.Equal(t, []string{"https://kubernetes.default.svc"}, status.Audiences)
|
||||
}
|
||||
|
||||
func TestAuthenticate_GrantsWhenEntryExists(t *testing.T) {
|
||||
lookup := func(arn string) (*handlers_eks.AccessEntryRecord, error) {
|
||||
assert.Equal(t, testARN, arn)
|
||||
return &handlers_eks.AccessEntryRecord{
|
||||
KubernetesUsername: testARN,
|
||||
KubernetesGroups: []string{"system:masters"},
|
||||
}, nil
|
||||
func TestHandle_DeniesWithoutStampingAudiences(t *testing.T) {
|
||||
a := &authenticator{
|
||||
clusterName: "alpha",
|
||||
review: func(string) (handlers_eks.WebhookTokenReviewResult, error) {
|
||||
return handlers_eks.WebhookTokenReviewResult{Authenticated: false}, nil
|
||||
},
|
||||
}
|
||||
|
||||
st := authenticate(validToken("https://sts.amazonaws.com/?Action=GetCallerIdentity"), okVerify, lookup)
|
||||
|
||||
require.True(t, st.Authenticated)
|
||||
assert.Equal(t, testARN, st.User.Username)
|
||||
assert.Equal(t, "AROAEXAMPLE:session", st.User.UID)
|
||||
assert.Equal(t, []string{"system:masters"}, st.User.Groups)
|
||||
status, code := postReview(t, a, "tok", []string{"https://kubernetes.default.svc"})
|
||||
require.Equal(t, http.StatusOK, code)
|
||||
assert.False(t, status.Authenticated)
|
||||
assert.Empty(t, status.User.Username)
|
||||
assert.Empty(t, status.Audiences, "unauthenticated review must not stamp audiences")
|
||||
}
|
||||
|
||||
func TestAuthenticate_DeniesMalformedToken(t *testing.T) {
|
||||
called := false
|
||||
verify := func(string) (*handlers_eks.TokenVerifyResponse, error) { called = true; return nil, nil }
|
||||
lookup := func(string) (*handlers_eks.AccessEntryRecord, error) { return nil, nil }
|
||||
|
||||
st := authenticate("not-a-k8s-aws-token", verify, lookup)
|
||||
|
||||
assert.False(t, st.Authenticated)
|
||||
assert.False(t, called, "must not call STS for a token that fails to decode")
|
||||
}
|
||||
|
||||
func TestAuthenticate_DeniesWhenVerifyFails(t *testing.T) {
|
||||
verify := func(string) (*handlers_eks.TokenVerifyResponse, error) {
|
||||
return nil, errors.New("signature mismatch")
|
||||
}
|
||||
lookup := func(string) (*handlers_eks.AccessEntryRecord, error) {
|
||||
t.Fatal("lookup must not run when verify fails")
|
||||
return nil, nil
|
||||
// A broker fault is a 5xx (retryable), not a silent deny — a transient gateway
|
||||
// outage must not look identical to a rejected token.
|
||||
func TestHandle_BrokerFaultReturns5xx(t *testing.T) {
|
||||
a := &authenticator{
|
||||
clusterName: "alpha",
|
||||
review: func(string) (handlers_eks.WebhookTokenReviewResult, error) {
|
||||
return handlers_eks.WebhookTokenReviewResult{}, errors.New("gateway returned 503")
|
||||
},
|
||||
}
|
||||
|
||||
st := authenticate(validToken("https://sts/?x=1"), verify, lookup)
|
||||
assert.False(t, st.Authenticated)
|
||||
}
|
||||
|
||||
func TestAuthenticate_DeniesWhenNoAccessEntry(t *testing.T) {
|
||||
lookup := func(string) (*handlers_eks.AccessEntryRecord, error) {
|
||||
return nil, handlers_eks.ErrAccessEntryNotFound
|
||||
}
|
||||
|
||||
st := authenticate(validToken("https://sts/?x=1"), okVerify, lookup)
|
||||
assert.False(t, st.Authenticated)
|
||||
assert.Empty(t, st.User.Username)
|
||||
}
|
||||
|
||||
func TestAuthenticate_FallsBackUIDToARN(t *testing.T) {
|
||||
verify := func(string) (*handlers_eks.TokenVerifyResponse, error) {
|
||||
return &handlers_eks.TokenVerifyResponse{ARN: testARN}, nil // no UserID
|
||||
}
|
||||
lookup := func(string) (*handlers_eks.AccessEntryRecord, error) {
|
||||
return &handlers_eks.AccessEntryRecord{KubernetesUsername: testARN, KubernetesGroups: []string{"system:masters"}}, nil
|
||||
}
|
||||
|
||||
st := authenticate(validToken("https://sts/?x=1"), verify, lookup)
|
||||
require.True(t, st.Authenticated)
|
||||
assert.Equal(t, testARN, st.User.UID)
|
||||
_, code := postReview(t, a, "tok", nil)
|
||||
assert.Equal(t, http.StatusServiceUnavailable, code)
|
||||
}
|
||||
|
||||
func TestEnsureServingCert_PersistsAndReuses(t *testing.T) {
|
||||
|
||||
@@ -19,7 +19,12 @@ type tokenReviewSpec struct {
|
||||
type tokenReviewStatus struct {
|
||||
Authenticated bool `json:"authenticated"`
|
||||
User userInfo `json:"user,omitzero"`
|
||||
Error string `json:"error,omitempty"`
|
||||
// Audiences echoes the request's spec.audiences on success. When the
|
||||
// apiserver runs with --api-audiences, it sends spec.audiences and discards
|
||||
// an authenticated=true response that does not confirm a matching audience —
|
||||
// so an empty Audiences here surfaces as a 401 despite a valid principal.
|
||||
Audiences []string `json:"audiences,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type userInfo struct {
|
||||
|
||||
@@ -538,7 +538,7 @@ var awsgwStartCmd = &cobra.Command{
|
||||
awsgwHost := viper.GetString("host")
|
||||
if awsgwHost != "" {
|
||||
fmt.Println("Overwriting awsgw host to:", awsgwHost)
|
||||
//nodeConfig.AWSGW.Host = awsgwHost
|
||||
nodeConfig.AWSGW.Host = awsgwHost
|
||||
}
|
||||
|
||||
awsgwTlsCert := viper.GetString("tls-cert")
|
||||
|
||||
@@ -82,6 +82,25 @@ spx admin images import --name debian-13-arm64
|
||||
|
||||
Catalog imports verify the image against the catalog-declared SHA-256/SHA-512 digest before extraction. Use `--file` to import operator-supplied media (verification skipped — operator is responsible for integrity), or `--force` to re-download after a checksum mismatch.
|
||||
|
||||
### EKS node image
|
||||
|
||||
To run EKS, import the prebuilt node image from the catalog:
|
||||
|
||||
```bash
|
||||
spx admin images import --name spinifex-eks-node
|
||||
```
|
||||
|
||||
This pulls the Alpine + K3s node AMI from `iso.mulgadc.com`, verifies its
|
||||
checksum, and registers it tagged `spinifex:managed-by=eks`. `eks create-cluster`
|
||||
and `eks create-nodegroup` resolve the boot AMI by that tag, so no further
|
||||
configuration is needed.
|
||||
|
||||
Operators publishing a freshly built node image to the mirror run the
|
||||
**Publish EKS node AMI** GitHub Actions workflow (manual dispatch; uses the
|
||||
`CF_R2_*` repo secrets) or, locally, `make publish-eks-node-image` (see
|
||||
`scripts/publish-system-image.sh`; requires `R2_ENDPOINT` + R2-scoped
|
||||
`AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY`).
|
||||
|
||||
|
||||
## Cluster Shutdown
|
||||
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
// Package eksgw is the on-VM SigV4 client EKS control-plane helpers use to
|
||||
// reach the AWS gateway, instead of dialing core NATS. Both eks-gateway-publish
|
||||
// (bootstrap + state relay) and eks-token-webhook (TokenReview relay) sign with
|
||||
// the system (Predastore) credentials for service "eks" and POST over HTTPS;
|
||||
// the gateway authorizes via IAM and relays to the host-side NATS/STS/KV.
|
||||
package eksgw
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/predastore/auth"
|
||||
"github.com/mulgadc/spinifex/internal/tlsconfig"
|
||||
)
|
||||
|
||||
// Client posts SigV4-signed requests to the AWS gateway for the "eks" service.
|
||||
type Client struct {
|
||||
baseURL string
|
||||
accessKey string
|
||||
secretKey string
|
||||
region string
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
// New builds a client. caPath optionally pins the gateway TLS CA; when empty the
|
||||
// client relies on the system trust store. region defaults to us-east-1 when
|
||||
// empty (SigV4 requires a non-empty region).
|
||||
func New(baseURL, caPath, accessKey, secretKey, region string) (*Client, error) {
|
||||
if baseURL == "" {
|
||||
return nil, fmt.Errorf("eksgw: baseURL is required")
|
||||
}
|
||||
if accessKey == "" || secretKey == "" {
|
||||
return nil, fmt.Errorf("eksgw: access key and secret key are required")
|
||||
}
|
||||
if region == "" {
|
||||
region = "us-east-1"
|
||||
}
|
||||
|
||||
tlsCfg := &tls.Config{
|
||||
MinVersion: tls.VersionTLS13,
|
||||
CurvePreferences: tlsconfig.Curves,
|
||||
}
|
||||
if caPath != "" {
|
||||
pem, err := os.ReadFile(caPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("eksgw: read gateway CA %q: %w", caPath, err)
|
||||
}
|
||||
pool := x509.NewCertPool()
|
||||
if !pool.AppendCertsFromPEM(pem) {
|
||||
return nil, fmt.Errorf("eksgw: gateway CA %q has no usable certificates", caPath)
|
||||
}
|
||||
tlsCfg.RootCAs = pool
|
||||
}
|
||||
|
||||
return &Client{
|
||||
baseURL: strings.TrimRight(baseURL, "/"),
|
||||
accessKey: accessKey,
|
||||
secretKey: secretKey,
|
||||
region: region,
|
||||
httpClient: &http.Client{
|
||||
Timeout: 10 * time.Second,
|
||||
Transport: &http.Transport{TLSClientConfig: tlsCfg, MaxIdleConns: 2, IdleConnTimeout: 30 * time.Second},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Post SigV4-signs and sends a single POST of body to path (e.g.
|
||||
// "/clusters/alpha/token-review"). It returns the response body on a 2xx, or an
|
||||
// error (including the gateway status + body) otherwise. No retry — callers that
|
||||
// need it wrap Post in their own loop.
|
||||
func (c *Client) Post(path string, body []byte) ([]byte, error) {
|
||||
req, err := http.NewRequest(http.MethodPost, c.baseURL+path, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/x-amz-json-1.1")
|
||||
|
||||
sum := sha256.Sum256(body)
|
||||
if err := auth.SignReq(req, c.accessKey, c.secretKey, hex.EncodeToString(sum[:]), "eks", c.region); err != nil {
|
||||
return nil, fmt.Errorf("sign request: %w", err)
|
||||
}
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("send request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
respBody, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return nil, fmt.Errorf("gateway returned %d: %s", resp.StatusCode, strings.TrimSpace(string(respBody)))
|
||||
}
|
||||
return respBody, nil
|
||||
}
|
||||
@@ -224,8 +224,34 @@ for kver_dir in "$CHROOT_DIR/lib/modules"/*/; do
|
||||
done)
|
||||
rm -rf "$tmp_keep"
|
||||
|
||||
# Regenerate module dependency map from the surviving modules.
|
||||
depmod -b "$CHROOT_DIR" "$kver" 2>/dev/null || true
|
||||
# Decompress kept modules to plain .ko. Alpine ships .ko.gz, but a host
|
||||
# kmod that lacks gzip support (e.g. Debian trixie's kmod 34) silently
|
||||
# produces an EMPTY modules.dep from compressed modules — the guest's
|
||||
# modprobe then no-ops and qemu_fw_cfg never loads. Plain .ko sidesteps the
|
||||
# host-kmod dependency entirely and the guest needs no decompressor either.
|
||||
find "$kernel_dir" -name "*.ko.gz" -exec gunzip -f {} +
|
||||
|
||||
# Regenerate the module dependency map when a host depmod is available. A
|
||||
# populated modules.dep lets the guest init's load_mod() resolve modules via
|
||||
# modprobe; without it, load_mod() falls back to insmod-by-path (see
|
||||
# build/microvm/init.sh), so modules.dep is an optimisation, not a boot
|
||||
# requirement. depmod (kmod) is absent on some self-hosted CI runners and
|
||||
# cannot be apt-installed there — skip rather than hard-fail. When depmod
|
||||
# IS present, still assert a non-empty result so the trixie-kmod-can't-read
|
||||
# -compressed-modules bug (empty modules.dep from .ko.gz) fails loudly.
|
||||
if command -v depmod >/dev/null 2>&1; then
|
||||
if ! depmod -b "$CHROOT_DIR" "$kver"; then
|
||||
echo "ERROR: depmod failed for $kver" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -s "$kver_dir/modules.dep" ]; then
|
||||
echo "ERROR: ${kver_dir}modules.dep missing or empty after depmod" >&2
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "[build-microvm-image] depmod not found; skipping modules.dep" \
|
||||
"(guest init load_mod() will insmod modules by path)" >&2
|
||||
fi
|
||||
done
|
||||
|
||||
# --- Strip package-manager artifacts (not needed at runtime) ---
|
||||
|
||||
+101
-211
@@ -3,17 +3,28 @@ set -euo pipefail
|
||||
|
||||
# build-system-image.sh — Build a minimal system AMI from a manifest
|
||||
#
|
||||
# Supports Alpine Linux and Ubuntu 24.04 as base distros.
|
||||
# Creates a pre-baked image with custom packages, binaries, and setup scripts
|
||||
# installed, ready for import as a Spinifex AMI.
|
||||
# Supports Alpine Linux and Ubuntu 24.04+ as base distros. Creates a pre-baked
|
||||
# image with custom packages, binaries, and setup scripts installed, ready for
|
||||
# import as a Spinifex AMI.
|
||||
#
|
||||
# Requirements: qemu-nbd, qemu-img, sudo (for mount/chroot), curl
|
||||
# Customization runs entirely inside the libguestfs appliance (its own kernel +
|
||||
# userspace), so the build touches NO host block device, NO host mount, and
|
||||
# needs NO sudo. This deliberately replaces the previous qemu-nbd + `sudo mount`
|
||||
# + chroot + /dev bind-mount flow, whose leaked binds could be traversed by a
|
||||
# later `rm -rf` and wipe the host's /dev (it wedged CI runners). libguestfs
|
||||
# isolates all of that in a throwaway VM.
|
||||
#
|
||||
# Requirements: virt-customize, guestfish, virt-resize (libguestfs-tools),
|
||||
# qemu-img, curl
|
||||
# Usage: ./scripts/build-system-image.sh <manifest.conf> [--import]
|
||||
# --import Also import the image as an AMI via spx admin images import
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
|
||||
# No libvirt on CI runners — drive qemu directly. Harmless when libvirt exists.
|
||||
export LIBGUESTFS_BACKEND="${LIBGUESTFS_BACKEND:-direct}"
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 <manifest.conf> [--import]"
|
||||
echo ""
|
||||
@@ -60,8 +71,6 @@ done
|
||||
|
||||
# Derived paths
|
||||
BUILD_DIR="/tmp/${IMAGE_NAME}-image-build"
|
||||
NBD_DEV="/dev/nbd0"
|
||||
MOUNT_DIR="${BUILD_DIR}/mnt"
|
||||
|
||||
case "$DISTRO" in
|
||||
alpine)
|
||||
@@ -74,7 +83,6 @@ case "$DISTRO" in
|
||||
OUTPUT_IMAGE="${BUILD_DIR}/${IMAGE_NAME}-alpine.qcow2"
|
||||
OUTPUT_RAW="${BUILD_DIR}/${IMAGE_NAME}-alpine.raw"
|
||||
DISTRO_VERSION="${ALPINE_VERSION}"
|
||||
ROOT_PART="${NBD_DEV}"
|
||||
BOOT_MODE="bios"
|
||||
;;
|
||||
ubuntu)
|
||||
@@ -87,7 +95,6 @@ case "$DISTRO" in
|
||||
OUTPUT_IMAGE="${BUILD_DIR}/${IMAGE_NAME}-ubuntu.qcow2"
|
||||
OUTPUT_RAW="${BUILD_DIR}/${IMAGE_NAME}-ubuntu.raw"
|
||||
DISTRO_VERSION="${UBUNTU_VERSION}"
|
||||
ROOT_PART="${NBD_DEV}p1"
|
||||
BOOT_MODE="uefi"
|
||||
;;
|
||||
*)
|
||||
@@ -96,19 +103,20 @@ case "$DISTRO" in
|
||||
;;
|
||||
esac
|
||||
|
||||
cleanup() {
|
||||
echo "Cleaning up..."
|
||||
# Ubuntu chroot has bind mounts that must be released first
|
||||
sudo umount "${MOUNT_DIR}/dev/pts" 2>/dev/null || true
|
||||
sudo umount "${MOUNT_DIR}/dev" 2>/dev/null || true
|
||||
sudo umount "${MOUNT_DIR}/sys" 2>/dev/null || true
|
||||
sudo umount "${MOUNT_DIR}/proc" 2>/dev/null || true
|
||||
sudo umount "${MOUNT_DIR}" 2>/dev/null || true
|
||||
sudo qemu-nbd --disconnect "${NBD_DEV}" 2>/dev/null || true
|
||||
exec 9>&- 2>/dev/null || true # release nbd lock if held
|
||||
echo "Done."
|
||||
# import_ami registers the built raw image as a Spinifex AMI. Shared by the
|
||||
# fresh-build and cached-build paths.
|
||||
import_ami() {
|
||||
echo "Importing as AMI..."
|
||||
rm -f "$OUTPUT_IMAGE"
|
||||
local args=(--file "$OUTPUT_RAW" --distro "${DISTRO}" --version "${DISTRO_VERSION}" --arch x86_64 --boot-mode "${BOOT_MODE}")
|
||||
if [[ -n "${AMI_NAME:-}" ]]; then
|
||||
args+=(--ami-name "$AMI_NAME")
|
||||
fi
|
||||
if [[ -n "${SYSTEM_TAG:-}" ]]; then
|
||||
args+=(--tag "$SYSTEM_TAG")
|
||||
fi
|
||||
spx admin images import "${args[@]}"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# In quiet mode, redirect build output to /dev/null (import output still shown)
|
||||
if [[ "$QUIET" == true ]]; then
|
||||
@@ -116,7 +124,7 @@ if [[ "$QUIET" == true ]]; then
|
||||
exec 1>/dev/null # suppress build output
|
||||
fi
|
||||
|
||||
echo "=== System Image Builder ==="
|
||||
echo "=== System Image Builder (libguestfs) ==="
|
||||
echo "Image: ${IMAGE_NAME} — ${IMAGE_DESCRIPTION:-}"
|
||||
echo "Distro: ${DISTRO} ${DISTRO_VERSION}"
|
||||
echo "Size: ${IMAGE_SIZE}"
|
||||
@@ -124,18 +132,18 @@ echo "Build: ${BUILD_DIR}"
|
||||
echo ""
|
||||
|
||||
# Step 0: Check prerequisites
|
||||
if ! command -v qemu-nbd &>/dev/null; then
|
||||
echo "ERROR: qemu-nbd not found. Install qemu-utils."
|
||||
for tool in virt-customize qemu-img; do
|
||||
if ! command -v "$tool" &>/dev/null; then
|
||||
echo "ERROR: $tool not found. Install libguestfs-tools + qemu-utils."
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
if [[ "$DISTRO" == "alpine" ]] && ! command -v guestfish &>/dev/null; then
|
||||
echo "ERROR: guestfish not found. Install libguestfs-tools."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v qemu-img &>/dev/null; then
|
||||
echo "ERROR: qemu-img not found. Install qemu-utils."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$DISTRO" == "ubuntu" ]] && ! command -v parted &>/dev/null && ! [[ -x /usr/sbin/parted ]]; then
|
||||
echo "ERROR: parted not found. Install parted."
|
||||
if [[ "$DISTRO" == "ubuntu" ]] && ! command -v virt-resize &>/dev/null; then
|
||||
echo "ERROR: virt-resize not found. Install libguestfs-tools."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -166,11 +174,11 @@ if [[ -n "${INSTALL_BINARIES:-}" ]]; then
|
||||
done
|
||||
fi
|
||||
|
||||
# Serialize the entire image build with flock — concurrent builds on the same
|
||||
# host (e.g. CI single-node + multi-node jobs) share /dev/nbd0 and BUILD_DIR.
|
||||
NBD_LOCK="/tmp/build-system-image.lock"
|
||||
# Serialize the build with flock — concurrent builds on the same host (e.g. CI
|
||||
# single-node + multi-node jobs) share BUILD_DIR + the cached raw image.
|
||||
BUILD_LOCK="/tmp/build-system-image.lock"
|
||||
echo "Acquiring build lock..."
|
||||
exec 9>"$NBD_LOCK"
|
||||
exec 9>"$BUILD_LOCK"
|
||||
flock 9
|
||||
echo "Lock acquired"
|
||||
|
||||
@@ -189,21 +197,12 @@ if [[ -f "$OUTPUT_RAW" ]] && [[ $(( $(date +%s) - $(stat -c %Y "$OUTPUT_RAW") ))
|
||||
echo " raw: $OUTPUT_RAW ($(du -h "$OUTPUT_RAW" | cut -f1))"
|
||||
|
||||
if [[ "$DO_IMPORT" == true ]]; then
|
||||
echo "Importing as AMI..."
|
||||
rm -f "$OUTPUT_IMAGE"
|
||||
IMPORT_ARGS=(--file "$OUTPUT_RAW" --distro "${DISTRO}" --version "${DISTRO_VERSION}" --arch x86_64 --boot-mode "${BOOT_MODE}")
|
||||
if [[ -n "${AMI_NAME:-}" ]]; then
|
||||
IMPORT_ARGS+=(--ami-name "$AMI_NAME")
|
||||
fi
|
||||
if [[ -n "${SYSTEM_TAG:-}" ]]; then
|
||||
IMPORT_ARGS+=(--tag "$SYSTEM_TAG")
|
||||
fi
|
||||
spx admin images import "${IMPORT_ARGS[@]}"
|
||||
import_ami
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
mkdir -p "$BUILD_DIR" "$MOUNT_DIR"
|
||||
mkdir -p "$BUILD_DIR"
|
||||
|
||||
# Step 1: Download base cloud image
|
||||
if [[ -f "${BUILD_DIR}/${SOURCE_IMAGE}" ]]; then
|
||||
@@ -228,126 +227,66 @@ rm -f "$OUTPUT_IMAGE"
|
||||
echo "Copying image for customization..."
|
||||
cp "${BUILD_DIR}/${SOURCE_IMAGE}" "$OUTPUT_IMAGE"
|
||||
|
||||
# Resize the image to provide room for packages
|
||||
qemu-img resize "$OUTPUT_IMAGE" "$IMAGE_SIZE"
|
||||
|
||||
# Step 3: Connect via qemu-nbd
|
||||
echo "Connecting image via qemu-nbd..."
|
||||
sudo modprobe nbd max_part=4 2>/dev/null || true
|
||||
if [[ ! -e "${NBD_DEV}" ]]; then
|
||||
echo "ERROR: ${NBD_DEV} does not exist. Is the nbd kernel module loaded? Try: sudo modprobe nbd"
|
||||
exit 1
|
||||
# Step 3: Resize to provide room for packages. Both paths grow the root
|
||||
# filesystem entirely inside the libguestfs appliance — no host loop/nbd device.
|
||||
echo "Resizing image to ${IMAGE_SIZE}..."
|
||||
if [[ "$DISTRO" == "alpine" ]]; then
|
||||
# Alpine cloud images have ext4 directly on the whole disk (no partition
|
||||
# table); grow the disk file then expand the fs from inside the appliance.
|
||||
qemu-img resize "$OUTPUT_IMAGE" "$IMAGE_SIZE"
|
||||
guestfish --rw -a "$OUTPUT_IMAGE" run \
|
||||
: e2fsck /dev/sda forceall:true \
|
||||
: resize2fs /dev/sda
|
||||
else
|
||||
# Ubuntu images use a GPT partition table with the root fs on /dev/sda1;
|
||||
# virt-resize expands the partition + fs into a fresh IMAGE_SIZE disk.
|
||||
RESIZED="${OUTPUT_IMAGE}.resized"
|
||||
rm -f "$RESIZED"
|
||||
qemu-img create -f qcow2 "$RESIZED" "$IMAGE_SIZE"
|
||||
virt-resize --expand /dev/sda1 "$OUTPUT_IMAGE" "$RESIZED"
|
||||
mv -f "$RESIZED" "$OUTPUT_IMAGE"
|
||||
fi
|
||||
sudo qemu-nbd --disconnect "${NBD_DEV}" 2>/dev/null || true
|
||||
sudo qemu-nbd --connect="${NBD_DEV}" "$OUTPUT_IMAGE"
|
||||
|
||||
# Wait for the nbd device to be ready
|
||||
for i in $(seq 1 10); do
|
||||
if sudo blockdev --getsize64 "${NBD_DEV}" &>/dev/null; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
sleep 1
|
||||
|
||||
# Ubuntu images have a partition table; wait for partition devices then resize
|
||||
# the partition to fill the expanded image before resizing the filesystem.
|
||||
# Step 4: Assemble the virt-customize operation list. virt-customize runs the
|
||||
# operations in the order they appear on the command line, so this mirrors the
|
||||
# old chroot sequence: packages → binaries → files → setup → services → clean.
|
||||
CUST=(virt-customize -a "$OUTPUT_IMAGE" --network)
|
||||
if [[ "$DISTRO" == "ubuntu" ]]; then
|
||||
for i in $(seq 1 10); do
|
||||
if [[ -e "${ROOT_PART}" ]]; then break; fi
|
||||
sleep 1
|
||||
done
|
||||
echo "Resizing partition..."
|
||||
# After qemu-img resize the GPT backup header sits at the old end of disk.
|
||||
# Relocate it to the new end (sfdisk --relocate is util-linux built-in,
|
||||
# no extra packages needed), then re-read the partition table before parted
|
||||
# runs — without this, parted sees stale GPT geometry and fails with
|
||||
# "Unable to satisfy all constraints on the partition."
|
||||
sudo sgdisk --move-second-header "${NBD_DEV}"
|
||||
sudo partprobe "${NBD_DEV}" 2>/dev/null || true
|
||||
sleep 1
|
||||
sudo parted --script "${NBD_DEV}" resizepart 1 100%
|
||||
# DKMS driver bakes (GPU images) need real resources in the appliance.
|
||||
CUST+=(--memsize 6144 --smp 4)
|
||||
else
|
||||
CUST+=(--memsize 2048)
|
||||
fi
|
||||
|
||||
# Alpine cloud images have ext4 directly on the block device (no partition table).
|
||||
echo "Checking filesystem..."
|
||||
sudo e2fsck -f -y "${ROOT_PART}" || {
|
||||
ec=$?
|
||||
if [[ $ec -gt 1 ]]; then
|
||||
echo "ERROR: e2fsck failed with exit code $ec on ${ROOT_PART}"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
echo "Resizing filesystem..."
|
||||
if ! sudo resize2fs "${ROOT_PART}"; then
|
||||
echo "ERROR: resize2fs failed on ${ROOT_PART}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Step 4: Mount and customize
|
||||
echo "Mounting root filesystem..."
|
||||
sudo mount "${ROOT_PART}" "$MOUNT_DIR"
|
||||
|
||||
# Set up resolv.conf for DNS inside chroot.
|
||||
# Ubuntu cloud images symlink /etc/resolv.conf → /run/systemd/resolve/stub-resolv.conf
|
||||
# which doesn't exist yet. Remove the symlink and write a real file.
|
||||
sudo rm -f "${MOUNT_DIR}/etc/resolv.conf"
|
||||
sudo cp /etc/resolv.conf "${MOUNT_DIR}/etc/resolv.conf"
|
||||
|
||||
# Ubuntu chroot requires /proc /sys /dev bind mounts for systemd and DKMS
|
||||
if [[ "$DISTRO" == "ubuntu" ]]; then
|
||||
sudo mount --bind /proc "${MOUNT_DIR}/proc"
|
||||
sudo mount --bind /sys "${MOUNT_DIR}/sys"
|
||||
sudo mount --bind /dev "${MOUNT_DIR}/dev"
|
||||
sudo mount --bind /dev/pts "${MOUNT_DIR}/dev/pts"
|
||||
fi
|
||||
|
||||
# Install packages
|
||||
# Packages
|
||||
if [[ "$DISTRO" == "alpine" ]] && [[ -n "${APK_PACKAGES:-}" ]]; then
|
||||
echo "Installing packages: ${APK_PACKAGES}..."
|
||||
sudo chroot "$MOUNT_DIR" /bin/sh -c "
|
||||
set -e
|
||||
# Enable community repo
|
||||
sed -i 's|^#\(.*community\)|\1|' /etc/apk/repositories 2>/dev/null || true
|
||||
|
||||
# Ensure community repo is present
|
||||
if ! grep -q community /etc/apk/repositories; then
|
||||
MIRROR=\$(grep main /etc/apk/repositories | head -1 | sed 's|/main|/community|')
|
||||
echo \"\$MIRROR\" >> /etc/apk/repositories
|
||||
fi
|
||||
|
||||
apk update
|
||||
apk add ${APK_PACKAGES}
|
||||
"
|
||||
echo "Will install packages: ${APK_PACKAGES}"
|
||||
CUST+=(--run-command 'sed -i "s|^#\(.*community\)|\1|" /etc/apk/repositories 2>/dev/null || true')
|
||||
CUST+=(--run-command 'grep -q community /etc/apk/repositories || grep main /etc/apk/repositories | head -1 | sed "s|/main|/community|" >> /etc/apk/repositories')
|
||||
CUST+=(--run-command 'apk update')
|
||||
CUST+=(--run-command "apk add ${APK_PACKAGES}")
|
||||
elif [[ "$DISTRO" == "ubuntu" ]] && [[ -n "${APT_PACKAGES:-}" ]]; then
|
||||
echo "Installing packages: ${APT_PACKAGES}..."
|
||||
sudo chroot "$MOUNT_DIR" /bin/bash -c "
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update
|
||||
apt-get install -y --no-install-recommends ${APT_PACKAGES}
|
||||
"
|
||||
echo "Will install packages: ${APT_PACKAGES}"
|
||||
CUST+=(--run-command 'export DEBIAN_FRONTEND=noninteractive; apt-get update')
|
||||
CUST+=(--run-command "export DEBIAN_FRONTEND=noninteractive; apt-get install -y --no-install-recommends ${APT_PACKAGES}")
|
||||
fi
|
||||
|
||||
# Step 5: Copy binaries into the image (before setup script, which may reference them)
|
||||
# Binaries (mode 0755). --mkdir is idempotent (mkdir -p semantics).
|
||||
if [[ -n "${INSTALL_BINARIES:-}" ]]; then
|
||||
echo "Installing binaries..."
|
||||
IFS=' ' read -ra BINARY_PAIRS <<< "$INSTALL_BINARIES"
|
||||
for pair in "${BINARY_PAIRS[@]}"; do
|
||||
src="${pair%%:*}"
|
||||
dst="${pair#*:}"
|
||||
src_path="${PROJECT_DIR}/${src}"
|
||||
echo " ${src} -> ${dst}"
|
||||
sudo cp "$src_path" "${MOUNT_DIR}${dst}"
|
||||
sudo chmod 755 "${MOUNT_DIR}${dst}"
|
||||
CUST+=(--mkdir "$(dirname "$dst")")
|
||||
CUST+=(--upload "${src_path}:${dst}")
|
||||
CUST+=(--chmod "0755:${dst}")
|
||||
done
|
||||
fi
|
||||
|
||||
# Copy auxiliary files (systemd units, OpenRC initd scripts, cron entries, configs).
|
||||
# Same src:dst pair grammar as INSTALL_BINARIES; mode 0644, parent dirs auto-created.
|
||||
# Optional — manifests without INSTALL_FILES (e.g. existing GPU images) skip this step.
|
||||
# Auxiliary files (systemd units, OpenRC initd scripts, cron entries, configs).
|
||||
# Mode 0644; setup scripts may chmod role-specific files to 0755 afterwards.
|
||||
if [[ -n "${INSTALL_FILES:-}" ]]; then
|
||||
echo "Installing files..."
|
||||
IFS=' ' read -ra FILE_PAIRS <<< "$INSTALL_FILES"
|
||||
for pair in "${FILE_PAIRS[@]}"; do
|
||||
src="${pair%%:*}"
|
||||
@@ -357,85 +296,45 @@ if [[ -n "${INSTALL_FILES:-}" ]]; then
|
||||
echo "ERROR: INSTALL_FILES source not found: $src_path"
|
||||
exit 1
|
||||
fi
|
||||
echo " ${src} -> ${dst}"
|
||||
sudo mkdir -p "${MOUNT_DIR}$(dirname "$dst")"
|
||||
sudo cp "$src_path" "${MOUNT_DIR}${dst}"
|
||||
sudo chmod 0644 "${MOUNT_DIR}${dst}"
|
||||
CUST+=(--mkdir "$(dirname "$dst")")
|
||||
CUST+=(--upload "${src_path}:${dst}")
|
||||
CUST+=(--chmod "0644:${dst}")
|
||||
done
|
||||
fi
|
||||
|
||||
# Run custom setup script inside chroot (after binaries are installed)
|
||||
# Custom setup script (uploaded + executed + removed by --run, inside the guest).
|
||||
if [[ -n "${SETUP_SCRIPT:-}" ]]; then
|
||||
setup_path="${PROJECT_DIR}/${SETUP_SCRIPT}"
|
||||
if [[ ! -f "$setup_path" ]]; then
|
||||
echo "ERROR: Setup script not found: $setup_path"
|
||||
exit 1
|
||||
fi
|
||||
echo "Running setup script: ${SETUP_SCRIPT}..."
|
||||
sudo cp "$setup_path" "${MOUNT_DIR}/tmp/setup.sh"
|
||||
sudo chmod 755 "${MOUNT_DIR}/tmp/setup.sh"
|
||||
if [[ "$DISTRO" == "ubuntu" ]]; then
|
||||
if [[ ! -x "${MOUNT_DIR}/bin/bash" ]]; then
|
||||
echo "ERROR: /bin/bash not found or not executable in chroot"
|
||||
exit 1
|
||||
fi
|
||||
sudo chroot "$MOUNT_DIR" /bin/bash -x /tmp/setup.sh
|
||||
else
|
||||
sudo chroot "$MOUNT_DIR" /tmp/setup.sh
|
||||
fi
|
||||
sudo rm -f "${MOUNT_DIR}/tmp/setup.sh"
|
||||
CUST+=(--run "$setup_path")
|
||||
fi
|
||||
|
||||
# Enable services (runs after INSTALL_FILES + SETUP_SCRIPT so init scripts the
|
||||
# manifest references are already on disk).
|
||||
# Enable services (after files + setup so referenced init scripts exist).
|
||||
if [[ -n "${ENABLE_SERVICES:-}" ]]; then
|
||||
echo "Enabling services: ${ENABLE_SERVICES}..."
|
||||
IFS=' ' read -ra SERVICES <<< "$ENABLE_SERVICES"
|
||||
for svc in "${SERVICES[@]}"; do
|
||||
if [[ "$DISTRO" == "alpine" ]]; then
|
||||
if ! sudo chroot "$MOUNT_DIR" /bin/sh -c "rc-update add ${svc} default"; then
|
||||
echo "ERROR: Failed to enable service '${svc}' — does it exist in the image?"
|
||||
exit 1
|
||||
fi
|
||||
CUST+=(--run-command "rc-update add ${svc} default")
|
||||
else
|
||||
if ! sudo chroot "$MOUNT_DIR" /bin/bash -c "systemctl enable ${svc}"; then
|
||||
echo "ERROR: Failed to enable service '${svc}'"
|
||||
exit 1
|
||||
fi
|
||||
CUST+=(--run-command "systemctl enable ${svc}")
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# Step 6: Clean up and unmount
|
||||
echo "Cleaning up image..."
|
||||
# Final cleanup of package caches + tmp.
|
||||
if [[ "$DISTRO" == "alpine" ]]; then
|
||||
sudo chroot "$MOUNT_DIR" /bin/sh -c '
|
||||
apk cache clean 2>/dev/null || true
|
||||
rm -rf /var/cache/apk/* /tmp/*
|
||||
'
|
||||
CUST+=(--run-command 'apk cache clean 2>/dev/null || true; rm -rf /var/cache/apk/* /tmp/* 2>/dev/null || true')
|
||||
else
|
||||
sudo chroot "$MOUNT_DIR" /bin/bash -c '
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get clean
|
||||
rm -rf /var/lib/apt/lists/* /tmp/*
|
||||
'
|
||||
CUST+=(--run-command 'export DEBIAN_FRONTEND=noninteractive; apt-get clean; rm -rf /var/lib/apt/lists/* /tmp/* 2>/dev/null || true')
|
||||
fi
|
||||
|
||||
# Restore the systemd-resolved symlink (Ubuntu default); cloud-init sets DNS on boot.
|
||||
sudo rm -f "${MOUNT_DIR}/etc/resolv.conf"
|
||||
sudo ln -sf /run/systemd/resolve/stub-resolv.conf "${MOUNT_DIR}/etc/resolv.conf"
|
||||
echo "Customizing image (libguestfs appliance)..."
|
||||
"${CUST[@]}"
|
||||
|
||||
echo "Unmounting..."
|
||||
if [[ "$DISTRO" == "ubuntu" ]]; then
|
||||
sudo umount "${MOUNT_DIR}/dev/pts"
|
||||
sudo umount "${MOUNT_DIR}/dev"
|
||||
sudo umount "${MOUNT_DIR}/sys"
|
||||
sudo umount "${MOUNT_DIR}/proc"
|
||||
fi
|
||||
sudo umount "$MOUNT_DIR"
|
||||
sudo qemu-nbd --disconnect "${NBD_DEV}"
|
||||
|
||||
# Step 7: Convert to raw for import
|
||||
# Step 5: Convert to raw for import
|
||||
echo "Converting to raw format..."
|
||||
qemu-img convert -f qcow2 -O raw "$OUTPUT_IMAGE" "$OUTPUT_RAW"
|
||||
|
||||
@@ -452,16 +351,7 @@ echo " raw: $OUTPUT_RAW ($(du -h "$OUTPUT_RAW" | cut -f1))"
|
||||
echo ""
|
||||
|
||||
if [[ "$DO_IMPORT" == true ]]; then
|
||||
echo "Importing as AMI..."
|
||||
rm -f "$OUTPUT_IMAGE"
|
||||
IMPORT_ARGS=(--file "$OUTPUT_RAW" --distro "${DISTRO}" --version "${DISTRO_VERSION}" --arch x86_64 --boot-mode "${BOOT_MODE}")
|
||||
if [[ -n "${AMI_NAME:-}" ]]; then
|
||||
IMPORT_ARGS+=(--ami-name "$AMI_NAME")
|
||||
fi
|
||||
if [[ -n "${SYSTEM_TAG:-}" ]]; then
|
||||
IMPORT_ARGS+=(--tag "$SYSTEM_TAG")
|
||||
fi
|
||||
spx admin images import "${IMPORT_ARGS[@]}"
|
||||
import_ami
|
||||
else
|
||||
echo "To import as AMI, run:"
|
||||
echo " spx admin images import \\"
|
||||
|
||||
@@ -49,9 +49,11 @@ case "${ROLE}" in
|
||||
rc-update add eks-token-webhook default
|
||||
rc-update add k3s default
|
||||
rc-update add k3s-first-boot default
|
||||
rc-update add mulga-eks-state-report default
|
||||
rc-service eks-token-webhook start
|
||||
rc-service k3s start
|
||||
rc-service k3s-first-boot start
|
||||
rc-service mulga-eks-state-report start
|
||||
;;
|
||||
agent)
|
||||
log "configuring agent role"
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
#!/sbin/openrc-run
|
||||
# eks-token-webhook — K8s TokenReview authenticator for IAM-backed kubectl.
|
||||
# Binds 127.0.0.1:8443 only; kube-apiserver is the sole client (loopback path).
|
||||
# Decodes `aws eks get-token` presigned URLs, verifies them via Mulga STS over
|
||||
# NATS, and resolves the principal to a TokenReview via the cluster AccessEntry
|
||||
# KV. Ordered `before k3s` so its apiserver kubeconfig exists before k3s reads
|
||||
# Relays `aws eks get-token` bearer tokens through the AWS gateway broker (SigV4
|
||||
# HTTPS), which host-side verifies via Mulga STS + resolves the principal to a
|
||||
# TokenReview via the cluster AccessEntry KV — the VM never speaks core NATS.
|
||||
# Ordered `before k3s` so its apiserver kubeconfig exists before k3s reads
|
||||
# --authentication-token-webhook-config-file.
|
||||
|
||||
description="EKS Token Review webhook (Mulga)"
|
||||
@@ -11,16 +12,20 @@ command="/usr/local/bin/eks-token-webhook"
|
||||
command_args="--addr=127.0.0.1:8443"
|
||||
command_background="yes"
|
||||
pidfile="/run/eks-token-webhook.pid"
|
||||
output_log="/var/log/eks-token-webhook.log"
|
||||
error_log="/var/log/eks-token-webhook.log"
|
||||
# Headless appliance: no SSH. Route the webhook's stdout/stderr to the serial
|
||||
# console so its startup + per-request decisions land in the host-captured
|
||||
# console log (the only off-VM window into webhook health).
|
||||
output_log="/dev/console"
|
||||
error_log="/dev/console"
|
||||
|
||||
# NATS creds + cluster identity come from the cloud-init-seeded first-boot env.
|
||||
# Sourced here and exported so start-stop-daemon hands them to the binary, which
|
||||
# reads SPINIFEX_NATS_* / EKS_ACCOUNT_ID / EKS_CLUSTER_NAME from its environment.
|
||||
# Gateway endpoint + SigV4 creds + cluster identity come from the cloud-init-
|
||||
# seeded first-boot env. Sourced here and exported so start-stop-daemon hands
|
||||
# them to the binary, which reads EKS_GATEWAY_* / EKS_ACCESS_KEY / EKS_SECRET_KEY
|
||||
# / EKS_REGION / EKS_ACCOUNT_ID / EKS_CLUSTER_NAME from its environment.
|
||||
if [ -f /etc/spinifex-eks/first-boot.env ]; then
|
||||
# shellcheck disable=SC1091
|
||||
. /etc/spinifex-eks/first-boot.env
|
||||
export SPINIFEX_NATS_URL SPINIFEX_NATS_TOKEN SPINIFEX_NATS_CA EKS_ACCOUNT_ID EKS_CLUSTER_NAME
|
||||
export EKS_GATEWAY_URL EKS_GATEWAY_CA EKS_ACCESS_KEY EKS_SECRET_KEY EKS_REGION EKS_ACCOUNT_ID EKS_CLUSTER_NAME
|
||||
fi
|
||||
|
||||
depend() {
|
||||
@@ -30,7 +35,7 @@ depend() {
|
||||
|
||||
start_pre() {
|
||||
if [ ! -f /etc/spinifex-eks/first-boot.env ]; then
|
||||
eerror "/etc/spinifex-eks/first-boot.env not present — cloud-init did not seed NATS creds"
|
||||
eerror "/etc/spinifex-eks/first-boot.env not present — cloud-init did not seed gateway creds"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -5,13 +5,17 @@ set -eu
|
||||
# Reads the bootstrap node-token and admin kubeconfig that K3s writes at
|
||||
# server startup, rewrites the kubeconfig server address to the cluster's
|
||||
# NLB endpoint (so workers and external kubectl can use it), and publishes
|
||||
# both as one-shot NATS messages for the spinifex cluster reconciler to
|
||||
# consume into KV.
|
||||
# the bootstrap artifacts to the host via the AWS gateway (SigV4-signed HTTPS
|
||||
# POST — the eks-gateway-publish helper) for the spinifex cluster reconciler
|
||||
# to consume into KV. The gateway relays onto the eks.bus.* NATS subjects; the
|
||||
# VM never speaks NATS directly.
|
||||
#
|
||||
# Required env (from cloud-init user-data /etc/spinifex-eks/first-boot.env):
|
||||
# SPINIFEX_NATS_URL nats://...
|
||||
# SPINIFEX_NATS_TOKEN shared NATS auth token
|
||||
# SPINIFEX_NATS_CA /etc/spinifex-eks/nats-ca.pem (TLS CA bundle)
|
||||
# EKS_GATEWAY_URL https://{mgmt-host}:9999 (AWS gateway)
|
||||
# EKS_GATEWAY_CA /etc/spinifex-eks/gateway-ca.pem (TLS CA bundle)
|
||||
# EKS_ACCESS_KEY system SigV4 access key id
|
||||
# EKS_SECRET_KEY system SigV4 secret access key
|
||||
# EKS_REGION SigV4 signing region
|
||||
# EKS_ACCOUNT_ID
|
||||
# EKS_CLUSTER_NAME
|
||||
# EKS_NLB_ENDPOINT https://{cluster}.{accountID}.eks.{region}.{suffix}
|
||||
@@ -36,10 +40,15 @@ if [ ! -f "${ENVFILE}" ]; then
|
||||
die "${ENVFILE} not found — cloud-init did not seed first-boot env"
|
||||
fi
|
||||
|
||||
# set -a so the sourced KEY=value lines are exported: the eks-gateway-publish
|
||||
# child reads its config from the environment (EKS_ACCOUNT_ID etc.). A bare
|
||||
# source leaves them unexported and the helper exits "--account-id is required".
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
. "${ENVFILE}"
|
||||
set +a
|
||||
|
||||
for v in SPINIFEX_NATS_URL EKS_ACCOUNT_ID EKS_CLUSTER_NAME EKS_NLB_ENDPOINT; do
|
||||
for v in EKS_GATEWAY_URL EKS_ACCESS_KEY EKS_SECRET_KEY EKS_ACCOUNT_ID EKS_CLUSTER_NAME EKS_NLB_ENDPOINT; do
|
||||
eval "val=\${$v:-}"
|
||||
[ -n "${val}" ] || die "env ${v} not set"
|
||||
done
|
||||
@@ -105,23 +114,18 @@ CA_B64=$(awk '/certificate-authority-data:/ {print $2; exit}' "${KUBECONFIG_FILE
|
||||
JWKS=$(kubectl --kubeconfig "${KUBECONFIG_FILE}" get --raw='/openid/v1/jwks' 2>/dev/null)
|
||||
[ -n "${JWKS}" ] || die "apiserver returned empty /openid/v1/jwks"
|
||||
|
||||
# 3. Publish the four one-shot bootstrap messages. Each is a BootstrapEnvelope
|
||||
# JSON document (handlers/eks/nats_bootstrap.go); jq encodes the values so
|
||||
# embedded newlines/quotes in the kubeconfig and JWKS stay valid JSON.
|
||||
NATS_ARGS="-s ${SPINIFEX_NATS_URL}"
|
||||
if [ -n "${SPINIFEX_NATS_TOKEN:-}" ]; then
|
||||
NATS_ARGS="${NATS_ARGS} --token ${SPINIFEX_NATS_TOKEN}"
|
||||
fi
|
||||
if [ -n "${SPINIFEX_NATS_CA:-}" ] && [ -f "${SPINIFEX_NATS_CA}" ]; then
|
||||
NATS_ARGS="${NATS_ARGS} --tlsca ${SPINIFEX_NATS_CA}"
|
||||
fi
|
||||
# 3. Publish the four bootstrap messages through the AWS gateway. Each is a
|
||||
# BootstrapEnvelope JSON document (handlers/eks/nats_bootstrap.go); jq encodes
|
||||
# the values so embedded newlines/quotes in the kubeconfig and JWKS stay valid
|
||||
# JSON. eks-gateway-publish SigV4-signs the POST and retries until the gateway
|
||||
# acks, then the gateway relays onto eks.bus.{account}.{cluster}.{kind}. It
|
||||
# reads EKS_GATEWAY_URL/EKS_GATEWAY_CA/EKS_ACCESS_KEY/EKS_SECRET_KEY/EKS_REGION/
|
||||
# EKS_ACCOUNT_ID/EKS_CLUSTER_NAME from the env file already sourced above.
|
||||
|
||||
# publish_envelope <kind-suffix>: reads the envelope JSON from stdin.
|
||||
publish_envelope() {
|
||||
subj="eks.bus.${EKS_ACCOUNT_ID}.${EKS_CLUSTER_NAME}.$1"
|
||||
log "publishing $1 to ${subj}"
|
||||
# shellcheck disable=SC2086
|
||||
nats ${NATS_ARGS} pub "${subj}" --force-stdin
|
||||
log "publishing $1 via gateway"
|
||||
eks-gateway-publish -channel bootstrap -kind "$1"
|
||||
}
|
||||
|
||||
jq -n --arg t "${NODE_TOKEN}" '{token: $t}' | publish_envelope k3s-bootstrap-token
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
#!/sbin/openrc-run
|
||||
# K3s server control plane. v1 = single-node embedded SQLite (kine) datastore;
|
||||
# the per-cluster config.yaml omits cluster-init so k3s uses SQLite, not etcd
|
||||
# (etcd's per-commit fsync stalls on the viperblock root volume). Data lives at
|
||||
# /var/lib/rancher/k3s; subsequent starts reuse it. Multi-server HA needs etcd
|
||||
# and is deferred.
|
||||
# K3s server control plane. The per-cluster config.yaml sets cluster-init, so
|
||||
# k3s uses the embedded etcd datastore (required for multi-server HA). Data
|
||||
# lives at /var/lib/rancher/k3s; subsequent starts reuse it.
|
||||
|
||||
description="K3s Kubernetes server (Mulga EKS control plane)"
|
||||
command="/usr/local/bin/k3s"
|
||||
@@ -13,8 +11,8 @@ pidfile="/run/k3s.pid"
|
||||
output_log="/var/log/k3s.log"
|
||||
error_log="/var/log/k3s.log"
|
||||
|
||||
# Generous start timeout — K3s bootstraps containerd, initialises the SQLite
|
||||
# datastore, pulls system images. First boot can take 2-3 minutes on cold cache.
|
||||
# Generous start timeout — K3s bootstraps containerd, initialises the embedded
|
||||
# etcd datastore, pulls system images. First boot can take 2-3 minutes on cold cache.
|
||||
start_stop_daemon_args="--wait 5000"
|
||||
|
||||
depend() {
|
||||
|
||||
@@ -3,22 +3,24 @@ IMAGE_DESCRIPTION="Unified EKS K3s node VM — Alpine + K3s v1.32.5+k3s1 + eks-t
|
||||
DISTRO="alpine"
|
||||
ALPINE_VERSION="3.21.7"
|
||||
|
||||
# 4G — server superset (K3s server binary + SQLite datastore working set under
|
||||
# 4G — server superset (K3s server binary + embedded-etcd datastore working set under
|
||||
# /var/lib/rancher/k3s on the root disk). Agents use the same image; the ~10MB
|
||||
# of unused server binary + webhook is dwarfed by the headroom for node images.
|
||||
IMAGE_SIZE="4G"
|
||||
|
||||
# Server superset of both package sets (etcd-ctl is ~5MB, not worth branching).
|
||||
# K3s pulls cgroups + iptables; nats-cli used by k3s-first-boot publisher +
|
||||
# state-report; jq builds the bootstrap-message JSON envelopes; etcd-client
|
||||
# used by the snapshot script (server role only).
|
||||
# K3s pulls cgroups + iptables; jq builds the bootstrap-message JSON envelopes;
|
||||
# etcd-client used by the snapshot script (server role only). The control plane
|
||||
# reaches the host via the AWS gateway (eks-gateway-publish), not a NATS CLI.
|
||||
APK_PACKAGES="ca-certificates curl jq iptables ip6tables conntrack-tools cgroup-tools etcd-ctl openrc"
|
||||
|
||||
# eks-token-webhook binary always installed (agents never start it). The k3s
|
||||
# binary is downloaded + SHA256-verified by setup.sh (network in chroot).
|
||||
BUILD_COMMANDS="CGO_ENABLED=0 go build -o bin/eks-token-webhook ./cmd/eks-token-webhook"
|
||||
# eks-token-webhook binary always installed (agents never start it).
|
||||
# eks-gateway-publish relays the bootstrap envelopes + state reports to the AWS
|
||||
# gateway over SigV4 HTTPS (server role only). The k3s binary is downloaded +
|
||||
# SHA256-verified by setup.sh (network in chroot).
|
||||
BUILD_COMMANDS="CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-token-webhook ./cmd/eks-token-webhook && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-gateway-publish ./cmd/eks-gateway-publish"
|
||||
|
||||
INSTALL_BINARIES="bin/eks-token-webhook:/usr/local/bin/eks-token-webhook"
|
||||
INSTALL_BINARIES="bin/eks-token-webhook:/usr/local/bin/eks-token-webhook bin/eks-gateway-publish:/usr/local/bin/eks-gateway-publish"
|
||||
|
||||
# All role services ship; none but the selector are enabled at bake time.
|
||||
# eks-node-role picks server vs agent at first boot and enables the rest.
|
||||
@@ -30,12 +32,13 @@ scripts/images/eks-node/k3s-agent.initd:/etc/init.d/k3s-agent \
|
||||
scripts/images/eks-node/eks-token-webhook.initd:/etc/init.d/eks-token-webhook \
|
||||
scripts/images/eks-node/k3s-first-boot.initd:/etc/init.d/k3s-first-boot \
|
||||
scripts/images/eks-node/k3s-first-boot.sh:/usr/local/sbin/k3s-first-boot \
|
||||
scripts/images/eks-node/mulga-eks-state-report.sh:/etc/periodic/15min/mulga-eks-state-report \
|
||||
scripts/images/eks-node/mulga-eks-state-report.initd:/etc/init.d/mulga-eks-state-report \
|
||||
scripts/images/eks-node/mulga-eks-state-report.sh:/usr/local/sbin/mulga-eks-state-report \
|
||||
scripts/images/eks-node/mulga-eks-etcd-snapshot.sh:/etc/periodic/daily/mulga-eks-etcd-snapshot"
|
||||
|
||||
# Only crond (state-report) + the role selector at bake. The selector adds the
|
||||
# role-specific services (k3s/eks-token-webhook/k3s-first-boot or k3s-agent) to
|
||||
# the default runlevel on first boot.
|
||||
# Only crond (etcd snapshot) + the role selector at bake. The selector adds the
|
||||
# role-specific services (k3s/eks-token-webhook/k3s-first-boot/state-report or
|
||||
# k3s-agent) to the default runlevel on first boot.
|
||||
ENABLE_SERVICES="crond eks-node-role"
|
||||
|
||||
SETUP_SCRIPT="scripts/images/eks-node/setup.sh"
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
#!/sbin/openrc-run
|
||||
# mulga-eks-state-report — server-role service that publishes K3s health +
|
||||
# node_count to the management NATS bus every STATE_REPORT_INTERVAL seconds. The
|
||||
# spinifex cluster reconciler consumes this to gate CREATING → ACTIVE and to
|
||||
# track nodegroup join/scale, since the apiserver (bound to the VPC node-ip) is
|
||||
# unreachable from the host. Enabled on the server role by eks-node-role.sh.
|
||||
|
||||
description="Mulga EKS control-plane state reporter"
|
||||
command="/usr/local/sbin/mulga-eks-state-report"
|
||||
command_background="yes"
|
||||
pidfile="/run/mulga-eks-state-report.pid"
|
||||
output_log="/var/log/mulga-eks-state-report.log"
|
||||
error_log="/var/log/mulga-eks-state-report.log"
|
||||
|
||||
# Cadence the report loop sleeps between publishes. 30s is well under the
|
||||
# reconciler's 90s stale window, so a missed tick does not flap health.
|
||||
export STATE_REPORT_INTERVAL="${STATE_REPORT_INTERVAL:-30}"
|
||||
|
||||
depend() {
|
||||
need net
|
||||
after k3s
|
||||
}
|
||||
|
||||
start_pre() {
|
||||
if [ ! -f /etc/spinifex-eks/first-boot.env ]; then
|
||||
eerror "/etc/spinifex-eks/first-boot.env not present — cloud-init did not seed NATS creds"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
@@ -1,47 +1,61 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
# mulga-eks-state-report — periodic NATS publish of K3s server health.
|
||||
# Invoked every 15 minutes via /etc/periodic/15min (Alpine crond). One-shot
|
||||
# best-effort; failures log but do not retry — the spinifex reconciler also
|
||||
# does its own NLB healthcheck so this is supplementary signal.
|
||||
# mulga-eks-state-report — publishes K3s server health to the spinifex cluster
|
||||
# reconciler through the AWS gateway. The apiserver binds the VPC node-ip,
|
||||
# unreachable from the host-side daemon, so the daemon cannot probe /healthz
|
||||
# directly — this report carries {healthz, node_count}. eks-gateway-publish
|
||||
# SigV4-signs the POST to the gateway, which relays it onto the management NATS
|
||||
# bus the daemon shares; the VM never speaks NATS directly.
|
||||
#
|
||||
# Subject: eks.state.{accountID}.{clusterName}.server
|
||||
# Subject (gateway-side): eks.state.{accountID}.{clusterName}.server
|
||||
# Payload: JSON { healthz: "ok"|"fail", node_count: N, ts: <unix-seconds> }
|
||||
#
|
||||
# Reads gateway creds + cluster identity from the cloud-init-seeded first-boot
|
||||
# env (same vars k3s-first-boot.sh publishes its bootstrap artifacts with). Run
|
||||
# as a loop by the mulga-eks-state-report OpenRC service when
|
||||
# STATE_REPORT_INTERVAL is set; one-shot otherwise (e.g. a manual invocation).
|
||||
|
||||
ENVFILE=/etc/spinifex-eks/state-report.env
|
||||
ENVFILE=/etc/spinifex-eks/first-boot.env
|
||||
[ -f "${ENVFILE}" ] || { logger -t mulga-eks-state-report "${ENVFILE} missing — exiting"; exit 0; }
|
||||
# set -a so the sourced KEY=value lines are exported to the eks-gateway-publish
|
||||
# child (it reads EKS_ACCOUNT_ID etc. from its environment); a bare source
|
||||
# leaves them unexported and the helper exits "--account-id is required".
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
. "${ENVFILE}"
|
||||
set +a
|
||||
|
||||
: "${SPINIFEX_NATS_URL:?}"
|
||||
: "${EKS_GATEWAY_URL:?}"
|
||||
: "${EKS_ACCESS_KEY:?}"
|
||||
: "${EKS_SECRET_KEY:?}"
|
||||
: "${EKS_ACCOUNT_ID:?}"
|
||||
: "${EKS_CLUSTER_NAME:?}"
|
||||
|
||||
KUBECONFIG=${KUBECONFIG:-/etc/rancher/k3s/k3s.yaml}
|
||||
export KUBECONFIG
|
||||
|
||||
if curl -sk --max-time 3 https://127.0.0.1:6443/healthz | grep -q '^ok$'; then
|
||||
HEALTH=ok
|
||||
publish_report() {
|
||||
if curl -sk --max-time 3 https://127.0.0.1:6443/healthz | grep -q '^ok$'; then
|
||||
health=ok
|
||||
# Count Ready nodes only — k3s retains terminated workers as NotReady
|
||||
# until manually pruned, so a raw node count never drops on scale-down.
|
||||
node_count=$(kubectl get nodes --no-headers 2>/dev/null | awk '$2=="Ready"' | wc -l | tr -d ' ')
|
||||
else
|
||||
health=fail
|
||||
node_count=0
|
||||
fi
|
||||
payload=$(printf '{"healthz":"%s","node_count":%s,"ts":%s}' "${health}" "${node_count}" "$(date +%s)")
|
||||
printf '%s' "${payload}" | eks-gateway-publish -channel state 2>&1 \
|
||||
| logger -t mulga-eks-state-report
|
||||
}
|
||||
|
||||
# STATE_REPORT_INTERVAL set → run forever (service mode); unset → one-shot.
|
||||
if [ -n "${STATE_REPORT_INTERVAL:-}" ] && [ "${STATE_REPORT_INTERVAL}" -gt 0 ] 2>/dev/null; then
|
||||
while true; do
|
||||
publish_report
|
||||
sleep "${STATE_REPORT_INTERVAL}"
|
||||
done
|
||||
else
|
||||
HEALTH=fail
|
||||
publish_report
|
||||
fi
|
||||
|
||||
if [ "${HEALTH}" = "ok" ]; then
|
||||
NODE_COUNT=$(kubectl get nodes --no-headers 2>/dev/null | wc -l | tr -d ' ')
|
||||
else
|
||||
NODE_COUNT=0
|
||||
fi
|
||||
|
||||
TS=$(date +%s)
|
||||
PAYLOAD=$(printf '{"healthz":"%s","node_count":%s,"ts":%s}' "${HEALTH}" "${NODE_COUNT}" "${TS}")
|
||||
SUBJ="eks.state.${EKS_ACCOUNT_ID}.${EKS_CLUSTER_NAME}.server"
|
||||
|
||||
NATS_ARGS="-s ${SPINIFEX_NATS_URL}"
|
||||
if [ -n "${SPINIFEX_NATS_CREDS_FILE:-}" ] && [ -f "${SPINIFEX_NATS_CREDS_FILE}" ]; then
|
||||
NATS_ARGS="${NATS_ARGS} --creds ${SPINIFEX_NATS_CREDS_FILE}"
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
printf '%s' "${PAYLOAD}" | nats ${NATS_ARGS} pub "${SUBJ}" --stdin 2>&1 \
|
||||
| logger -t mulga-eks-state-report
|
||||
|
||||
@@ -1,16 +1,17 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
# setup.sh — chroot customisation for the unified eks-node AMI (server + agent).
|
||||
# setup.sh — guest customisation for the unified eks-node AMI (server + agent).
|
||||
#
|
||||
# Runs inside an Alpine 3.21 chroot under build-system-image.sh after packages
|
||||
# and binaries are installed. Downloads the pinned K3s binary, verifies its
|
||||
# SHA256, drops it into /usr/local/bin, sets executable bits on all role init
|
||||
# scripts + cron entries, and writes the K3s server config skeleton. The role
|
||||
# (server vs agent) is selected per-instance at first boot by eks-node-role.
|
||||
# Runs inside the libguestfs appliance (via virt-customize --run) under
|
||||
# build-system-image.sh after packages and binaries are installed. Downloads the
|
||||
# pinned K3s binary, verifies its SHA256, drops it into /usr/local/bin, sets
|
||||
# executable bits on all role init scripts + cron entries, and writes the K3s
|
||||
# server config skeleton. The role (server vs agent) is selected per-instance at
|
||||
# first boot by eks-node-role.
|
||||
#
|
||||
# Network access inside the chroot is provided by the host's /etc/resolv.conf
|
||||
# (build-system-image.sh copies it in). curl is in APK_PACKAGES.
|
||||
# Network access inside the appliance is provided by virt-customize --network.
|
||||
# curl is in APK_PACKAGES.
|
||||
|
||||
K3S_VERSION="v1.32.5+k3s1"
|
||||
K3S_URL_BASE="https://github.com/k3s-io/k3s/releases/download/${K3S_VERSION}"
|
||||
@@ -39,25 +40,12 @@ ln -sf /usr/local/bin/k3s /usr/local/bin/kubectl
|
||||
ln -sf /usr/local/bin/k3s /usr/local/bin/crictl
|
||||
ln -sf /usr/local/bin/k3s /usr/local/bin/ctr
|
||||
|
||||
# nats-cli is not in Alpine main/community — pull the upstream release binary.
|
||||
# Used by k3s-first-boot.sh (one-shot publishes) and mulga-eks-state-report.sh
|
||||
# (periodic publishes).
|
||||
NATS_CLI_VERSION="0.4.0"
|
||||
NATS_CLI_URL="https://github.com/nats-io/natscli/releases/download/v${NATS_CLI_VERSION}/nats-${NATS_CLI_VERSION}-linux-amd64.zip"
|
||||
echo "[eks-node-setup] downloading nats-cli ${NATS_CLI_VERSION}"
|
||||
curl -fsSL -o /tmp/nats-cli.zip "${NATS_CLI_URL}"
|
||||
apk add --no-cache unzip
|
||||
unzip -q -d /tmp/nats-cli /tmp/nats-cli.zip
|
||||
install -m 0755 /tmp/nats-cli/nats-${NATS_CLI_VERSION}-linux-amd64/nats /usr/local/bin/nats
|
||||
rm -rf /tmp/nats-cli /tmp/nats-cli.zip
|
||||
apk del --no-cache unzip
|
||||
|
||||
# Init scripts ship as 0644 from INSTALL_FILES; OpenRC requires 0755. Every
|
||||
# role's services are baked; the selector enables the right ones at first boot.
|
||||
chmod 0755 /etc/init.d/eks-node-role /etc/init.d/k3s /etc/init.d/k3s-agent \
|
||||
/etc/init.d/eks-token-webhook /etc/init.d/k3s-first-boot
|
||||
chmod 0755 /usr/local/sbin/eks-node-role /usr/local/sbin/k3s-first-boot
|
||||
chmod 0755 /etc/periodic/15min/mulga-eks-state-report
|
||||
/etc/init.d/eks-token-webhook /etc/init.d/k3s-first-boot /etc/init.d/mulga-eks-state-report
|
||||
chmod 0755 /usr/local/sbin/eks-node-role /usr/local/sbin/k3s-first-boot \
|
||||
/usr/local/sbin/mulga-eks-state-report
|
||||
chmod 0755 /etc/periodic/daily/mulga-eks-etcd-snapshot
|
||||
|
||||
# K3s server config — skeleton; cloud-init / first-boot fills in the
|
||||
@@ -72,7 +60,9 @@ mkdir -p /etc/rancher/k3s
|
||||
cat > /etc/rancher/k3s/config.yaml.skel <<'EOF'
|
||||
# Populated at first boot by cloud-init user-data via k3s-first-boot.sh.
|
||||
# Fields documented at https://docs.k3s.io/installation/configuration.
|
||||
# No cluster-init: single-node v1 uses the embedded SQLite datastore, not etcd.
|
||||
# cluster-init selects the embedded etcd datastore. cloud-init write_files
|
||||
# normally overrides this skeleton; kept consistent as the boot fallback.
|
||||
cluster-init: true
|
||||
disable:
|
||||
- traefik
|
||||
- servicelb
|
||||
|
||||
Executable
+147
@@ -0,0 +1,147 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# publish-system-image.sh — Publish a built system AMI to Cloudflare R2
|
||||
#
|
||||
# Converts the raw image produced by build-system-image.sh into a compressed
|
||||
# qcow2, writes a .sha256 sidecar (coreutils format, matched by the spx catalog
|
||||
# checksum verifier), and uploads both to the R2 bucket that backs
|
||||
# iso.mulgadc.com/system-ami/. The matching catalog entry in
|
||||
# spinifex/spinifex/utils/images.go then lets operators run:
|
||||
#
|
||||
# spx admin images import --name <catalog-key>
|
||||
#
|
||||
# Object name is <AMI_NAME or IMAGE_NAME>-<arch>.qcow2 — this MUST match the
|
||||
# basename of the catalog entry's URL.
|
||||
#
|
||||
# Requirements: qemu-img, sha256sum, aws (CLI; R2 speaks the S3 API)
|
||||
# Usage: ./scripts/publish-system-image.sh <manifest.conf> [--build] [--dry-run]
|
||||
# --build Run build-system-image.sh first (otherwise reuse the cached raw)
|
||||
# --dry-run Print the convert/checksum/upload steps without uploading
|
||||
#
|
||||
# Env (required unless --dry-run):
|
||||
# R2_ENDPOINT https://<account>.r2.cloudflarestorage.com
|
||||
# R2_BUCKET defaults to "spinifex-iso" (backs iso.mulgadc.com)
|
||||
# AWS_ACCESS_KEY_ID R2 token access key
|
||||
# AWS_SECRET_ACCESS_KEY R2 token secret
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 <manifest.conf> [--build] [--dry-run]"
|
||||
exit 1
|
||||
}
|
||||
|
||||
if [[ $# -lt 1 || "$1" == "-h" || "$1" == "--help" ]]; then
|
||||
usage
|
||||
fi
|
||||
|
||||
MANIFEST="$1"
|
||||
shift
|
||||
|
||||
if [[ ! -f "$MANIFEST" ]]; then
|
||||
echo "ERROR: Manifest not found: $MANIFEST"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
DO_BUILD=false
|
||||
DRY_RUN=false
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--build) DO_BUILD=true ;;
|
||||
--dry-run) DRY_RUN=true ;;
|
||||
*) echo "ERROR: unknown flag: $arg"; usage ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# shellcheck source=/dev/null
|
||||
source "$MANIFEST"
|
||||
|
||||
DISTRO="${DISTRO:-alpine}"
|
||||
ARCH="x86_64" # build-system-image.sh builds x86_64 only today
|
||||
R2_BUCKET="${R2_BUCKET:-spinifex-iso}" # iso.mulgadc.com is served from this bucket
|
||||
|
||||
if [[ -z "${IMAGE_NAME:-}" ]]; then
|
||||
echo "ERROR: Manifest missing required field: IMAGE_NAME"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Mirror build-system-image.sh's derived paths so we reuse its cached raw.
|
||||
BUILD_DIR="/tmp/${IMAGE_NAME}-image-build"
|
||||
RAW_IMAGE="${BUILD_DIR}/${IMAGE_NAME}-${DISTRO}.raw"
|
||||
|
||||
OBJECT_BASE="${AMI_NAME:-$IMAGE_NAME}-${ARCH}"
|
||||
QCOW2_OUT="${BUILD_DIR}/${OBJECT_BASE}.qcow2"
|
||||
SHA_OUT="${QCOW2_OUT}.sha256"
|
||||
OBJECT_QCOW2="${OBJECT_BASE}.qcow2"
|
||||
OBJECT_SHA="${OBJECT_BASE}.qcow2.sha256"
|
||||
|
||||
for tool in qemu-img sha256sum; do
|
||||
if ! command -v "$tool" &>/dev/null; then
|
||||
echo "ERROR: $tool not found"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
if [[ "$DRY_RUN" == false ]] && ! command -v aws &>/dev/null; then
|
||||
echo "ERROR: aws CLI not found (required for R2 upload; use --dry-run to skip)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$DO_BUILD" == true ]]; then
|
||||
echo "=== Building ${IMAGE_NAME} ==="
|
||||
"${SCRIPT_DIR}/build-system-image.sh" "$MANIFEST"
|
||||
fi
|
||||
|
||||
if [[ ! -f "$RAW_IMAGE" ]]; then
|
||||
echo "ERROR: Raw image not found: $RAW_IMAGE"
|
||||
echo " Run with --build, or 'make build-system-image IMAGE=...' first."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
for var in R2_ENDPOINT AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; do
|
||||
if [[ -z "${!var:-}" ]]; then
|
||||
echo "ERROR: $var must be set (or use --dry-run)"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
echo "=== Publish System Image ==="
|
||||
echo " Manifest: ${MANIFEST}"
|
||||
echo " Raw: ${RAW_IMAGE}"
|
||||
echo " Object: s3://${R2_BUCKET}/system-ami/${OBJECT_QCOW2}"
|
||||
echo ""
|
||||
|
||||
run() {
|
||||
echo "+ $*"
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
"$@"
|
||||
fi
|
||||
}
|
||||
|
||||
# Compressed qcow2 keeps the transfer small (consistent with the GPU images).
|
||||
run qemu-img convert -c -f raw -O qcow2 "$RAW_IMAGE" "$QCOW2_OUT"
|
||||
|
||||
# Sidecar in coreutils format: "<hex> <basename>". The spx catalog verifier
|
||||
# matches the entry by the downloaded file's basename, so write the basename
|
||||
# the catalog URL resolves to, not the local build path.
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
( cd "$(dirname "$QCOW2_OUT")" && sha256sum "$(basename "$QCOW2_OUT")" > "$(basename "$SHA_OUT")" )
|
||||
echo " sha256: $(cut -d' ' -f1 "$SHA_OUT")"
|
||||
else
|
||||
echo "+ sha256sum ${OBJECT_QCOW2} > ${OBJECT_SHA}"
|
||||
fi
|
||||
|
||||
S3_PREFIX="s3://${R2_BUCKET}/system-ami"
|
||||
run aws s3 cp --endpoint-url "${R2_ENDPOINT:-<R2_ENDPOINT>}" "$QCOW2_OUT" "${S3_PREFIX}/${OBJECT_QCOW2}"
|
||||
run aws s3 cp --endpoint-url "${R2_ENDPOINT:-<R2_ENDPOINT>}" "$SHA_OUT" "${S3_PREFIX}/${OBJECT_SHA}"
|
||||
|
||||
echo ""
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "=== Dry run complete (no upload) ==="
|
||||
else
|
||||
echo "=== Published ==="
|
||||
echo " https://iso.mulgadc.com/system-ami/${OBJECT_QCOW2}"
|
||||
echo " https://iso.mulgadc.com/system-ami/${OBJECT_SHA}"
|
||||
fi
|
||||
@@ -0,0 +1,61 @@
|
||||
package daemon
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestParsePhysicalCores(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
cpuinfo string
|
||||
want int
|
||||
wantOK bool
|
||||
}{
|
||||
{
|
||||
name: "hyperthreaded single socket: 2 cores / 4 threads",
|
||||
cpuinfo: "processor\t: 0\nphysical id\t: 0\ncore id\t: 0\n\n" +
|
||||
"processor\t: 1\nphysical id\t: 0\ncore id\t: 1\n\n" +
|
||||
"processor\t: 2\nphysical id\t: 0\ncore id\t: 0\n\n" +
|
||||
"processor\t: 3\nphysical id\t: 0\ncore id\t: 1\n",
|
||||
want: 2,
|
||||
wantOK: true,
|
||||
},
|
||||
{
|
||||
name: "hybrid: P-cores share core ids across HT, E-cores distinct",
|
||||
// 2 P-cores (core id 0,1) each 2 threads + 2 E-cores (core id 16,17).
|
||||
cpuinfo: "processor\t: 0\nphysical id\t: 0\ncore id\t: 0\n\n" +
|
||||
"processor\t: 1\nphysical id\t: 0\ncore id\t: 0\n\n" +
|
||||
"processor\t: 2\nphysical id\t: 0\ncore id\t: 1\n\n" +
|
||||
"processor\t: 3\nphysical id\t: 0\ncore id\t: 1\n\n" +
|
||||
"processor\t: 4\nphysical id\t: 0\ncore id\t: 16\n\n" +
|
||||
"processor\t: 5\nphysical id\t: 0\ncore id\t: 17\n",
|
||||
want: 4,
|
||||
wantOK: true,
|
||||
},
|
||||
{
|
||||
name: "dual socket: 2 sockets x 1 core, same core id",
|
||||
cpuinfo: "processor\t: 0\nphysical id\t: 0\ncore id\t: 0\n\n" +
|
||||
"processor\t: 1\nphysical id\t: 1\ncore id\t: 0\n",
|
||||
want: 2,
|
||||
wantOK: true,
|
||||
},
|
||||
{
|
||||
name: "no topology fields (e.g. ARM/container) falls back",
|
||||
cpuinfo: "processor\t: 0\nvendor_id\t: GenuineIntel\n\nprocessor\t: 1\nvendor_id\t: GenuineIntel\n",
|
||||
want: 0,
|
||||
wantOK: false,
|
||||
},
|
||||
{
|
||||
name: "empty",
|
||||
cpuinfo: "",
|
||||
want: 0,
|
||||
wantOK: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, ok := parsePhysicalCores([]byte(tt.cpuinfo))
|
||||
if got != tt.want || ok != tt.wantOK {
|
||||
t.Fatalf("parsePhysicalCores() = (%d, %v), want (%d, %v)", got, ok, tt.want, tt.wantOK)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+77
-10
@@ -85,9 +85,9 @@ type EBS struct {
|
||||
// when full, it unsubscribes so NATS routes requests to other nodes.
|
||||
type ResourceManager struct {
|
||||
mu sync.RWMutex
|
||||
// hostVCPU / hostMemGB are the raw figures reported by the host
|
||||
// (runtime.NumCPU, /proc/meminfo). Schedulable capacity for guest VMs
|
||||
// is host - reserved - allocated.
|
||||
// hostVCPU / hostMemGB are the host figures guests schedule against:
|
||||
// physical cores (physicalCoreCount, not SMT threads) and /proc/meminfo.
|
||||
// Schedulable capacity for guest VMs is host - reserved - allocated.
|
||||
hostVCPU int
|
||||
hostMemGB float64
|
||||
// reservedVCPU / reservedMem are held back from guest scheduling for
|
||||
@@ -403,6 +403,69 @@ func getSystemMemory() (float64, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// physicalCoreCount returns the number of physical CPU cores. Guest vCPUs are
|
||||
// scheduled 1:1 against physical cores rather than SMT threads:
|
||||
// runtime.NumCPU() counts logical threads, so on an SMT or hybrid host (e.g.
|
||||
// 12 cores / 16 threads) it would let guest vCPUs pack past the physical core
|
||||
// count and thrash them during the boot spike. Parses /proc/cpuinfo, counting
|
||||
// distinct (physical id, core id) pairs. Falls back to runtime.NumCPU() on
|
||||
// non-Linux hosts or when the topology fields are absent (some VMs/containers
|
||||
// omit them).
|
||||
func physicalCoreCount() int {
|
||||
logical := runtime.NumCPU()
|
||||
if runtime.GOOS != "linux" {
|
||||
return logical
|
||||
}
|
||||
data, err := os.ReadFile("/proc/cpuinfo")
|
||||
if err != nil {
|
||||
slog.Warn("physical core detection failed, scheduling against logical CPUs",
|
||||
"err", err, "logicalCPU", logical)
|
||||
return logical
|
||||
}
|
||||
if n, ok := parsePhysicalCores(data); ok {
|
||||
return n
|
||||
}
|
||||
return logical
|
||||
}
|
||||
|
||||
// parsePhysicalCores counts distinct (physical id, core id) pairs in
|
||||
// /proc/cpuinfo contents — i.e. physical cores, with SMT siblings collapsed.
|
||||
// Returns ok=false when the topology fields are absent (some VMs/containers
|
||||
// omit them) so the caller can fall back to the logical CPU count.
|
||||
func parsePhysicalCores(data []byte) (int, bool) {
|
||||
cores := make(map[string]struct{})
|
||||
var phys, core string
|
||||
sawCore := false
|
||||
flush := func() {
|
||||
if core != "" {
|
||||
cores[phys+":"+core] = struct{}{}
|
||||
}
|
||||
phys, core = "", ""
|
||||
}
|
||||
for line := range strings.SplitSeq(string(data), "\n") {
|
||||
if strings.TrimSpace(line) == "" {
|
||||
flush()
|
||||
continue
|
||||
}
|
||||
key, val, ok := strings.Cut(line, ":")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
switch strings.TrimSpace(key) {
|
||||
case "physical id":
|
||||
phys = strings.TrimSpace(val)
|
||||
case "core id":
|
||||
core = strings.TrimSpace(val)
|
||||
sawCore = true
|
||||
}
|
||||
}
|
||||
flush()
|
||||
if !sawCore || len(cores) == 0 {
|
||||
return 0, false
|
||||
}
|
||||
return len(cores), true
|
||||
}
|
||||
|
||||
// NewResourceManager creates a new resource manager with system capabilities.
|
||||
// Returns an error if system memory cannot be detected, since an incorrect
|
||||
// default would either under-provision (large servers) or over-commit (small devices).
|
||||
@@ -412,8 +475,11 @@ func getSystemMemory() (float64, error) {
|
||||
// gpuModels is the list of recognised GPU models present on the host; pass nil if
|
||||
// GPU passthrough is disabled or no GPUs were found.
|
||||
func NewResourceManager(gpuModels []instancetypes.GPUModel, migProfiles []instancetypes.MIGProfileSpec, gpuMgr *gpu.Manager) (*ResourceManager, error) {
|
||||
// Get system CPU cores
|
||||
numCPU := runtime.NumCPU()
|
||||
// Schedule guest vCPUs against physical cores, not SMT threads (see
|
||||
// physicalCoreCount), so a hyperthreaded/hybrid host can't be packed past
|
||||
// what it can actually run. SPINIFEX_HOST_VCPU overrides detection for
|
||||
// hosts that misreport topology.
|
||||
hostVCPU := resolveHostVCPU(os.Getenv, physicalCoreCount())
|
||||
|
||||
// Get system memory (in GB)
|
||||
totalMemGB, err := getSystemMemory()
|
||||
@@ -422,10 +488,10 @@ func NewResourceManager(gpuModels []instancetypes.GPUModel, migProfiles []instan
|
||||
}
|
||||
|
||||
reserve := resolveHostReserve(os.Getenv)
|
||||
reservedVCPU, reservedMem, err := applyHostReserve(reserve, numCPU, totalMemGB)
|
||||
reservedVCPU, reservedMem, err := applyHostReserve(reserve, hostVCPU, totalMemGB)
|
||||
if err != nil {
|
||||
slog.Error("host below minimum reserve — daemon refuses to start",
|
||||
"err", err, "hostVCPU", numCPU, "hostMemGB", totalMemGB,
|
||||
"err", err, "hostVCPU", hostVCPU, "hostMemGB", totalMemGB,
|
||||
"reserveVCPU", reserve.vCPU, "reserveMemGB", reserve.memGB)
|
||||
return nil, fmt.Errorf("validate host reserve: %w", err)
|
||||
}
|
||||
@@ -444,13 +510,13 @@ func NewResourceManager(gpuModels []instancetypes.GPUModel, migProfiles []instan
|
||||
}
|
||||
|
||||
slog.Info("System resources detected",
|
||||
"hostVCPU", numCPU, "hostMemGB", totalMemGB,
|
||||
"hostVCPU", hostVCPU, "logicalCPU", runtime.NumCPU(), "hostMemGB", totalMemGB,
|
||||
"reservedVCPU", reservedVCPU, "reservedMemGB", reservedMem,
|
||||
"schedulableVCPU", numCPU-reservedVCPU, "schedulableMemGB", totalMemGB-reservedMem,
|
||||
"schedulableVCPU", hostVCPU-reservedVCPU, "schedulableMemGB", totalMemGB-reservedMem,
|
||||
"instanceTypes", len(instanceTypes))
|
||||
|
||||
return &ResourceManager{
|
||||
hostVCPU: numCPU,
|
||||
hostVCPU: hostVCPU,
|
||||
hostMemGB: totalMemGB,
|
||||
reservedVCPU: reservedVCPU,
|
||||
reservedMem: reservedMem,
|
||||
@@ -853,6 +919,7 @@ func (d *Daemon) subscribeAll() error {
|
||||
natsSub{"elbv2.DescribeLoadBalancerAttributes", d.handleELBv2DescribeLoadBalancerAttributes, "spinifex-workers"},
|
||||
natsSub{"elbv2.SetSecurityGroups", d.handleELBv2SetSecurityGroups, "spinifex-workers"},
|
||||
natsSub{"elbv2.SetIpAddressType", d.handleELBv2SetIpAddressType, "spinifex-workers"},
|
||||
natsSub{"elbv2.SetSubnets", d.handleELBv2SetSubnets, "spinifex-workers"},
|
||||
natsSub{"elbv2.AddListenerCertificates", d.handleELBv2AddListenerCertificates, "spinifex-workers"},
|
||||
natsSub{"elbv2.RemoveListenerCertificates", d.handleELBv2RemoveListenerCertificates, "spinifex-workers"},
|
||||
natsSub{"elbv2.DescribeListenerCertificates", d.handleELBv2DescribeListenerCertificates, "spinifex-workers"},
|
||||
|
||||
@@ -61,6 +61,11 @@ func TestDaemonHandleEKS_AllHandlersDispatchToService(t *testing.T) {
|
||||
// no error, so wantCode is "" (success — assert a non-error payload).
|
||||
notImpl := awserrors.ErrorNotImplemented
|
||||
invalid := awserrors.ErrorInvalidParameterValue
|
||||
// Nodegroup mutators gate on orchestration deps, which the shim service
|
||||
// lacks → ServiceUnavailable. The read paths reach input validation and an
|
||||
// empty body fails with InvalidParameterValue. UpdateNodegroupVersion stays
|
||||
// NotImplemented (v1 doesn't do AMI upgrades).
|
||||
unavailable := awserrors.ErrorServiceUnavailable
|
||||
cases := []struct {
|
||||
subject string
|
||||
handler nats.MsgHandler
|
||||
@@ -68,12 +73,12 @@ func TestDaemonHandleEKS_AllHandlersDispatchToService(t *testing.T) {
|
||||
}{
|
||||
{"eks.UpdateClusterConfig", d.handleEKSUpdateClusterConfig, notImpl},
|
||||
{"eks.UpdateClusterVersion", d.handleEKSUpdateClusterVersion, notImpl},
|
||||
{"eks.CreateNodegroup", d.handleEKSCreateNodegroup, notImpl},
|
||||
{"eks.DescribeNodegroup", d.handleEKSDescribeNodegroup, notImpl},
|
||||
{"eks.ListNodegroups", d.handleEKSListNodegroups, notImpl},
|
||||
{"eks.UpdateNodegroupConfig", d.handleEKSUpdateNodegroupConfig, notImpl},
|
||||
{"eks.CreateNodegroup", d.handleEKSCreateNodegroup, unavailable},
|
||||
{"eks.DescribeNodegroup", d.handleEKSDescribeNodegroup, invalid},
|
||||
{"eks.ListNodegroups", d.handleEKSListNodegroups, invalid},
|
||||
{"eks.UpdateNodegroupConfig", d.handleEKSUpdateNodegroupConfig, unavailable},
|
||||
{"eks.UpdateNodegroupVersion", d.handleEKSUpdateNodegroupVersion, notImpl},
|
||||
{"eks.DeleteNodegroup", d.handleEKSDeleteNodegroup, notImpl},
|
||||
{"eks.DeleteNodegroup", d.handleEKSDeleteNodegroup, unavailable},
|
||||
{"eks.CreateAccessEntry", d.handleEKSCreateAccessEntry, invalid},
|
||||
{"eks.DescribeAccessEntry", d.handleEKSDescribeAccessEntry, invalid},
|
||||
{"eks.ListAccessEntries", d.handleEKSListAccessEntries, invalid},
|
||||
|
||||
@@ -122,6 +122,10 @@ func (d *Daemon) handleELBv2SetIpAddressType(msg *nats.Msg) {
|
||||
handleNATSRequest(msg, d.elbv2Service.SetIpAddressType)
|
||||
}
|
||||
|
||||
func (d *Daemon) handleELBv2SetSubnets(msg *nats.Msg) {
|
||||
handleNATSRequest(msg, d.elbv2Service.SetSubnets)
|
||||
}
|
||||
|
||||
func (d *Daemon) handleELBv2AddListenerCertificates(msg *nats.Msg) {
|
||||
handleNATSRequest(msg, d.elbv2Service.AddListenerCertificates)
|
||||
}
|
||||
|
||||
@@ -67,6 +67,15 @@ func (d *Daemon) subscribeSystemTerminate(instanceID string) error {
|
||||
}
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
return d.subscribeSystemTerminateLocked(instanceID)
|
||||
}
|
||||
|
||||
// subscribeSystemTerminateLocked is the body of subscribeSystemTerminate for
|
||||
// callers that already hold d.mu (e.g. onInstanceUpHook). Idempotent.
|
||||
func (d *Daemon) subscribeSystemTerminateLocked(instanceID string) error {
|
||||
if d.natsConn == nil {
|
||||
return nil
|
||||
}
|
||||
subject := fmt.Sprintf("system.TerminateInstance.%s", instanceID)
|
||||
if _, exists := d.natsSubscriptions[subject]; exists {
|
||||
return nil
|
||||
|
||||
@@ -2,7 +2,10 @@ package daemon
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
|
||||
)
|
||||
|
||||
@@ -28,4 +31,22 @@ func (a *daemonEKSSubnetResolver) GetSubnetVPC(accountID, subnetID string) (stri
|
||||
return rec.VpcId, nil
|
||||
}
|
||||
|
||||
func (a *daemonEKSSubnetResolver) GetVPCCIDR(accountID, vpcID string) (string, error) {
|
||||
if a.d == nil || a.d.vpcService == nil {
|
||||
return "", errors.New("eks: VPC service not initialized")
|
||||
}
|
||||
out, err := a.d.vpcService.DescribeVpcs(&ec2.DescribeVpcsInput{
|
||||
VpcIds: aws.StringSlice([]string{vpcID}),
|
||||
}, accountID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, v := range out.Vpcs {
|
||||
if v != nil && aws.StringValue(v.VpcId) == vpcID {
|
||||
return aws.StringValue(v.CidrBlock), nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("eks: VPC %s not found", vpcID)
|
||||
}
|
||||
|
||||
var _ handlers_eks.SubnetVPCResolver = (*daemonEKSSubnetResolver)(nil)
|
||||
|
||||
+40
-53
@@ -29,36 +29,39 @@ func (d *Daemon) buildEKSServiceDeps() handlers_eks.EKSServiceDeps {
|
||||
masterKey = nil
|
||||
}
|
||||
|
||||
// CA PEM the K3s server VM uses to verify the daemon's NATS TLS. Same CA
|
||||
// that signs the AWSGW server cert; read here so the VM can publish its
|
||||
// one-shot bootstrap messages over the token+TLS NATS endpoint.
|
||||
natsCA := ""
|
||||
// CA PEM the K3s server VM uses to verify the AWS gateway's HTTPS cert when
|
||||
// the eks-gateway-publish helper POSTs its bootstrap envelopes + state
|
||||
// reports. The same CA signs the AWSGW server cert and the NATS server cert,
|
||||
// so it is read from the configured NATS CA path.
|
||||
gatewayCA := ""
|
||||
if d.config.NATS.CACert != "" {
|
||||
if caBytes, readErr := os.ReadFile(d.config.NATS.CACert); readErr == nil {
|
||||
natsCA = string(caBytes)
|
||||
gatewayCA = string(caBytes)
|
||||
} else {
|
||||
slog.Warn("EKS: read NATS CACert failed; K3s server VM will not reach NATS over TLS",
|
||||
slog.Warn("EKS: read gateway CACert failed; K3s server VM will not verify the gateway over TLS",
|
||||
"path", d.config.NATS.CACert, "err", readErr)
|
||||
}
|
||||
}
|
||||
|
||||
return handlers_eks.EKSServiceDeps{
|
||||
Config: d.config,
|
||||
NATSConn: d.natsConn,
|
||||
MasterKey: masterKey,
|
||||
GatewayBaseURL: d.resolveGatewayBaseURL(),
|
||||
Region: d.config.Region,
|
||||
HolderID: d.node,
|
||||
NATSURL: d.resolveSystemNATSURL(),
|
||||
NATSToken: d.config.NATS.ACL.Token,
|
||||
NATSCACert: natsCA,
|
||||
VPCSG: d.vpcService,
|
||||
VPCK3s: d.vpcService,
|
||||
VPCSubnet: &daemonEKSSubnetResolver{d: d},
|
||||
NLB: d.elbv2Service,
|
||||
Instance: d,
|
||||
Image: d.imageService,
|
||||
EIP: d.eipService,
|
||||
Config: d.config,
|
||||
NATSConn: d.natsConn,
|
||||
MasterKey: masterKey,
|
||||
GatewayBaseURL: d.resolveGatewayBaseURL(),
|
||||
Region: d.config.Region,
|
||||
HolderID: d.node,
|
||||
SystemGatewayURL: d.resolveSystemGatewayBaseURL(),
|
||||
SystemAccessKey: d.config.Predastore.AccessKey,
|
||||
SystemSecretKey: d.config.Predastore.SecretKey,
|
||||
GatewayCACert: gatewayCA,
|
||||
VPCSG: d.vpcService,
|
||||
VPCK3s: d.vpcService,
|
||||
VPCSubnet: &daemonEKSSubnetResolver{d: d},
|
||||
NLB: d.elbv2Service,
|
||||
Instance: d,
|
||||
Image: d.imageService,
|
||||
EIP: d.eipService,
|
||||
Worker: d,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -120,40 +123,24 @@ func (d *Daemon) resolveGatewayBaseURL() string {
|
||||
return "https://" + net.JoinHostPort(host, gatewayPort)
|
||||
}
|
||||
|
||||
// resolveSystemNATSURL derives the NATS URL a system instance (the K3s server
|
||||
// VM) dials to publish its one-shot bootstrap messages. Unlike a customer VM,
|
||||
// resolveSystemGatewayBaseURL derives the AWS gateway URL a system instance
|
||||
// (the K3s server VM) POSTs its bootstrap envelopes + state reports to. Unlike
|
||||
// a customer LB VM — which reaches the gateway via VPC→external (AdvertiseIP) —
|
||||
// a system instance reaches the daemon only over the mgmt bridge: its cloud-init
|
||||
// mgmt0 NIC is on-link to mgmtBridgeIP with no off-link route, so NATS must be
|
||||
// dialed at the bridge address directly. NATS listens on 0.0.0.0:4222, so the
|
||||
// bridge IP is reachable. Falls back to resolveNATSURL() (gateway-host
|
||||
// precedence) when no mgmt bridge exists — e.g. dev-shim networking.
|
||||
func (d *Daemon) resolveSystemNATSURL() string {
|
||||
// mgmt0 NIC is on-link to mgmtBridgeIP with no off-link route, so the gateway
|
||||
// must be dialed at the bridge address directly. AWSGW binds 0.0.0.0 (or the
|
||||
// mgmt IP), and the server cert SANs include the bridge IP, so HTTPS validates.
|
||||
// Falls back to resolveGatewayBaseURL() (gateway-host precedence) when no mgmt
|
||||
// bridge exists — e.g. dev-shim networking.
|
||||
func (d *Daemon) resolveSystemGatewayBaseURL() string {
|
||||
if d.mgmtBridgeIP == "" {
|
||||
return d.resolveNATSURL()
|
||||
return d.resolveGatewayBaseURL()
|
||||
}
|
||||
natsPort := "4222"
|
||||
if d.config.NATS.Host != "" {
|
||||
if _, port, splitErr := net.SplitHostPort(d.config.NATS.Host); splitErr == nil && port != "" {
|
||||
natsPort = port
|
||||
gatewayPort := "9999"
|
||||
if d.config.AWSGW.Host != "" {
|
||||
if _, port, splitErr := net.SplitHostPort(d.config.AWSGW.Host); splitErr == nil && port != "" {
|
||||
gatewayPort = port
|
||||
}
|
||||
}
|
||||
return "nats://" + net.JoinHostPort(d.mgmtBridgeIP, natsPort)
|
||||
}
|
||||
|
||||
// resolveNATSURL derives the NATS URL the K3s server VM dials to publish its
|
||||
// one-shot bootstrap messages. Host reuses the gateway host resolution; port
|
||||
// comes from the configured NATS listen address (default 4222). Returns "" if
|
||||
// no reachable host can be derived.
|
||||
func (d *Daemon) resolveNATSURL() string {
|
||||
host := d.resolveGatewayHost()
|
||||
if host == "" {
|
||||
return ""
|
||||
}
|
||||
natsPort := "4222"
|
||||
if d.config.NATS.Host != "" {
|
||||
if _, port, splitErr := net.SplitHostPort(d.config.NATS.Host); splitErr == nil && port != "" {
|
||||
natsPort = port
|
||||
}
|
||||
}
|
||||
return "nats://" + net.JoinHostPort(host, natsPort)
|
||||
return "https://" + net.JoinHostPort(d.mgmtBridgeIP, gatewayPort)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
|
||||
spxtypes "github.com/mulgadc/spinifex/spinifex/types"
|
||||
)
|
||||
|
||||
// Compile-time check that Daemon satisfies the EKS worker-launch surface.
|
||||
var _ handlers_eks.WorkerLauncher = (*Daemon)(nil)
|
||||
|
||||
// RunWorkerInstance launches a nodegroup worker through the normal customer
|
||||
// RunInstances path: customer-owned, primary ENI auto-created in input.SubnetId
|
||||
// with input.SecurityGroupIds, no ManagedBy tag, no mgmt NIC — so the worker is
|
||||
// visible in DescribeInstances and reclaimed by TerminateInstances like any
|
||||
// other customer EC2. The EKS service hands us raw cloud-config YAML in
|
||||
// input.UserData; RunInstances expects it base64-encoded, so wrap it here.
|
||||
func (d *Daemon) RunWorkerInstance(input *ec2.RunInstancesInput, accountID string) (*ec2.Reservation, error) {
|
||||
if d.instanceService == nil {
|
||||
return nil, errors.New("eks worker: instance service not initialized")
|
||||
}
|
||||
if input == nil {
|
||||
return nil, errors.New("eks worker: nil RunInstancesInput")
|
||||
}
|
||||
if input.UserData != nil && *input.UserData != "" {
|
||||
input.UserData = aws.String(base64.StdEncoding.EncodeToString([]byte(*input.UserData)))
|
||||
}
|
||||
return d.instanceService.RunInstances(input, accountID)
|
||||
}
|
||||
|
||||
// TerminateWorkerInstances terminates each nodegroup worker via the same path
|
||||
// the EC2 NATS terminate handler uses. A worker no longer tracked by the local
|
||||
// vmMgr is treated as already-gone (idempotent success) so a retried
|
||||
// DeleteNodegroup / scale-down does not wedge on instances that already drained.
|
||||
func (d *Daemon) TerminateWorkerInstances(instanceIDs []string, accountID string) error {
|
||||
if d.instanceService == nil {
|
||||
return errors.New("eks worker: instance service not initialized")
|
||||
}
|
||||
var errs []error
|
||||
for _, id := range instanceIDs {
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
instance, ok := d.vmMgr.Get(id)
|
||||
if !ok {
|
||||
slog.Debug("TerminateWorkerInstances: instance already gone", "instanceId", id)
|
||||
continue
|
||||
}
|
||||
cmd := spxtypes.EC2InstanceCommand{
|
||||
ID: id,
|
||||
Attributes: spxtypes.EC2CommandAttributes{TerminateInstance: true},
|
||||
}
|
||||
if err := d.instanceService.StopOrTerminateInstance(instance, cmd); err != nil {
|
||||
errs = append(errs, fmt.Errorf("terminate worker %s: %w", id, err))
|
||||
}
|
||||
}
|
||||
if len(errs) > 0 {
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
handlers_ec2_instance "github.com/mulgadc/spinifex/spinifex/handlers/ec2/instance"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// A worker whose ID is not tracked by the local vmMgr is already-gone, so
|
||||
// terminating it is an idempotent no-op success (a retried DeleteNodegroup /
|
||||
// scale-down must not wedge on drained instances).
|
||||
func TestTerminateWorkerInstances_NotFoundIsIdempotent(t *testing.T) {
|
||||
d := newDaemonWithVMs()
|
||||
// Non-nil service passes the init guard; no VM matches the IDs so
|
||||
// StopOrTerminateInstance is never reached — the not-found branch returns
|
||||
// success.
|
||||
d.instanceService = &handlers_ec2_instance.InstanceServiceImpl{}
|
||||
|
||||
require.NoError(t, d.TerminateWorkerInstances([]string{"i-gone1", "i-gone2"}, "111122223333"))
|
||||
// Empty / blank IDs are skipped.
|
||||
require.NoError(t, d.TerminateWorkerInstances([]string{"", ""}, "111122223333"))
|
||||
require.NoError(t, d.TerminateWorkerInstances(nil, "111122223333"))
|
||||
}
|
||||
|
||||
// Both worker methods must guard against an uninitialized instance service
|
||||
// rather than nil-panic.
|
||||
func TestWorkerLauncher_NilInstanceService(t *testing.T) {
|
||||
d := &Daemon{}
|
||||
|
||||
_, err := d.RunWorkerInstance(&ec2.RunInstancesInput{ImageId: aws.String("ami-1")}, "111122223333")
|
||||
require.Error(t, err)
|
||||
|
||||
require.Error(t, d.TerminateWorkerInstances([]string{"i-1"}, "111122223333"))
|
||||
}
|
||||
@@ -18,6 +18,14 @@ var (
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
// Pin host vCPU so the resource manager is deterministic regardless of the
|
||||
// runner's CPU topology. Without this, scheduling is denominated by physical
|
||||
// cores (physicalCoreCount), and a 2-physical-core CI runner would leave 0
|
||||
// schedulable after the default 2-vCPU reserve, so NewResourceManager would
|
||||
// refuse to start and every daemon-constructing test would fail. 4 keeps the
|
||||
// default reserve intact with 2 schedulable cores — enough for these tests.
|
||||
os.Setenv("SPINIFEX_HOST_VCPU", "4")
|
||||
|
||||
// Start a plain NATS server (no JetStream)
|
||||
ns, err := server.NewServer(&server.Options{
|
||||
Host: "127.0.0.1",
|
||||
|
||||
@@ -52,6 +52,23 @@ func resolveHostReserve(getenv func(string) string) hostReserve {
|
||||
return r
|
||||
}
|
||||
|
||||
// resolveHostVCPU returns the detected core count, overridden by
|
||||
// SPINIFEX_HOST_VCPU when set to a valid positive integer. The override is an
|
||||
// escape hatch: it pins the schedulable core count for hosts where
|
||||
// /proc/cpuinfo topology detection misreports (some VMs/containers expose no
|
||||
// "core id", or fewer physical cores than the host can actually schedule), and
|
||||
// lets tests decouple from runner topology. Invalid values are logged and
|
||||
// ignored so a typo can't silently shrink capacity below the reserve.
|
||||
func resolveHostVCPU(getenv func(string) string, detected int) int {
|
||||
if v := getenv("SPINIFEX_HOST_VCPU"); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil && n > 0 {
|
||||
return n
|
||||
}
|
||||
slog.Warn("ignoring SPINIFEX_HOST_VCPU", "value", v)
|
||||
}
|
||||
return detected
|
||||
}
|
||||
|
||||
// minHostMemHeadroomGB is the minimum schedulable memory we require above
|
||||
// the reserve, so a host that just meets the reserve still has a small
|
||||
// amount left to launch the smallest guest type.
|
||||
|
||||
@@ -442,3 +442,24 @@ func TestResolveHostReserve(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveHostVCPU(t *testing.T) {
|
||||
const detected = 12
|
||||
tests := []struct {
|
||||
name string
|
||||
env string
|
||||
want int
|
||||
}{
|
||||
{name: "no env returns detected", env: "", want: detected},
|
||||
{name: "valid override", env: "8", want: 8},
|
||||
{name: "non-numeric falls back to detected", env: "garbage", want: detected},
|
||||
{name: "zero falls back to detected", env: "0", want: detected},
|
||||
{name: "negative falls back to detected", env: "-4", want: detected},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := resolveHostVCPU(func(string) string { return tc.env }, detected)
|
||||
assert.Equal(t, tc.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/mulgadc/spinifex/spinifex/awserrors"
|
||||
handlers_ec2_placementgroup "github.com/mulgadc/spinifex/spinifex/handlers/ec2/placementgroup"
|
||||
"github.com/mulgadc/spinifex/spinifex/network/topology"
|
||||
"github.com/mulgadc/spinifex/spinifex/tags"
|
||||
"github.com/mulgadc/spinifex/spinifex/types"
|
||||
"github.com/mulgadc/spinifex/spinifex/utils"
|
||||
"github.com/mulgadc/spinifex/spinifex/vm"
|
||||
@@ -398,6 +399,19 @@ func (d *Daemon) onInstanceUpHook() func(*vm.VM) error {
|
||||
}
|
||||
d.natsSubscriptions[consoleSubKey] = consoleSub
|
||||
|
||||
// Re-arm the per-instance terminate subscription for system-managed VMs
|
||||
// (ELBv2 load balancers). The launch handler binds it at first launch,
|
||||
// but it is lost across a daemon restart; OnInstanceUp fires on both the
|
||||
// relaunch and the reconnect-to-surviving-QEMU recovery paths, so without
|
||||
// this system.TerminateInstance.{id} has no responder and the VM can
|
||||
// never be torn down (SetSubnets relaunch, DeleteLoadBalancer).
|
||||
if instance.ManagedBy == tags.ManagedByELBv2 {
|
||||
if subErr := d.subscribeSystemTerminateLocked(instance.ID); subErr != nil {
|
||||
slog.Error("OnInstanceUp: failed to re-arm system terminate subscription",
|
||||
"instanceId", instance.ID, "err", subErr)
|
||||
}
|
||||
}
|
||||
|
||||
// Re-claim GPU(s) after a daemon restart with a still-running QEMU
|
||||
// process: the manager's reconnect path fires OnInstanceUp without
|
||||
// going through the handler-side Claim, so the GPU pool would
|
||||
@@ -469,6 +483,16 @@ func (d *Daemon) onInstanceDownHook() func(string) {
|
||||
}
|
||||
delete(d.natsSubscriptions, consoleSubKey)
|
||||
}
|
||||
// Drop the system terminate subscription (system VMs only; a no-op for
|
||||
// regular instances that never registered one).
|
||||
terminateSubKey := fmt.Sprintf("system.TerminateInstance.%s", instanceID)
|
||||
if sub, ok := d.natsSubscriptions[terminateSubKey]; ok {
|
||||
if err := sub.Unsubscribe(); err != nil {
|
||||
slog.Error("OnInstanceDown: failed to unsubscribe system terminate topic",
|
||||
"instanceId", instanceID, "err", err)
|
||||
}
|
||||
delete(d.natsSubscriptions, terminateSubKey)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -46,6 +46,45 @@ func TestOnInstanceUpHook_RegistersBothPerInstanceTopics(t *testing.T) {
|
||||
assert.Equal(t, "ec2.i-up-basic.GetConsoleOutput", consoleSub.Subject)
|
||||
}
|
||||
|
||||
func TestOnInstanceUpHook_ReArmsSystemTerminateForELBv2(t *testing.T) {
|
||||
d, _ := newHookTestDaemon(t)
|
||||
instance := &vm.VM{ID: "i-up-sys", ManagedBy: tags.ManagedByELBv2}
|
||||
|
||||
require.NoError(t, d.onInstanceUpHook()(instance))
|
||||
|
||||
// OnInstanceUp fires on both the relaunch and reconnect recovery paths, so
|
||||
// the terminate subject must be re-bound or the recovered LB VM can never
|
||||
// be torn down (SetSubnets relaunch, DeleteLoadBalancer).
|
||||
sub, ok := d.natsSubscriptions["system.TerminateInstance.i-up-sys"]
|
||||
require.True(t, ok, "system terminate subscription must be re-armed for ELBv2 VMs")
|
||||
assert.Equal(t, "system.TerminateInstance.i-up-sys", sub.Subject)
|
||||
}
|
||||
|
||||
func TestOnInstanceUpHook_NoSystemTerminateForRegularInstance(t *testing.T) {
|
||||
d, _ := newHookTestDaemon(t)
|
||||
instance := &vm.VM{ID: "i-up-regular"}
|
||||
|
||||
require.NoError(t, d.onInstanceUpHook()(instance))
|
||||
|
||||
_, ok := d.natsSubscriptions["system.TerminateInstance.i-up-regular"]
|
||||
assert.False(t, ok, "regular instances must not register a system terminate subscription")
|
||||
}
|
||||
|
||||
func TestOnInstanceDownHook_DropsSystemTerminateSubscription(t *testing.T) {
|
||||
d, _ := newHookTestDaemon(t)
|
||||
instance := &vm.VM{ID: "i-down-sys", ManagedBy: tags.ManagedByELBv2}
|
||||
|
||||
require.NoError(t, d.onInstanceUpHook()(instance))
|
||||
termSub := d.natsSubscriptions["system.TerminateInstance.i-down-sys"]
|
||||
require.NotNil(t, termSub)
|
||||
|
||||
d.onInstanceDownHook()(instance.ID)
|
||||
|
||||
_, present := d.natsSubscriptions["system.TerminateInstance.i-down-sys"]
|
||||
assert.False(t, present, "system terminate sub must be deleted from map")
|
||||
assert.False(t, termSub.IsValid(), "system terminate sub must be unsubscribed")
|
||||
}
|
||||
|
||||
func TestOnInstanceUpHook_ReplacesExistingSubsOnDoubleUp(t *testing.T) {
|
||||
d, _ := newHookTestDaemon(t)
|
||||
instance := &vm.VM{ID: "i-up-twice"}
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
package gateway_ec2_instance
|
||||
|
||||
import "github.com/aws/aws-sdk-go/service/iam"
|
||||
|
||||
// OIDC identity-provider registry stubs so fakeIAMService satisfies the
|
||||
// IAMService interface; EC2 instance-profile paths never call these.
|
||||
|
||||
func (f *fakeIAMService) CreateOpenIDConnectProvider(string, *iam.CreateOpenIDConnectProviderInput) (*iam.CreateOpenIDConnectProviderOutput, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (f *fakeIAMService) GetOpenIDConnectProvider(string, *iam.GetOpenIDConnectProviderInput) (*iam.GetOpenIDConnectProviderOutput, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (f *fakeIAMService) ListOpenIDConnectProviders(string, *iam.ListOpenIDConnectProvidersInput) (*iam.ListOpenIDConnectProvidersOutput, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (f *fakeIAMService) DeleteOpenIDConnectProvider(string, *iam.DeleteOpenIDConnectProviderInput) (*iam.DeleteOpenIDConnectProviderOutput, error) {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -60,6 +60,22 @@ var eksRoutes = []eksRoute{
|
||||
func(gw *GatewayConfig, acct, callerARN string, p []string, b []byte) (any, error) {
|
||||
return gateway_eks.UpdateClusterVersion(gw.NATSConn, acct, p[0], b)
|
||||
}},
|
||||
// Control-plane VM broker: the K3s server VM POSTs bootstrap envelopes and
|
||||
// state reports here (system SigV4 creds) and the gateway relays them onto
|
||||
// the existing eks.bus.*/eks.state.* NATS subjects. acct (system account)
|
||||
// and callerARN are ignored — the cluster account comes from the body.
|
||||
{"POST", regexp.MustCompile(`^/clusters/([^/]+)/internal-publish$`), "PublishInternal",
|
||||
func(gw *GatewayConfig, acct, callerARN string, p []string, b []byte) (any, error) {
|
||||
return gateway_eks.PublishInternal(gw.NATSConn, p[0], b)
|
||||
}},
|
||||
// Control-plane VM TokenReview broker: the eks-token-webhook POSTs the
|
||||
// `aws eks get-token` bearer token here (system SigV4 creds) and the gateway
|
||||
// resolves it host-side (STS verify + AccessEntry lookup), so the webhook
|
||||
// never speaks NATS. Cluster account comes from the body.
|
||||
{"POST", regexp.MustCompile(`^/clusters/([^/]+)/token-review$`), "WebhookTokenReview",
|
||||
func(gw *GatewayConfig, acct, callerARN string, p []string, b []byte) (any, error) {
|
||||
return gateway_eks.WebhookTokenReview(gw.NATSConn, p[0], b)
|
||||
}},
|
||||
|
||||
// Nodegroup
|
||||
{"POST", regexp.MustCompile(`^/clusters/([^/]+)/node-groups$`), "CreateNodegroup",
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go/private/protocol/json/jsonutil"
|
||||
"github.com/mulgadc/spinifex/spinifex/awserrors"
|
||||
@@ -28,11 +29,18 @@ type EKSJSONError struct {
|
||||
}
|
||||
|
||||
// GenerateEKSErrorResponse marshals the AWS REST-JSON error envelope.
|
||||
// The trailing "Exception" suffix matches the convention AWS uses
|
||||
// (e.g. ResourceNotFoundException, InvalidParameterException).
|
||||
// The wire __type carries the trailing "Exception" suffix AWS uses
|
||||
// (e.g. ResourceNotFoundException, InvalidParameterException). The suffix is
|
||||
// appended idempotently: many awserrors codes already end in "Exception"
|
||||
// (e.g. ErrorEKSResourceNotFound = "ResourceNotFoundException"), and adding a
|
||||
// second would emit ResourceNotFoundExceptionException, which SDK clients
|
||||
// reject when matching eks.ErrCodeResourceNotFoundException.
|
||||
func GenerateEKSErrorResponse(code, message string) []byte {
|
||||
if !strings.HasSuffix(code, "Exception") {
|
||||
code += "Exception"
|
||||
}
|
||||
body, err := json.Marshal(EKSJSONError{
|
||||
Type: code + "Exception",
|
||||
Type: code,
|
||||
Message: message,
|
||||
})
|
||||
if err != nil {
|
||||
|
||||
@@ -21,6 +21,16 @@ func TestGenerateEKSErrorResponse_ShapesExceptionSuffix(t *testing.T) {
|
||||
assert.Equal(t, "Cluster does not exist", env.Message)
|
||||
}
|
||||
|
||||
// Codes that already carry the "Exception" suffix (e.g.
|
||||
// awserrors.ErrorEKSResourceNotFound = "ResourceNotFoundException") must not be
|
||||
// doubled into ResourceNotFoundExceptionException, which SDK clients reject.
|
||||
func TestGenerateEKSErrorResponse_DoesNotDoubleExceptionSuffix(t *testing.T) {
|
||||
body := GenerateEKSErrorResponse("ResourceNotFoundException", "Cluster does not exist")
|
||||
var env EKSJSONError
|
||||
require.NoError(t, json.Unmarshal(body, &env))
|
||||
assert.Equal(t, eks.ErrCodeResourceNotFoundException, env.Type)
|
||||
}
|
||||
|
||||
func TestWriteJSONError_SetsContentTypeAndStatus(t *testing.T) {
|
||||
w := httptest.NewRecorder()
|
||||
WriteJSONError(w, "NotImplemented", "Operation not implemented", http.StatusNotImplemented)
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
package gateway_eks
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/awserrors"
|
||||
handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// internalPublishChannels are the two VM→host channels the broker relays.
|
||||
const (
|
||||
internalChannelBootstrap = "bootstrap"
|
||||
internalChannelState = "state"
|
||||
)
|
||||
|
||||
// internalPublishRequest is the wire shape the on-VM eks-gateway-publish helper
|
||||
// POSTs to /clusters/{name}/internal-publish. The control-plane VM holds system
|
||||
// SigV4 creds (system account, not the cluster account), so AccountID names the
|
||||
// customer cluster account explicitly rather than being taken from the SigV4
|
||||
// auth context. Payload is the already-encoded subject body (a BootstrapEnvelope
|
||||
// for the bootstrap channel, a ServerStateReport for the state channel) — the
|
||||
// broker relays it verbatim onto the existing NATS subject, so the daemon-side
|
||||
// subscriber, KV persistence, and reconciler are unchanged.
|
||||
type internalPublishRequest struct {
|
||||
AccountID string `json:"accountId"`
|
||||
Channel string `json:"channel"`
|
||||
Kind string `json:"kind"`
|
||||
Payload json.RawMessage `json:"payload"`
|
||||
}
|
||||
|
||||
// publishInternalOutput is the empty success body (`{}`).
|
||||
type publishInternalOutput struct{}
|
||||
|
||||
// validBootstrapKinds is the closed set of bootstrap subject suffixes the broker
|
||||
// will relay — a malformed/hostile kind cannot be turned into an arbitrary
|
||||
// subject suffix.
|
||||
var validBootstrapKinds = map[string]struct{}{
|
||||
handlers_eks.BootstrapSubjectToken: {},
|
||||
handlers_eks.BootstrapSubjectKubeconfig: {},
|
||||
handlers_eks.BootstrapSubjectJWKS: {},
|
||||
handlers_eks.BootstrapSubjectCA: {},
|
||||
}
|
||||
|
||||
// PublishInternal — POST /clusters/{name}/internal-publish. Relays a
|
||||
// control-plane VM publication onto the existing bootstrap/state NATS subjects.
|
||||
// This is the gateway-broker replacement for the VM dialing core NATS directly:
|
||||
// the VM reaches the daemon over the mgmt-reachable AWSGW with SigV4 + retry
|
||||
// (DDIL-correct), and NATS stays cluster-internal.
|
||||
func PublishInternal(natsConn *nats.Conn, clusterName string, body []byte) (*publishInternalOutput, error) {
|
||||
if natsConn == nil {
|
||||
return nil, errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
if clusterName == "" {
|
||||
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
|
||||
}
|
||||
|
||||
var req internalPublishRequest
|
||||
if err := json.Unmarshal(body, &req); err != nil {
|
||||
slog.Debug("PublishInternal: bad body", "cluster", clusterName, "err", err)
|
||||
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
|
||||
}
|
||||
if req.AccountID == "" || len(req.Payload) == 0 {
|
||||
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
|
||||
}
|
||||
|
||||
var subject string
|
||||
switch req.Channel {
|
||||
case internalChannelBootstrap:
|
||||
if _, ok := validBootstrapKinds[req.Kind]; !ok {
|
||||
slog.Debug("PublishInternal: unknown bootstrap kind", "cluster", clusterName, "kind", req.Kind)
|
||||
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
|
||||
}
|
||||
subject = handlers_eks.BootstrapSubject(req.AccountID, clusterName, req.Kind)
|
||||
case internalChannelState:
|
||||
subject = handlers_eks.StateSubject(req.AccountID, clusterName)
|
||||
default:
|
||||
slog.Debug("PublishInternal: unknown channel", "cluster", clusterName, "channel", req.Channel)
|
||||
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
|
||||
}
|
||||
|
||||
if err := natsConn.Publish(subject, req.Payload); err != nil {
|
||||
slog.Error("PublishInternal: NATS publish failed", "subject", subject, "err", err)
|
||||
return nil, errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
slog.Debug("PublishInternal: relayed", "subject", subject, "bytes", len(req.Payload))
|
||||
|
||||
return &publishInternalOutput{}, nil
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package gateway_eks
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
|
||||
"github.com/mulgadc/spinifex/spinifex/testutil"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestPublishInternal_RelaysToSubjects(t *testing.T) {
|
||||
_, nc := testutil.StartTestNATS(t)
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
body string
|
||||
subject string
|
||||
payload string
|
||||
}{
|
||||
{
|
||||
name: "bootstrap token",
|
||||
body: `{"accountId":"111122223333","channel":"bootstrap","kind":"k3s-bootstrap-token","payload":{"token":"k3s-secret"}}`,
|
||||
subject: handlers_eks.BootstrapSubject("111122223333", "alpha", handlers_eks.BootstrapSubjectToken),
|
||||
payload: `{"token":"k3s-secret"}`,
|
||||
},
|
||||
{
|
||||
name: "state report",
|
||||
body: `{"accountId":"111122223333","channel":"state","payload":{"healthz":"ok","node_count":1,"ts":42}}`,
|
||||
subject: handlers_eks.StateSubject("111122223333", "alpha"),
|
||||
payload: `{"healthz":"ok","node_count":1,"ts":42}`,
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
sub, err := nc.SubscribeSync(tc.subject)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
|
||||
out, err := PublishInternal(nc, "alpha", []byte(tc.body))
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, out)
|
||||
|
||||
msg, err := sub.NextMsg(2 * time.Second)
|
||||
require.NoError(t, err)
|
||||
assert.JSONEq(t, tc.payload, string(msg.Data))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublishInternal_Rejects(t *testing.T) {
|
||||
_, nc := testutil.StartTestNATS(t)
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
body string
|
||||
}{
|
||||
{"empty account", `{"accountId":"","channel":"state","payload":{"healthz":"ok"}}`},
|
||||
{"empty payload", `{"accountId":"111122223333","channel":"state"}`},
|
||||
{"unknown channel", `{"accountId":"111122223333","channel":"wat","payload":{"x":1}}`},
|
||||
{"unknown bootstrap kind", `{"accountId":"111122223333","channel":"bootstrap","kind":"evil","payload":{"x":1}}`},
|
||||
{"malformed json", `{not-json`},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := PublishInternal(nc, "alpha", []byte(tc.body))
|
||||
require.Error(t, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublishInternal_NilConnAndEmptyCluster(t *testing.T) {
|
||||
_, err := PublishInternal(nil, "alpha", []byte(`{}`))
|
||||
require.Error(t, err)
|
||||
|
||||
_, nc := testutil.StartTestNATS(t)
|
||||
_, err = PublishInternal(nc, "", []byte(`{"accountId":"1","channel":"state","payload":{"x":1}}`))
|
||||
require.Error(t, err)
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package gateway_eks
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/awserrors"
|
||||
handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// tokenReviewVerifyTimeout bounds the host-side STS verify NATS round-trip. The
|
||||
// apiserver fails the TokenReview on its own timeout, so keep this tight.
|
||||
const tokenReviewVerifyTimeout = 5 * time.Second
|
||||
|
||||
// webhookTokenReviewRequest is the body the on-VM eks-token-webhook POSTs to
|
||||
// /clusters/{name}/token-review. The webhook holds system SigV4 creds (system
|
||||
// account, not the cluster account), so AccountID names the cluster account
|
||||
// explicitly — same pattern as PublishInternal.
|
||||
type webhookTokenReviewRequest struct {
|
||||
AccountID string `json:"accountId"`
|
||||
Token string `json:"token"`
|
||||
}
|
||||
|
||||
// WebhookTokenReview — POST /clusters/{name}/token-review. Resolves an
|
||||
// `aws eks get-token` bearer token to a K8s identity host-side (STS verify +
|
||||
// AccessEntry lookup) and returns it. This is the gateway-broker replacement
|
||||
// for the webhook dialing core NATS/STS/KV directly.
|
||||
func WebhookTokenReview(natsConn *nats.Conn, clusterName string, body []byte) (*handlers_eks.WebhookTokenReviewResult, error) {
|
||||
if natsConn == nil {
|
||||
return nil, errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
if clusterName == "" {
|
||||
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
|
||||
}
|
||||
|
||||
var req webhookTokenReviewRequest
|
||||
if err := json.Unmarshal(body, &req); err != nil {
|
||||
slog.Debug("WebhookTokenReview: bad body", "cluster", clusterName, "err", err)
|
||||
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
|
||||
}
|
||||
if req.AccountID == "" || req.Token == "" {
|
||||
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
|
||||
}
|
||||
|
||||
res, err := handlers_eks.ResolveTokenReview(natsConn, req.AccountID, clusterName, req.Token, tokenReviewVerifyTimeout)
|
||||
if err != nil {
|
||||
slog.Error("WebhookTokenReview: resolve failed", "cluster", clusterName, "err", err)
|
||||
return nil, errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
return &res, nil
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package gateway_eks
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/awserrors"
|
||||
handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
|
||||
"github.com/mulgadc/spinifex/spinifex/testutil"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const testARN = "arn:aws:iam::111122223333:role/admin"
|
||||
|
||||
// validToken builds a k8s-aws-v1 bearer token (the `aws eks get-token` shape):
|
||||
// the literal prefix + base64 raw-url of the presigned GetCallerIdentity URL.
|
||||
func validToken(url string) string {
|
||||
return "k8s-aws-v1." + base64.RawURLEncoding.EncodeToString([]byte(url))
|
||||
}
|
||||
|
||||
func TestWebhookTokenReview_Rejects(t *testing.T) {
|
||||
_, nc := testutil.StartTestNATS(t)
|
||||
|
||||
// nil conn → ServerInternal
|
||||
_, err := WebhookTokenReview(nil, "alpha", []byte(`{"accountId":"1","token":"t"}`))
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
|
||||
|
||||
cases := []struct {
|
||||
name, cluster, body, want string
|
||||
}{
|
||||
{"empty cluster", "", `{"accountId":"1","token":"t"}`, awserrors.ErrorInvalidParameterValue},
|
||||
{"malformed json", "alpha", `{not-json`, awserrors.ErrorInvalidParameterValue},
|
||||
{"empty account", "alpha", `{"accountId":"","token":"t"}`, awserrors.ErrorInvalidParameterValue},
|
||||
{"empty token", "alpha", `{"accountId":"1","token":""}`, awserrors.ErrorInvalidParameterValue},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := WebhookTokenReview(nc, tc.cluster, []byte(tc.body))
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, tc.want, err.Error())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A genuine infra fault (account bucket absent) maps to ServerInternal.
|
||||
func TestWebhookTokenReview_InfraFaultIsServerInternal(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
_, err := WebhookTokenReview(nc, "alpha", []byte(`{"accountId":"111122223333","token":"`+validToken("https://sts/?x=1")+`"}`))
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
|
||||
}
|
||||
|
||||
func TestWebhookTokenReview_ResolvesIdentity(t *testing.T) {
|
||||
_, nc, js := testutil.StartTestJetStream(t)
|
||||
kv := testutil.SeedKV(t, js, handlers_eks.AccountBucketName("111122223333"), nil)
|
||||
require.NoError(t, handlers_eks.PutAccessEntryRecord(kv, &handlers_eks.AccessEntryRecord{
|
||||
ClusterName: "alpha",
|
||||
PrincipalARN: testARN,
|
||||
KubernetesUsername: testARN,
|
||||
KubernetesGroups: []string{"system:masters"},
|
||||
Type: handlers_eks.AccessEntryTypeStandard,
|
||||
}))
|
||||
|
||||
sub, err := nc.Subscribe(handlers_eks.TokenVerifySubject, func(m *nats.Msg) {
|
||||
resp, _ := json.Marshal(handlers_eks.TokenVerifyResponse{
|
||||
AccountID: "111122223333",
|
||||
ARN: testARN,
|
||||
UserID: "AROAEXAMPLE:session",
|
||||
PrincipalType: "AssumedRole",
|
||||
})
|
||||
_ = m.Respond(resp)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
|
||||
out, err := WebhookTokenReview(nc, "alpha", []byte(`{"accountId":"111122223333","token":"`+validToken("https://sts/?Action=GetCallerIdentity")+`"}`))
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, out)
|
||||
assert.True(t, out.Authenticated)
|
||||
assert.Equal(t, testARN, out.Username)
|
||||
assert.Equal(t, []string{"system:masters"}, out.Groups)
|
||||
}
|
||||
@@ -97,9 +97,14 @@ func TestLookupEKSAction_EncodedPrincipalARN(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLookupEKSAction_CoversAll34Actions(t *testing.T) {
|
||||
func TestLookupEKSAction_CoversAllActions(t *testing.T) {
|
||||
expected := map[string]bool{
|
||||
"CreateCluster": false,
|
||||
"CreateCluster": false,
|
||||
// PublishInternal + WebhookTokenReview are internal control-plane VM
|
||||
// broker routes, not AWS-SDK EKS actions — registered alongside the
|
||||
// public actions.
|
||||
"PublishInternal": false,
|
||||
"WebhookTokenReview": false,
|
||||
"DescribeCluster": false,
|
||||
"ListClusters": false,
|
||||
"UpdateClusterConfig": false,
|
||||
|
||||
@@ -132,6 +132,9 @@ var elbv2Actions = map[string]ELBv2Handler{
|
||||
"SetIpAddressType": elbv2Handler(func(input *elbv2.SetIpAddressTypeInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return gateway_elbv2.SetIpAddressType(input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"SetSubnets": elbv2Handler(func(input *elbv2.SetSubnetsInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return gateway_elbv2.SetSubnets(input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"AddListenerCertificates": elbv2Handler(func(input *elbv2.AddListenerCertificatesInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return gateway_elbv2.AddListenerCertificates(input, gw.NATSConn, accountID)
|
||||
}),
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
package gateway_elbv2
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/elbv2"
|
||||
"github.com/mulgadc/spinifex/spinifex/awserrors"
|
||||
handlers_elbv2 "github.com/mulgadc/spinifex/spinifex/handlers/elbv2"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// SetSubnets handles the ELBv2 SetSubnets API call. It enables the subnets (and
|
||||
// their backing ENIs) for the load balancer, adding and removing them to match
|
||||
// the request.
|
||||
func SetSubnets(input *elbv2.SetSubnetsInput, natsConn *nats.Conn, accountID string) (elbv2.SetSubnetsOutput, error) {
|
||||
var output elbv2.SetSubnetsOutput
|
||||
|
||||
if input == nil || input.LoadBalancerArn == nil || *input.LoadBalancerArn == "" {
|
||||
return output, errors.New(awserrors.ErrorMissingParameter)
|
||||
}
|
||||
if len(input.Subnets) == 0 && len(input.SubnetMappings) == 0 {
|
||||
return output, errors.New(awserrors.ErrorMissingParameter)
|
||||
}
|
||||
|
||||
svc := handlers_elbv2.NewNATSELBv2Service(natsConn)
|
||||
result, err := svc.SetSubnets(input, accountID)
|
||||
if err != nil {
|
||||
return output, err
|
||||
}
|
||||
|
||||
return *result, nil
|
||||
}
|
||||
@@ -56,6 +56,7 @@ func TestELBv2ActionsMap_AllActionsRegistered(t *testing.T) {
|
||||
"RemoveTags",
|
||||
"SetSecurityGroups",
|
||||
"SetIpAddressType",
|
||||
"SetSubnets",
|
||||
"LBAgentHeartbeat",
|
||||
"GetLBConfig",
|
||||
"ModifyTargetGroupAttributes",
|
||||
|
||||
+35
-14
@@ -76,6 +76,7 @@ var supportedServices = map[string]bool{
|
||||
"elasticloadbalancing": true,
|
||||
"eks": true,
|
||||
"acm": true,
|
||||
"tagging": true,
|
||||
"spinifex": true,
|
||||
}
|
||||
|
||||
@@ -135,19 +136,37 @@ func (gw *GatewayConfig) SetupRoutes() http.Handler {
|
||||
r.Use(slogRequestLogger)
|
||||
}
|
||||
|
||||
// AWS SigV4 authentication middleware
|
||||
r.Use(gw.SigV4AuthMiddleware())
|
||||
// Anonymous STS bootstrap (AssumeRoleWithWebIdentity) is dispatched here,
|
||||
// ahead of the SigV4 surface — these calls hold no AWS credentials, only a
|
||||
// web-identity JWT in the body. Signed requests pass straight through.
|
||||
r.Use(gw.anonymousSTSInterceptor)
|
||||
|
||||
// API request throttling (post-auth, per-account+action token bucket)
|
||||
if gw.Throttler != nil {
|
||||
r.Use(gw.Throttler.Middleware(
|
||||
gw.throttleKeyFuncs(),
|
||||
gw.writeThrottleError,
|
||||
))
|
||||
}
|
||||
// Public, unauthenticated OIDC discovery endpoints (IRSA). They serve the
|
||||
// per-cluster issuer discovery document + JWKS so AWS STS / SDKs can
|
||||
// validate ServiceAccount tokens. These carry no SigV4 and must bypass the
|
||||
// auth + throttle middleware, so they live in their own group registered
|
||||
// ahead of the authenticated surface.
|
||||
r.Group(func(pub chi.Router) {
|
||||
pub.Get("/oidc/eks/{region}/{accountID}/{clusterName}/.well-known/openid-configuration", gw.OIDCDiscoveryDocument)
|
||||
pub.Get("/oidc/eks/{region}/{accountID}/{clusterName}/keys", gw.OIDCJWKS)
|
||||
})
|
||||
|
||||
// Catch-all routes
|
||||
r.HandleFunc("/*", gw.Request)
|
||||
// Authenticated AWS API surface.
|
||||
r.Group(func(auth chi.Router) {
|
||||
// AWS SigV4 authentication middleware
|
||||
auth.Use(gw.SigV4AuthMiddleware())
|
||||
|
||||
// API request throttling (post-auth, per-account+action token bucket)
|
||||
if gw.Throttler != nil {
|
||||
auth.Use(gw.Throttler.Middleware(
|
||||
gw.throttleKeyFuncs(),
|
||||
gw.writeThrottleError,
|
||||
))
|
||||
}
|
||||
|
||||
// Catch-all routes
|
||||
auth.HandleFunc("/*", gw.Request)
|
||||
})
|
||||
|
||||
return r
|
||||
}
|
||||
@@ -308,6 +327,8 @@ func (gw *GatewayConfig) Request(w http.ResponseWriter, r *http.Request) {
|
||||
err = gw.EKS_Request(w, r)
|
||||
case "acm":
|
||||
err = gw.ACM_Request(w, r)
|
||||
case "tagging":
|
||||
err = gw.Tagging_Request(w, r)
|
||||
case "spinifex":
|
||||
err = gw.Spinifex_Request(w, r)
|
||||
default:
|
||||
@@ -460,10 +481,10 @@ func (gw *GatewayConfig) ErrorHandler(w http.ResponseWriter, r *http.Request, er
|
||||
errorMsg.HTTPCode = 500
|
||||
}
|
||||
|
||||
// EKS and ACM both use the AWS JSON 1.1 error envelope
|
||||
// ({"__type":"<Code>Exception","message":...}, Content-Type
|
||||
// EKS, ACM, and the Resource Groups Tagging API all use the AWS JSON 1.1
|
||||
// error envelope ({"__type":"<Code>Exception","message":...}, Content-Type
|
||||
// application/x-amz-json-1.1). Query/XML services fall through below.
|
||||
if svc == "eks" || svc == "acm" {
|
||||
if svc == "eks" || svc == "acm" || svc == "tagging" {
|
||||
body := GenerateEKSErrorResponse(err.Error(), errorMsg.Message, requestId)
|
||||
slog.Debug("Generated JSON error response", "service", svc, "error", err.Error(), "json", string(body), "requestId", requestId)
|
||||
w.Header().Set("Content-Type", eksJSONContentType)
|
||||
|
||||
@@ -508,6 +508,11 @@ func TestGetService(t *testing.T) {
|
||||
ctxVal: "account",
|
||||
wantSvc: "account",
|
||||
},
|
||||
{
|
||||
name: "tagging service",
|
||||
ctxVal: "tagging",
|
||||
wantSvc: "tagging",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
|
||||
@@ -154,6 +154,20 @@ var iamActions = map[string]IAMHandler{
|
||||
"RemoveRoleFromInstanceProfile": iamHandler(func(accountID string, input *iam.RemoveRoleFromInstanceProfileInput, gw *GatewayConfig) (any, error) {
|
||||
return gateway_iam.RemoveRoleFromInstanceProfile(accountID, input, gw.IAMService)
|
||||
}),
|
||||
|
||||
// OIDC identity-provider registry (IRSA)
|
||||
"CreateOpenIDConnectProvider": iamHandler(func(accountID string, input *iam.CreateOpenIDConnectProviderInput, gw *GatewayConfig) (any, error) {
|
||||
return gateway_iam.CreateOpenIDConnectProvider(accountID, input, gw.IAMService)
|
||||
}),
|
||||
"GetOpenIDConnectProvider": iamHandler(func(accountID string, input *iam.GetOpenIDConnectProviderInput, gw *GatewayConfig) (any, error) {
|
||||
return gateway_iam.GetOpenIDConnectProvider(accountID, input, gw.IAMService)
|
||||
}),
|
||||
"ListOpenIDConnectProviders": iamHandler(func(accountID string, input *iam.ListOpenIDConnectProvidersInput, gw *GatewayConfig) (any, error) {
|
||||
return gateway_iam.ListOpenIDConnectProviders(accountID, input, gw.IAMService)
|
||||
}),
|
||||
"DeleteOpenIDConnectProvider": iamHandler(func(accountID string, input *iam.DeleteOpenIDConnectProviderInput, gw *GatewayConfig) (any, error) {
|
||||
return gateway_iam.DeleteOpenIDConnectProvider(accountID, input, gw.IAMService)
|
||||
}),
|
||||
}
|
||||
|
||||
func (gw *GatewayConfig) IAM_Request(w http.ResponseWriter, r *http.Request) error {
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
package gateway_iam
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/iam"
|
||||
"github.com/mulgadc/spinifex/spinifex/awserrors"
|
||||
handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
|
||||
)
|
||||
|
||||
func CreateOpenIDConnectProvider(accountID string, input *iam.CreateOpenIDConnectProviderInput, svc handlers_iam.IAMService) (*iam.CreateOpenIDConnectProviderOutput, error) {
|
||||
if input.Url == nil || *input.Url == "" {
|
||||
return nil, errors.New(awserrors.ErrorMissingParameter)
|
||||
}
|
||||
return svc.CreateOpenIDConnectProvider(accountID, input)
|
||||
}
|
||||
|
||||
func GetOpenIDConnectProvider(accountID string, input *iam.GetOpenIDConnectProviderInput, svc handlers_iam.IAMService) (*iam.GetOpenIDConnectProviderOutput, error) {
|
||||
if input.OpenIDConnectProviderArn == nil || *input.OpenIDConnectProviderArn == "" {
|
||||
return nil, errors.New(awserrors.ErrorMissingParameter)
|
||||
}
|
||||
return svc.GetOpenIDConnectProvider(accountID, input)
|
||||
}
|
||||
|
||||
func ListOpenIDConnectProviders(accountID string, input *iam.ListOpenIDConnectProvidersInput, svc handlers_iam.IAMService) (*iam.ListOpenIDConnectProvidersOutput, error) {
|
||||
return svc.ListOpenIDConnectProviders(accountID, input)
|
||||
}
|
||||
|
||||
func DeleteOpenIDConnectProvider(accountID string, input *iam.DeleteOpenIDConnectProviderInput, svc handlers_iam.IAMService) (*iam.DeleteOpenIDConnectProviderOutput, error) {
|
||||
if input.OpenIDConnectProviderArn == nil || *input.OpenIDConnectProviderArn == "" {
|
||||
return nil, errors.New(awserrors.ErrorMissingParameter)
|
||||
}
|
||||
return svc.DeleteOpenIDConnectProvider(accountID, input)
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package gateway_iam
|
||||
|
||||
import "github.com/aws/aws-sdk-go/service/iam"
|
||||
|
||||
// OIDC identity-provider registry stubs so stubIAMService satisfies the
|
||||
// IAMService interface. Dedicated CRUD coverage lives in handlers/iam.
|
||||
|
||||
func (s *stubIAMService) CreateOpenIDConnectProvider(string, *iam.CreateOpenIDConnectProviderInput) (*iam.CreateOpenIDConnectProviderOutput, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (s *stubIAMService) GetOpenIDConnectProvider(string, *iam.GetOpenIDConnectProviderInput) (*iam.GetOpenIDConnectProviderOutput, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (s *stubIAMService) ListOpenIDConnectProviders(string, *iam.ListOpenIDConnectProvidersInput) (*iam.ListOpenIDConnectProvidersOutput, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (s *stubIAMService) DeleteOpenIDConnectProvider(string, *iam.DeleteOpenIDConnectProviderInput) (*iam.DeleteOpenIDConnectProviderOutput, error) {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// oidcDiscoveryDocument is the subset of the OpenID Connect Discovery metadata
|
||||
// (RFC 8414) that AWS STS and the AWS SDKs consume to validate IRSA
|
||||
// ServiceAccount tokens. Served unauthenticated at the per-cluster issuer URL.
|
||||
type oidcDiscoveryDocument struct {
|
||||
Issuer string `json:"issuer"`
|
||||
JWKSURI string `json:"jwks_uri"`
|
||||
ResponseTypesSupported []string `json:"response_types_supported"`
|
||||
SubjectTypesSupported []string `json:"subject_types_supported"`
|
||||
IDTokenSigningAlgValuesSupported []string `json:"id_token_signing_alg_values_supported"`
|
||||
ClaimsSupported []string `json:"claims_supported"`
|
||||
}
|
||||
|
||||
// eksAccountKV opens the read-only per-account EKS bucket. Returns
|
||||
// nats.ErrBucketNotFound (unwrapped) when the account has no clusters, which
|
||||
// the OIDC handlers translate to 404.
|
||||
func (gw *GatewayConfig) eksAccountKV(accountID string) (nats.KeyValue, error) {
|
||||
if gw.NATSConn == nil {
|
||||
return nil, errors.New("gateway NATS connection not initialised")
|
||||
}
|
||||
js, err := gw.NATSConn.JetStream()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return js.KeyValue(handlers_eks.AccountBucketName(accountID))
|
||||
}
|
||||
|
||||
// OIDCDiscoveryDocument serves the cluster's OpenID discovery document. The
|
||||
// `issuer` it returns is the cluster's stored IssuerURL — byte-identical to the
|
||||
// apiserver's `iss` claim — so an AWS SDK that fetches this document while
|
||||
// registering the provider sees a matching issuer.
|
||||
func (gw *GatewayConfig) OIDCDiscoveryDocument(w http.ResponseWriter, r *http.Request) {
|
||||
accountID := chi.URLParam(r, "accountID")
|
||||
clusterName := chi.URLParam(r, "clusterName")
|
||||
|
||||
kv, err := gw.eksAccountKV(accountID)
|
||||
if err != nil {
|
||||
if errors.Is(err, nats.ErrBucketNotFound) {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
slog.Error("OIDC discovery: open account bucket", "accountID", accountID, "err", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
meta, err := handlers_eks.GetClusterMeta(kv, clusterName)
|
||||
if err != nil || meta == nil || meta.OIDCIssuer == "" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
gw.writeOIDCJSON(w, oidcDiscoveryDocument{
|
||||
Issuer: meta.OIDCIssuer,
|
||||
JWKSURI: meta.OIDCIssuer + "/keys",
|
||||
ResponseTypesSupported: []string{"id_token"},
|
||||
SubjectTypesSupported: []string{"public"},
|
||||
IDTokenSigningAlgValuesSupported: []string{"ES256"},
|
||||
ClaimsSupported: []string{"sub", "iss", "aud", "exp", "iat"},
|
||||
})
|
||||
}
|
||||
|
||||
// OIDCJWKS serves the cluster's JWKS document (the public verification key) as
|
||||
// referenced by the discovery document's jwks_uri.
|
||||
func (gw *GatewayConfig) OIDCJWKS(w http.ResponseWriter, r *http.Request) {
|
||||
accountID := chi.URLParam(r, "accountID")
|
||||
clusterName := chi.URLParam(r, "clusterName")
|
||||
|
||||
kv, err := gw.eksAccountKV(accountID)
|
||||
if err != nil {
|
||||
if errors.Is(err, nats.ErrBucketNotFound) {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
slog.Error("OIDC JWKS: open account bucket", "accountID", accountID, "err", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
entry, err := kv.Get(handlers_eks.OIDCJWKSKey(clusterName))
|
||||
if err != nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Cache-Control", "public, max-age=300")
|
||||
if _, err := w.Write(entry.Value()); err != nil {
|
||||
slog.Debug("OIDC JWKS: write response", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (gw *GatewayConfig) writeOIDCJSON(w http.ResponseWriter, v any) {
|
||||
body, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Cache-Control", "public, max-age=300")
|
||||
if _, err := w.Write(body); err != nil {
|
||||
slog.Debug("OIDC discovery: write response", "err", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
|
||||
"github.com/mulgadc/spinifex/spinifex/testutil"
|
||||
)
|
||||
|
||||
const (
|
||||
testDiscAccount = "000000000001"
|
||||
testDiscCluster = "h1"
|
||||
testDiscIssuer = "https://10.0.0.1:9999/oidc/eks/ap-southeast-2/000000000001/h1"
|
||||
)
|
||||
|
||||
// seedDiscoveryCluster writes a ClusterMeta (with OIDCIssuer) and a JWKS doc
|
||||
// into the per-account EKS bucket, returning a router wired to the same NATS.
|
||||
func seedDiscoveryCluster(t *testing.T) http.Handler {
|
||||
t.Helper()
|
||||
_, nc, js := testutil.StartTestJetStream(t)
|
||||
|
||||
kv, err := handlers_eks.GetOrCreateAccountBucket(js, testDiscAccount)
|
||||
if err != nil {
|
||||
t.Fatalf("account bucket: %v", err)
|
||||
}
|
||||
if err := handlers_eks.PutClusterMeta(kv, &handlers_eks.ClusterMeta{
|
||||
Name: testDiscCluster,
|
||||
Status: handlers_eks.ClusterStatusActive,
|
||||
OIDCIssuer: testDiscIssuer,
|
||||
}); err != nil {
|
||||
t.Fatalf("put cluster meta: %v", err)
|
||||
}
|
||||
if _, err := kv.Put(handlers_eks.OIDCJWKSKey(testDiscCluster), []byte(`{"keys":[{"kty":"EC","kid":"abc"}]}`)); err != nil {
|
||||
t.Fatalf("put jwks: %v", err)
|
||||
}
|
||||
|
||||
gw := &GatewayConfig{DisableLogging: true, NATSConn: nc}
|
||||
return gw.SetupRoutes()
|
||||
}
|
||||
|
||||
func TestOIDCDiscoveryDocument(t *testing.T) {
|
||||
h := seedDiscoveryCluster(t)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet,
|
||||
"/oidc/eks/ap-southeast-2/"+testDiscAccount+"/"+testDiscCluster+"/.well-known/openid-configuration", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var doc oidcDiscoveryDocument
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &doc); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
if doc.Issuer != testDiscIssuer {
|
||||
t.Fatalf("issuer = %q, want %q", doc.Issuer, testDiscIssuer)
|
||||
}
|
||||
if doc.JWKSURI != testDiscIssuer+"/keys" {
|
||||
t.Fatalf("jwks_uri = %q, want issuer+/keys", doc.JWKSURI)
|
||||
}
|
||||
if len(doc.IDTokenSigningAlgValuesSupported) != 1 || doc.IDTokenSigningAlgValuesSupported[0] != "ES256" {
|
||||
t.Fatalf("alg = %v, want [ES256]", doc.IDTokenSigningAlgValuesSupported)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOIDCJWKS(t *testing.T) {
|
||||
h := seedDiscoveryCluster(t)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet,
|
||||
"/oidc/eks/ap-southeast-2/"+testDiscAccount+"/"+testDiscCluster+"/keys", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d", rec.Code)
|
||||
}
|
||||
if ct := rec.Header().Get("Content-Type"); ct != "application/json" {
|
||||
t.Fatalf("content-type = %q", ct)
|
||||
}
|
||||
var jwks struct {
|
||||
Keys []map[string]string `json:"keys"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &jwks); err != nil {
|
||||
t.Fatalf("unmarshal jwks: %v", err)
|
||||
}
|
||||
if len(jwks.Keys) != 1 || jwks.Keys[0]["kid"] != "abc" {
|
||||
t.Fatalf("jwks = %s", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestOIDCDiscovery_UnknownCluster404(t *testing.T) {
|
||||
h := seedDiscoveryCluster(t)
|
||||
|
||||
for _, path := range []string{
|
||||
"/oidc/eks/ap-southeast-2/" + testDiscAccount + "/nope/.well-known/openid-configuration",
|
||||
"/oidc/eks/ap-southeast-2/" + testDiscAccount + "/nope/keys",
|
||||
"/oidc/eks/ap-southeast-2/999999999999/h1/keys",
|
||||
} {
|
||||
req := httptest.NewRequest(http.MethodGet, path, nil)
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("path %s: status = %d, want 404", path, rec.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestOIDCDiscovery_NoAuth confirms the discovery routes bypass SigV4 — they
|
||||
// return 200 with no Authorization header, unlike the authenticated surface.
|
||||
func TestOIDCDiscovery_NoAuth(t *testing.T) {
|
||||
h := seedDiscoveryCluster(t)
|
||||
req := httptest.NewRequest(http.MethodGet,
|
||||
"/oidc/eks/ap-southeast-2/"+testDiscAccount+"/"+testDiscCluster+"/keys", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("unauthenticated discovery status = %d, want 200", rec.Code)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package gateway
|
||||
|
||||
import "github.com/aws/aws-sdk-go/service/iam"
|
||||
|
||||
// OIDC identity-provider registry methods — IRSA stubs so the mock IAM
|
||||
// services satisfy the IAMService interface. No gateway test exercises these
|
||||
// paths directly (covered by handlers/iam unit tests).
|
||||
|
||||
func (m *mockIAMService) CreateOpenIDConnectProvider(string, *iam.CreateOpenIDConnectProviderInput) (*iam.CreateOpenIDConnectProviderOutput, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (m *mockIAMService) GetOpenIDConnectProvider(string, *iam.GetOpenIDConnectProviderInput) (*iam.GetOpenIDConnectProviderOutput, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (m *mockIAMService) ListOpenIDConnectProviders(string, *iam.ListOpenIDConnectProvidersInput) (*iam.ListOpenIDConnectProvidersOutput, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (m *mockIAMService) DeleteOpenIDConnectProvider(string, *iam.DeleteOpenIDConnectProviderInput) (*iam.DeleteOpenIDConnectProviderOutput, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *flexMockIAMService) CreateOpenIDConnectProvider(string, *iam.CreateOpenIDConnectProviderInput) (*iam.CreateOpenIDConnectProviderOutput, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (m *flexMockIAMService) GetOpenIDConnectProvider(string, *iam.GetOpenIDConnectProviderInput) (*iam.GetOpenIDConnectProviderOutput, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (m *flexMockIAMService) ListOpenIDConnectProviders(string, *iam.ListOpenIDConnectProvidersInput) (*iam.ListOpenIDConnectProvidersOutput, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (m *flexMockIAMService) DeleteOpenIDConnectProvider(string, *iam.DeleteOpenIDConnectProviderInput) (*iam.DeleteOpenIDConnectProviderOutput, error) {
|
||||
return nil, nil
|
||||
}
|
||||
+21
-10
@@ -58,13 +58,11 @@ var stsActions = map[string]STSHandler{
|
||||
"AssumeRole": stsHandler(func(c stsCaller, input *sts.AssumeRoleInput, gw *GatewayConfig) (any, error) {
|
||||
return gateway_sts.AssumeRole(c.accountID, c.arn, c.identity, input, gw.STSService)
|
||||
}),
|
||||
// TODO: AssumeRoleWithWebIdentity is anonymous on AWS — the caller is
|
||||
// authenticated by the JWT body, not by a SigV4 envelope. The current
|
||||
// dispatch path runs SigV4 first, then enters this handler with a synthetic
|
||||
// stsCaller. That works for in-cluster callers that already hold SigV4
|
||||
// credentials but rejects the true anonymous code path (no AWS creds, only
|
||||
// a projected ServiceAccount token). The AWS gateway mux needs a pre-auth
|
||||
// route for this action so the JWT can be the sole authenticator.
|
||||
// AssumeRoleWithWebIdentity is anonymous on AWS — the caller is authenticated
|
||||
// by the JWT body, not by a SigV4 envelope. anonymousSTSInterceptor routes it
|
||||
// to this dispatcher ahead of the SigV4 middleware, so STS_Request enters with
|
||||
// a zero stsCaller (see anonymousSTSActions); the handler ignores it and
|
||||
// gates the request on the token + the role's web-identity trust policy.
|
||||
"AssumeRoleWithWebIdentity": stsHandler(func(_ stsCaller, input *sts.AssumeRoleWithWebIdentityInput, gw *GatewayConfig) (any, error) {
|
||||
return gateway_sts.AssumeRoleWithWebIdentity(input, gw.STSService)
|
||||
}),
|
||||
@@ -85,6 +83,14 @@ var stsSkipPolicyCheck = map[string]bool{
|
||||
"GetCallerIdentity": true,
|
||||
}
|
||||
|
||||
// anonymousSTSActions lists STS actions AWS accepts without SigV4: the caller
|
||||
// holds no AWS credentials and is authenticated solely by a web-identity JWT in
|
||||
// the request body. anonymousSTSInterceptor routes these ahead of the SigV4
|
||||
// middleware, and STS_Request skips SigV4-caller resolution for them.
|
||||
var anonymousSTSActions = map[string]bool{
|
||||
"AssumeRoleWithWebIdentity": true,
|
||||
}
|
||||
|
||||
func (gw *GatewayConfig) STS_Request(w http.ResponseWriter, r *http.Request) error {
|
||||
queryArgs, err := readQueryArgs(r)
|
||||
if err != nil {
|
||||
@@ -113,9 +119,14 @@ func (gw *GatewayConfig) STS_Request(w http.ResponseWriter, r *http.Request) err
|
||||
}
|
||||
}
|
||||
|
||||
caller, err := gw.resolveSTSCaller(r)
|
||||
if err != nil {
|
||||
return err
|
||||
// Anonymous actions (AssumeRoleWithWebIdentity) carry no SigV4 envelope, so
|
||||
// there is no caller to resolve — the handler ignores it.
|
||||
var caller stsCaller
|
||||
if !anonymousSTSActions[action] {
|
||||
caller, err = gw.resolveSTSCaller(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
xmlOutput, err := handler(action, queryArgs, gw, caller)
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
package gateway_sts
|
||||
|
||||
import "github.com/aws/aws-sdk-go/service/iam"
|
||||
|
||||
// OIDC identity-provider registry stubs so stubIAMService satisfies the
|
||||
// IAMService interface; STS gateway tests never call these.
|
||||
|
||||
func (s *stubIAMService) CreateOpenIDConnectProvider(string, *iam.CreateOpenIDConnectProviderInput) (*iam.CreateOpenIDConnectProviderOutput, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (s *stubIAMService) GetOpenIDConnectProvider(string, *iam.GetOpenIDConnectProviderInput) (*iam.GetOpenIDConnectProviderOutput, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (s *stubIAMService) ListOpenIDConnectProviders(string, *iam.ListOpenIDConnectProvidersInput) (*iam.ListOpenIDConnectProvidersOutput, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (s *stubIAMService) DeleteOpenIDConnectProvider(string, *iam.DeleteOpenIDConnectProviderInput) (*iam.DeleteOpenIDConnectProviderOutput, error) {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// anonymousSTSInterceptor routes the credential-bootstrap STS actions
|
||||
// (AssumeRoleWithWebIdentity) to the STS dispatcher ahead of the SigV4
|
||||
// middleware. These calls carry no AWS credentials — the caller is
|
||||
// authenticated by the projected ServiceAccount JWT in the request body — so
|
||||
// the signed surface would reject them as unauthenticated. Requests bearing an
|
||||
// Authorization header skip this path untouched, keeping the signed hot path
|
||||
// free of body buffering.
|
||||
func (gw *GatewayConfig) anonymousSTSInterceptor(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodPost && r.Header.Get("Authorization") == "" {
|
||||
if args, ok := gw.anonymousSTSArgs(r); ok {
|
||||
ctx := context.WithValue(r.Context(), ctxQueryArgs, args)
|
||||
ctx = context.WithValue(ctx, ctxService, "sts")
|
||||
r = r.WithContext(ctx)
|
||||
if err := gw.STS_Request(w, r); err != nil {
|
||||
gw.ErrorHandler(w, r, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// anonymousSTSArgs buffers and restores the request body, parses the AWS query
|
||||
// args, and reports whether Action is an STS action permitted without SigV4. A
|
||||
// non-anonymous request flows on unchanged with its body intact.
|
||||
func (gw *GatewayConfig) anonymousSTSArgs(r *http.Request) (map[string]string, bool) {
|
||||
body, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
r.Body = io.NopCloser(bytes.NewReader(body))
|
||||
|
||||
args, err := ParseAWSQueryArgs(string(body))
|
||||
if err != nil || !anonymousSTSActions[args["Action"]] {
|
||||
return nil, false
|
||||
}
|
||||
return args, true
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/sts"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestAnonymousSTS_WebIdentityNoAuthReachesHandler proves the IRSA bootstrap
|
||||
// call — AssumeRoleWithWebIdentity carrying no Authorization header — is routed
|
||||
// to the STS dispatcher ahead of the SigV4 middleware, rather than rejected as
|
||||
// unauthenticated. The mock asserts the parsed token reached the handler.
|
||||
func TestAnonymousSTS_WebIdentityNoAuthReachesHandler(t *testing.T) {
|
||||
var seenToken string
|
||||
gw := &GatewayConfig{
|
||||
DisableLogging: true,
|
||||
IAMService: &flexMockIAMService{},
|
||||
STSService: &flexMockSTSService{
|
||||
assumeWebIdentityFn: func(in *sts.AssumeRoleWithWebIdentityInput) (*sts.AssumeRoleWithWebIdentityOutput, error) {
|
||||
seenToken = aws.StringValue(in.WebIdentityToken)
|
||||
return &sts.AssumeRoleWithWebIdentityOutput{
|
||||
Credentials: &sts.Credentials{
|
||||
AccessKeyId: aws.String("ASIAIRSA"),
|
||||
SecretAccessKey: aws.String("secret"),
|
||||
SessionToken: aws.String("token"),
|
||||
},
|
||||
}, nil
|
||||
},
|
||||
},
|
||||
}
|
||||
handler := gw.SetupRoutes()
|
||||
|
||||
body := "Action=AssumeRoleWithWebIdentity&Version=2011-06-15" +
|
||||
"&RoleArn=arn:aws:iam::000000000001:role/demo&RoleSessionName=s1" +
|
||||
"&WebIdentityToken=header.payload.sig"
|
||||
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
// No Authorization header — this is the unsigned bootstrap path.
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rec, req)
|
||||
|
||||
require.Equal(t, http.StatusOK, rec.Code, "body=%s", rec.Body.String())
|
||||
b, _ := io.ReadAll(rec.Body)
|
||||
assert.Contains(t, string(b), "AssumeRoleWithWebIdentityResult")
|
||||
assert.Contains(t, string(b), "ASIAIRSA")
|
||||
assert.Equal(t, "header.payload.sig", seenToken)
|
||||
}
|
||||
|
||||
// TestAnonymousSTS_SignedRequestFallsThrough confirms a request bearing an
|
||||
// Authorization header is NOT intercepted — it flows to the signed surface, so
|
||||
// the anonymous dispatcher never runs and the body is left for the SigV4 path.
|
||||
func TestAnonymousSTS_SignedRequestFallsThrough(t *testing.T) {
|
||||
gw := &GatewayConfig{
|
||||
DisableLogging: true,
|
||||
STSService: &flexMockSTSService{
|
||||
assumeWebIdentityFn: func(*sts.AssumeRoleWithWebIdentityInput) (*sts.AssumeRoleWithWebIdentityOutput, error) {
|
||||
t.Fatal("signed request must not reach the anonymous STS dispatcher")
|
||||
return nil, nil
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
nextCalled := false
|
||||
next := http.HandlerFunc(func(http.ResponseWriter, *http.Request) { nextCalled = true })
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/",
|
||||
strings.NewReader("Action=AssumeRoleWithWebIdentity&WebIdentityToken=x"))
|
||||
req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential=...")
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
gw.anonymousSTSInterceptor(next).ServeHTTP(rec, req)
|
||||
assert.True(t, nextCalled, "signed request must pass through to next handler")
|
||||
}
|
||||
|
||||
// TestAnonymousSTSArgs_Classification covers the body-peek helper: an anonymous
|
||||
// action is recognised and the body is restored for the downstream handler; a
|
||||
// non-anonymous action is rejected and the body left intact for SigV4.
|
||||
func TestAnonymousSTSArgs_Classification(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true}
|
||||
|
||||
t.Run("anonymous action recognised, body restored", func(t *testing.T) {
|
||||
const body = "Action=AssumeRoleWithWebIdentity&WebIdentityToken=tok"
|
||||
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
|
||||
args, ok := gw.anonymousSTSArgs(req)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, "AssumeRoleWithWebIdentity", args["Action"])
|
||||
|
||||
restored, _ := io.ReadAll(req.Body)
|
||||
assert.Equal(t, body, string(restored), "body must be restored for downstream read")
|
||||
})
|
||||
|
||||
t.Run("non-anonymous action rejected", func(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("Action=GetCallerIdentity"))
|
||||
_, ok := gw.anonymousSTSArgs(req)
|
||||
assert.False(t, ok)
|
||||
})
|
||||
}
|
||||
@@ -26,6 +26,7 @@ type flexMockSTSService struct {
|
||||
assumeRoleFn func(string, string, string, *sts.AssumeRoleInput) (*sts.AssumeRoleOutput, error)
|
||||
getCallerIdentityFn func(string, string, string, *sts.GetCallerIdentityInput) (*sts.GetCallerIdentityOutput, error)
|
||||
lookupSessionFn func(string) (*handlers_sts.SessionCredential, error)
|
||||
assumeWebIdentityFn func(*sts.AssumeRoleWithWebIdentityInput) (*sts.AssumeRoleWithWebIdentityOutput, error)
|
||||
}
|
||||
|
||||
var _ handlers_sts.STSService = (*flexMockSTSService)(nil)
|
||||
@@ -65,7 +66,10 @@ func (m *flexMockSTSService) VerifySessionToken(*handlers_sts.SessionCredential,
|
||||
return true
|
||||
}
|
||||
|
||||
func (m *flexMockSTSService) AssumeRoleWithWebIdentity(*sts.AssumeRoleWithWebIdentityInput) (*sts.AssumeRoleWithWebIdentityOutput, error) {
|
||||
func (m *flexMockSTSService) AssumeRoleWithWebIdentity(input *sts.AssumeRoleWithWebIdentityInput) (*sts.AssumeRoleWithWebIdentityOutput, error) {
|
||||
if m.assumeWebIdentityFn != nil {
|
||||
return m.assumeWebIdentityFn(input)
|
||||
}
|
||||
return nil, errors.New(awserrors.ErrorNotImplemented)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/awserrors"
|
||||
gateway_tagging "github.com/mulgadc/spinifex/spinifex/gateway/tagging"
|
||||
)
|
||||
|
||||
// taggingHandler invokes a per-action Resource Groups Tagging API function: it
|
||||
// receives the parent gateway config, the caller's accountID, and the raw JSON
|
||||
// request body, and returns a typed *resourcegroupstaggingapi.<X>Output (as
|
||||
// any) or an error whose Error() is an awserrors code.
|
||||
type taggingHandler func(gw *GatewayConfig, accountID string, body []byte) (any, error)
|
||||
|
||||
// taggingActions maps a tagging action (the suffix of the X-Amz-Target header)
|
||||
// to its handler. The Resource Groups Tagging API speaks AWS JSON 1.1: the
|
||||
// action is carried in "X-Amz-Target: ResourceGroupsTaggingAPI_20170126.<Action>".
|
||||
var taggingActions = map[string]taggingHandler{
|
||||
"GetResources": func(gw *GatewayConfig, acct string, b []byte) (any, error) {
|
||||
return gateway_tagging.GetResources(gw.NATSConn, gw.Region, acct, b)
|
||||
},
|
||||
}
|
||||
|
||||
// taggingActionFromTarget extracts the bare action name from an X-Amz-Target
|
||||
// header value of the form "ResourceGroupsTaggingAPI_20170126.<Action>".
|
||||
func taggingActionFromTarget(target string) string {
|
||||
if i := strings.LastIndex(target, "."); i >= 0 {
|
||||
return target[i+1:]
|
||||
}
|
||||
return target
|
||||
}
|
||||
|
||||
// Tagging_Request dispatches AWS JSON 1.1 Resource Groups Tagging API requests.
|
||||
// The action is read from the X-Amz-Target header; the body is the JSON-encoded
|
||||
// operation input. Errors are returned as plain awserrors codes; the caller
|
||||
// (gateway.Request) routes them through ErrorHandler, which emits the JSON
|
||||
// error envelope.
|
||||
func (gw *GatewayConfig) Tagging_Request(w http.ResponseWriter, r *http.Request) error {
|
||||
action := taggingActionFromTarget(r.Header.Get("X-Amz-Target"))
|
||||
if action == "" {
|
||||
return errors.New(awserrors.ErrorMissingAction)
|
||||
}
|
||||
|
||||
handler, ok := taggingActions[action]
|
||||
if !ok {
|
||||
slog.Debug("tagging: unknown action", "action", action)
|
||||
return errors.New(awserrors.ErrorInvalidAction)
|
||||
}
|
||||
|
||||
if err := gw.checkPolicy(r, "tagging", action); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if gw.NATSConn == nil {
|
||||
return errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
|
||||
accountID, _ := r.Context().Value(ctxAccountID).(string)
|
||||
if accountID == "" {
|
||||
slog.Error("Tagging_Request: no account ID in auth context")
|
||||
return errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
slog.Error("Tagging_Request: failed to read body", "err", err)
|
||||
return errors.New(awserrors.ErrorInvalidParameterValue)
|
||||
}
|
||||
|
||||
output, err := handler(gw, accountID, body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
gateway_tagging.WriteJSONResponse(w, output)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,334 @@
|
||||
package gateway_tagging
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/aws/aws-sdk-go/service/elbv2"
|
||||
rgt "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
|
||||
"github.com/mulgadc/spinifex/spinifex/awserrors"
|
||||
handlers_ec2_tags "github.com/mulgadc/spinifex/spinifex/handlers/ec2/tags"
|
||||
handlers_elbv2 "github.com/mulgadc/spinifex/spinifex/handlers/elbv2"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// describeTagsBatchSize is the AWS ceiling on ARNs per ELBv2 DescribeTags call.
|
||||
const describeTagsBatchSize = 20
|
||||
|
||||
// defaultResourcesPerPage caps a page when the caller omits ResourcesPerPage.
|
||||
// AWS allows up to 100; matching that keeps the controller's reconcile fast.
|
||||
const defaultResourcesPerPage = 100
|
||||
|
||||
// resourceLister fetches an account's tagged resources by family, already
|
||||
// shaped as RGT mappings. Splitting the fetch out of getResources keeps the
|
||||
// filter/sort/paginate orchestration testable without a live NATS backend.
|
||||
type resourceLister interface {
|
||||
listELBv2(typeFilters map[string]bool) ([]*rgt.ResourceTagMapping, error)
|
||||
listEC2(typeFilters map[string]bool) ([]*rgt.ResourceTagMapping, error)
|
||||
}
|
||||
|
||||
// GetResources implements the Resource Groups Tagging API GetResources call by
|
||||
// aggregating tagged ELBv2 (load balancers + target groups) and EC2 resources
|
||||
// for the account, applying the request's resource-type and tag filters. The
|
||||
// aws-load-balancer-controller uses this to discover and adopt the resources it
|
||||
// manages (filtering on its cluster/stack tags). Pagination is an opaque
|
||||
// integer offset into a deterministically ARN-sorted result set.
|
||||
func GetResources(natsConn *nats.Conn, region, accountID string, body []byte) (any, error) {
|
||||
return getResources(&natsLister{natsConn: natsConn, region: region, accountID: accountID}, body)
|
||||
}
|
||||
|
||||
// getResources is the protocol-independent core: it parses the request, asks
|
||||
// the lister for the in-scope resource families, then applies tag filters,
|
||||
// stable ordering, and pagination.
|
||||
func getResources(lister resourceLister, body []byte) (any, error) {
|
||||
var input rgt.GetResourcesInput
|
||||
if err := unmarshalIfBody(body, &input); err != nil {
|
||||
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
|
||||
}
|
||||
|
||||
typeFilters := lowerStringSlice(input.ResourceTypeFilters)
|
||||
|
||||
var mappings []*rgt.ResourceTagMapping
|
||||
|
||||
elbMappings, err := lister.listELBv2(typeFilters)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mappings = append(mappings, elbMappings...)
|
||||
|
||||
ec2Mappings, err := lister.listEC2(typeFilters)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mappings = append(mappings, ec2Mappings...)
|
||||
|
||||
mappings = filterByTags(mappings, input.TagFilters)
|
||||
|
||||
// Deterministic order so pagination tokens are stable across pages.
|
||||
sort.Slice(mappings, func(i, j int) bool {
|
||||
return aws.StringValue(mappings[i].ResourceARN) < aws.StringValue(mappings[j].ResourceARN)
|
||||
})
|
||||
|
||||
page, next, err := paginate(mappings, input.PaginationToken, input.ResourcesPerPage)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out := &rgt.GetResourcesOutput{ResourceTagMappingList: page}
|
||||
// AWS returns an empty (not absent) token on the last page.
|
||||
out.PaginationToken = aws.String(next)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// natsLister fetches tagged resources from the elbv2 and ec2 tag stores over
|
||||
// NATS.
|
||||
type natsLister struct {
|
||||
natsConn *nats.Conn
|
||||
region string
|
||||
accountID string
|
||||
}
|
||||
|
||||
func (l *natsLister) listELBv2(typeFilters map[string]bool) ([]*rgt.ResourceTagMapping, error) {
|
||||
return elbv2Resources(l.natsConn, l.accountID, typeFilters)
|
||||
}
|
||||
|
||||
func (l *natsLister) listEC2(typeFilters map[string]bool) ([]*rgt.ResourceTagMapping, error) {
|
||||
svc := handlers_ec2_tags.NewNATSTagsService(l.natsConn)
|
||||
tagsOut, err := svc.DescribeTags(&ec2.DescribeTagsInput{}, l.accountID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buildEC2Mappings(tagsOut.Tags, l.region, l.accountID, typeFilters), nil
|
||||
}
|
||||
|
||||
// elbv2Resources lists the account's load balancers and target groups (when the
|
||||
// type filters admit them) and attaches their tags via batched DescribeTags.
|
||||
func elbv2Resources(natsConn *nats.Conn, accountID string, typeFilters map[string]bool) ([]*rgt.ResourceTagMapping, error) {
|
||||
wantLB := matchesType(typeFilters, "elasticloadbalancing:loadbalancer")
|
||||
wantTG := matchesType(typeFilters, "elasticloadbalancing:targetgroup")
|
||||
if !wantLB && !wantTG {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
svc := handlers_elbv2.NewNATSELBv2Service(natsConn)
|
||||
var arns []string
|
||||
|
||||
if wantLB {
|
||||
lbs, err := svc.DescribeLoadBalancers(&elbv2.DescribeLoadBalancersInput{}, accountID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, lb := range lbs.LoadBalancers {
|
||||
if lb.LoadBalancerArn != nil {
|
||||
arns = append(arns, *lb.LoadBalancerArn)
|
||||
}
|
||||
}
|
||||
}
|
||||
if wantTG {
|
||||
tgs, err := svc.DescribeTargetGroups(&elbv2.DescribeTargetGroupsInput{}, accountID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, tg := range tgs.TargetGroups {
|
||||
if tg.TargetGroupArn != nil {
|
||||
arns = append(arns, *tg.TargetGroupArn)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(arns) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
mappings := make([]*rgt.ResourceTagMapping, 0, len(arns))
|
||||
for start := 0; start < len(arns); start += describeTagsBatchSize {
|
||||
end := min(start+describeTagsBatchSize, len(arns))
|
||||
tagsOut, err := svc.DescribeTags(&elbv2.DescribeTagsInput{
|
||||
ResourceArns: aws.StringSlice(arns[start:end]),
|
||||
}, accountID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, td := range tagsOut.TagDescriptions {
|
||||
mappings = append(mappings, &rgt.ResourceTagMapping{
|
||||
ResourceARN: td.ResourceArn,
|
||||
Tags: elbv2TagsToRGT(td.Tags),
|
||||
})
|
||||
}
|
||||
}
|
||||
return mappings, nil
|
||||
}
|
||||
|
||||
// buildEC2Mappings groups EC2 tag descriptions by resource, synthesises an ARN
|
||||
// per resource, and keeps those that pass the type filters. EC2 tags carry a
|
||||
// bare resource ID and a type word (e.g. "subnet"); the RGT type string is
|
||||
// "ec2:<type>". Pure (no NATS) so the grouping/ARN logic is unit-tested.
|
||||
func buildEC2Mappings(tagDescriptions []*ec2.TagDescription, region, accountID string, typeFilters map[string]bool) []*rgt.ResourceTagMapping {
|
||||
type resourceTags struct {
|
||||
fullType string
|
||||
tags map[string]string
|
||||
}
|
||||
byResource := make(map[string]*resourceTags)
|
||||
for _, td := range tagDescriptions {
|
||||
id := aws.StringValue(td.ResourceId)
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
fullType := "ec2:" + aws.StringValue(td.ResourceType)
|
||||
rt := byResource[id]
|
||||
if rt == nil {
|
||||
rt = &resourceTags{fullType: fullType, tags: map[string]string{}}
|
||||
byResource[id] = rt
|
||||
}
|
||||
rt.tags[aws.StringValue(td.Key)] = aws.StringValue(td.Value)
|
||||
}
|
||||
|
||||
var mappings []*rgt.ResourceTagMapping
|
||||
for id, rt := range byResource {
|
||||
if !matchesType(typeFilters, rt.fullType) {
|
||||
continue
|
||||
}
|
||||
mappings = append(mappings, &rgt.ResourceTagMapping{
|
||||
ResourceARN: aws.String(ec2ARN(region, accountID, rt.fullType, id)),
|
||||
Tags: tagMapToRGT(rt.tags),
|
||||
})
|
||||
}
|
||||
return mappings
|
||||
}
|
||||
|
||||
// ec2ARN builds an arn:aws:ec2 ARN for a resource. fullType is "ec2:<type>".
|
||||
func ec2ARN(region, accountID, fullType, id string) string {
|
||||
resourceType := strings.TrimPrefix(fullType, "ec2:")
|
||||
return fmt.Sprintf("arn:aws:ec2:%s:%s:%s/%s", region, accountID, resourceType, id)
|
||||
}
|
||||
|
||||
// matchesType reports whether a resource of fullType (e.g.
|
||||
// "elasticloadbalancing:loadbalancer") passes the request's type filters. An
|
||||
// empty filter set admits everything. A service-only filter ("ec2") admits all
|
||||
// of that service's types; a fully-qualified filter must match exactly.
|
||||
func matchesType(typeFilters map[string]bool, fullType string) bool {
|
||||
if len(typeFilters) == 0 {
|
||||
return true
|
||||
}
|
||||
if typeFilters[fullType] {
|
||||
return true
|
||||
}
|
||||
if svc, _, ok := strings.Cut(fullType, ":"); ok {
|
||||
if typeFilters[svc] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// filterByTags keeps only resources matching every TagFilter (AND across
|
||||
// filters). Within a filter, a non-empty Values list is an OR over values; an
|
||||
// empty Values list matches any value for that key.
|
||||
func filterByTags(mappings []*rgt.ResourceTagMapping, filters []*rgt.TagFilter) []*rgt.ResourceTagMapping {
|
||||
if len(filters) == 0 {
|
||||
return mappings
|
||||
}
|
||||
out := mappings[:0]
|
||||
for _, m := range mappings {
|
||||
if matchesAllTagFilters(m.Tags, filters) {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func matchesAllTagFilters(tags []*rgt.Tag, filters []*rgt.TagFilter) bool {
|
||||
tagMap := make(map[string]string, len(tags))
|
||||
for _, t := range tags {
|
||||
tagMap[aws.StringValue(t.Key)] = aws.StringValue(t.Value)
|
||||
}
|
||||
for _, f := range filters {
|
||||
key := aws.StringValue(f.Key)
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
val, ok := tagMap[key]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
if len(f.Values) == 0 {
|
||||
continue
|
||||
}
|
||||
matched := false
|
||||
for _, v := range f.Values {
|
||||
if aws.StringValue(v) == val {
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// paginate slices an offset-token page out of the sorted mappings and returns
|
||||
// the page plus the next token ("" when exhausted).
|
||||
func paginate(mappings []*rgt.ResourceTagMapping, token *string, perPage *int64) ([]*rgt.ResourceTagMapping, string, error) {
|
||||
start := 0
|
||||
if token != nil && *token != "" {
|
||||
n, err := strconv.Atoi(*token)
|
||||
if err != nil || n < 0 {
|
||||
return nil, "", errors.New(awserrors.ErrorInvalidParameterValue)
|
||||
}
|
||||
start = n
|
||||
}
|
||||
if start > len(mappings) {
|
||||
start = len(mappings)
|
||||
}
|
||||
|
||||
size := defaultResourcesPerPage
|
||||
if perPage != nil && *perPage > 0 {
|
||||
size = int(*perPage)
|
||||
}
|
||||
|
||||
end := start + size
|
||||
if end >= len(mappings) {
|
||||
return mappings[start:], "", nil
|
||||
}
|
||||
return mappings[start:end], strconv.Itoa(end), nil
|
||||
}
|
||||
|
||||
func elbv2TagsToRGT(tags []*elbv2.Tag) []*rgt.Tag {
|
||||
out := make([]*rgt.Tag, 0, len(tags))
|
||||
for _, t := range tags {
|
||||
out = append(out, &rgt.Tag{Key: t.Key, Value: t.Value})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func tagMapToRGT(tags map[string]string) []*rgt.Tag {
|
||||
keys := make([]string, 0, len(tags))
|
||||
for k := range tags {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
out := make([]*rgt.Tag, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
out = append(out, &rgt.Tag{Key: aws.String(k), Value: aws.String(tags[k])})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func lowerStringSlice(in []*string) map[string]bool {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]bool, len(in))
|
||||
for _, s := range in {
|
||||
if s != nil {
|
||||
out[strings.ToLower(*s)] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
package gateway_tagging
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
rgt "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// fakeLister returns canned mappings so getResources can be exercised without a
|
||||
// NATS backend.
|
||||
type fakeLister struct {
|
||||
elbv2 []*rgt.ResourceTagMapping
|
||||
ec2 []*rgt.ResourceTagMapping
|
||||
elbv2Err error
|
||||
ec2Err error
|
||||
gotFilter map[string]bool
|
||||
}
|
||||
|
||||
func (f *fakeLister) listELBv2(typeFilters map[string]bool) ([]*rgt.ResourceTagMapping, error) {
|
||||
f.gotFilter = typeFilters
|
||||
return f.elbv2, f.elbv2Err
|
||||
}
|
||||
|
||||
func (f *fakeLister) listEC2(typeFilters map[string]bool) ([]*rgt.ResourceTagMapping, error) {
|
||||
return f.ec2, f.ec2Err
|
||||
}
|
||||
|
||||
func mustJSON(t *testing.T, v any) []byte {
|
||||
t.Helper()
|
||||
b, err := json.Marshal(v)
|
||||
require.NoError(t, err)
|
||||
return b
|
||||
}
|
||||
|
||||
func TestMatchesType(t *testing.T) {
|
||||
// Empty filter set admits everything.
|
||||
assert.True(t, matchesType(nil, "elasticloadbalancing:loadbalancer"))
|
||||
|
||||
exact := map[string]bool{"elasticloadbalancing:loadbalancer": true}
|
||||
assert.True(t, matchesType(exact, "elasticloadbalancing:loadbalancer"))
|
||||
assert.False(t, matchesType(exact, "elasticloadbalancing:targetgroup"))
|
||||
assert.False(t, matchesType(exact, "ec2:subnet"))
|
||||
|
||||
// Service-only filter admits all of that service's types.
|
||||
svcOnly := map[string]bool{"ec2": true}
|
||||
assert.True(t, matchesType(svcOnly, "ec2:subnet"))
|
||||
assert.True(t, matchesType(svcOnly, "ec2:security-group"))
|
||||
assert.False(t, matchesType(svcOnly, "elasticloadbalancing:loadbalancer"))
|
||||
}
|
||||
|
||||
func TestEC2ARN(t *testing.T) {
|
||||
assert.Equal(t,
|
||||
"arn:aws:ec2:us-east-1:123456789012:subnet/subnet-abc",
|
||||
ec2ARN("us-east-1", "123456789012", "ec2:subnet", "subnet-abc"))
|
||||
}
|
||||
|
||||
func mapping(arn string, kv ...string) *rgt.ResourceTagMapping {
|
||||
var tags []*rgt.Tag
|
||||
for i := 0; i+1 < len(kv); i += 2 {
|
||||
tags = append(tags, &rgt.Tag{Key: aws.String(kv[i]), Value: aws.String(kv[i+1])})
|
||||
}
|
||||
return &rgt.ResourceTagMapping{ResourceARN: aws.String(arn), Tags: tags}
|
||||
}
|
||||
|
||||
func tagFilter(key string, values ...string) *rgt.TagFilter {
|
||||
return &rgt.TagFilter{Key: aws.String(key), Values: aws.StringSlice(values)}
|
||||
}
|
||||
|
||||
func TestFilterByTags(t *testing.T) {
|
||||
in := []*rgt.ResourceTagMapping{
|
||||
mapping("arn:a", "elbv2.k8s.aws/cluster", "prod", "env", "live"),
|
||||
mapping("arn:b", "elbv2.k8s.aws/cluster", "dev"),
|
||||
mapping("arn:c", "env", "live"),
|
||||
}
|
||||
|
||||
// No filters: passthrough.
|
||||
assert.Len(t, filterByTags(in, nil), 3)
|
||||
|
||||
// Key-only filter: any value for the key.
|
||||
got := filterByTags(in, []*rgt.TagFilter{tagFilter("elbv2.k8s.aws/cluster")})
|
||||
require.Len(t, got, 2)
|
||||
|
||||
// Key+value filter: exact value match (OR over values).
|
||||
got = filterByTags(in, []*rgt.TagFilter{tagFilter("elbv2.k8s.aws/cluster", "prod", "staging")})
|
||||
require.Len(t, got, 1)
|
||||
assert.Equal(t, "arn:a", aws.StringValue(got[0].ResourceARN))
|
||||
|
||||
// AND across filters.
|
||||
got = filterByTags(in, []*rgt.TagFilter{
|
||||
tagFilter("elbv2.k8s.aws/cluster", "prod"),
|
||||
tagFilter("env", "live"),
|
||||
})
|
||||
require.Len(t, got, 1)
|
||||
assert.Equal(t, "arn:a", aws.StringValue(got[0].ResourceARN))
|
||||
|
||||
// Filter on a key no resource has: empty result.
|
||||
assert.Empty(t, filterByTags(in, []*rgt.TagFilter{tagFilter("missing")}))
|
||||
}
|
||||
|
||||
func TestPaginate(t *testing.T) {
|
||||
all := []*rgt.ResourceTagMapping{
|
||||
mapping("arn:0"), mapping("arn:1"), mapping("arn:2"), mapping("arn:3"), mapping("arn:4"),
|
||||
}
|
||||
|
||||
// First page of 2, expect a next token.
|
||||
page, next, err := paginate(all, nil, aws.Int64(2))
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, page, 2)
|
||||
assert.Equal(t, "2", next)
|
||||
|
||||
// Follow the token to the middle page.
|
||||
page, next, err = paginate(all, aws.String("2"), aws.Int64(2))
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, page, 2)
|
||||
assert.Equal(t, "4", next)
|
||||
|
||||
// Last partial page: no next token.
|
||||
page, next, err = paginate(all, aws.String("4"), aws.Int64(2))
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, page, 1)
|
||||
assert.Equal(t, "", next)
|
||||
|
||||
// Default page size returns everything in one page.
|
||||
page, next, err = paginate(all, nil, nil)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, page, 5)
|
||||
assert.Equal(t, "", next)
|
||||
|
||||
// Bad token is rejected.
|
||||
_, _, err = paginate(all, aws.String("not-a-number"), nil)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestGetResources_MergesFiltersSortsPaginates(t *testing.T) {
|
||||
lister := &fakeLister{
|
||||
elbv2: []*rgt.ResourceTagMapping{
|
||||
mapping("arn:elb:c", "elbv2.k8s.aws/cluster", "prod"),
|
||||
mapping("arn:elb:a", "elbv2.k8s.aws/cluster", "dev"),
|
||||
},
|
||||
ec2: []*rgt.ResourceTagMapping{
|
||||
mapping("arn:ec2:b", "elbv2.k8s.aws/cluster", "prod"),
|
||||
},
|
||||
}
|
||||
|
||||
// Tag-filter to prod + page size 1: two matches (arn:ec2:b, arn:elb:c),
|
||||
// ARN-sorted, first page returns arn:ec2:b with a next token.
|
||||
body := mustJSON(t, rgt.GetResourcesInput{
|
||||
TagFilters: []*rgt.TagFilter{tagFilter("elbv2.k8s.aws/cluster", "prod")},
|
||||
ResourcesPerPage: aws.Int64(1),
|
||||
})
|
||||
out, err := getResources(lister, body)
|
||||
require.NoError(t, err)
|
||||
res, ok := out.(*rgt.GetResourcesOutput)
|
||||
require.True(t, ok)
|
||||
require.Len(t, res.ResourceTagMappingList, 1)
|
||||
assert.Equal(t, "arn:ec2:b", aws.StringValue(res.ResourceTagMappingList[0].ResourceARN))
|
||||
assert.Equal(t, "1", aws.StringValue(res.PaginationToken))
|
||||
|
||||
// Follow the token: last page, arn:elb:c, empty token.
|
||||
body = mustJSON(t, rgt.GetResourcesInput{
|
||||
TagFilters: []*rgt.TagFilter{tagFilter("elbv2.k8s.aws/cluster", "prod")},
|
||||
ResourcesPerPage: aws.Int64(1),
|
||||
PaginationToken: aws.String("1"),
|
||||
})
|
||||
out, err = getResources(lister, body)
|
||||
require.NoError(t, err)
|
||||
res, ok = out.(*rgt.GetResourcesOutput)
|
||||
require.True(t, ok)
|
||||
require.Len(t, res.ResourceTagMappingList, 1)
|
||||
assert.Equal(t, "arn:elb:c", aws.StringValue(res.ResourceTagMappingList[0].ResourceARN))
|
||||
assert.Equal(t, "", aws.StringValue(res.PaginationToken))
|
||||
}
|
||||
|
||||
func TestGetResources_ResourceTypeFiltersLowercasedAndPassed(t *testing.T) {
|
||||
lister := &fakeLister{}
|
||||
body := mustJSON(t, rgt.GetResourcesInput{
|
||||
ResourceTypeFilters: aws.StringSlice([]string{"ElasticLoadBalancing:LoadBalancer"}),
|
||||
})
|
||||
_, err := getResources(lister, body)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, lister.gotFilter["elasticloadbalancing:loadbalancer"])
|
||||
}
|
||||
|
||||
func TestGetResources_ListerErrorPropagates(t *testing.T) {
|
||||
lister := &fakeLister{elbv2Err: assert.AnError}
|
||||
_, err := getResources(lister, []byte("{}"))
|
||||
require.ErrorIs(t, err, assert.AnError)
|
||||
}
|
||||
|
||||
func TestGetResources_InvalidBody(t *testing.T) {
|
||||
_, err := getResources(&fakeLister{}, []byte("not-json"))
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestBuildEC2Mappings(t *testing.T) {
|
||||
tags := []*ec2.TagDescription{
|
||||
{ResourceId: aws.String("subnet-1"), ResourceType: aws.String("subnet"), Key: aws.String("Name"), Value: aws.String("a")},
|
||||
{ResourceId: aws.String("subnet-1"), ResourceType: aws.String("subnet"), Key: aws.String("env"), Value: aws.String("prod")},
|
||||
{ResourceId: aws.String("sg-1"), ResourceType: aws.String("security-group"), Key: aws.String("Name"), Value: aws.String("b")},
|
||||
{ResourceId: aws.String(""), ResourceType: aws.String("subnet"), Key: aws.String("skip"), Value: aws.String("me")},
|
||||
}
|
||||
|
||||
// No type filter: both resources, tags merged per resource.
|
||||
all := buildEC2Mappings(tags, "us-east-1", "123456789012", nil)
|
||||
require.Len(t, all, 2)
|
||||
bySubnet := map[string]*rgt.ResourceTagMapping{}
|
||||
for _, m := range all {
|
||||
bySubnet[aws.StringValue(m.ResourceARN)] = m
|
||||
}
|
||||
subnet := bySubnet["arn:aws:ec2:us-east-1:123456789012:subnet/subnet-1"]
|
||||
require.NotNil(t, subnet)
|
||||
assert.Len(t, subnet.Tags, 2)
|
||||
|
||||
// Type filter narrows to subnets only.
|
||||
onlySubnets := buildEC2Mappings(tags, "us-east-1", "123456789012", map[string]bool{"ec2:subnet": true})
|
||||
require.Len(t, onlySubnets, 1)
|
||||
assert.Equal(t, "arn:aws:ec2:us-east-1:123456789012:subnet/subnet-1", aws.StringValue(onlySubnets[0].ResourceARN))
|
||||
}
|
||||
|
||||
func TestTagMapToRGT_SortedDeterministic(t *testing.T) {
|
||||
out := tagMapToRGT(map[string]string{"b": "2", "a": "1", "c": "3"})
|
||||
require.Len(t, out, 3)
|
||||
assert.Equal(t, "a", aws.StringValue(out[0].Key))
|
||||
assert.Equal(t, "b", aws.StringValue(out[1].Key))
|
||||
assert.Equal(t, "c", aws.StringValue(out[2].Key))
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
// Package gateway_tagging is the HTTP-side glue between awsgw and the Resource
|
||||
// Groups Tagging API. Like ACM it speaks AWS JSON 1.1 (X-Amz-Target dispatch,
|
||||
// Content-Type application/x-amz-json-1.1). GetResources is answered by
|
||||
// aggregating the elbv2 and ec2 tag stores over NATS — there is no dedicated
|
||||
// tagging store. Error envelopes are emitted centrally by the shared gateway
|
||||
// ErrorHandler.
|
||||
package gateway_tagging
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
"github.com/aws/aws-sdk-go/private/protocol/json/jsonutil"
|
||||
"github.com/mulgadc/spinifex/spinifex/awserrors"
|
||||
)
|
||||
|
||||
// JSONContentType is the AWS JSON 1.1 content type tagging clients expect.
|
||||
const JSONContentType = "application/x-amz-json-1.1"
|
||||
|
||||
// WriteJSONResponse serialises obj as a 200 AWS JSON 1.1 response using the
|
||||
// SDK's own marshaler (epoch-seconds timestamps), the same path ACM uses.
|
||||
func WriteJSONResponse(w http.ResponseWriter, obj any) {
|
||||
body, err := jsonutil.BuildJSON(obj)
|
||||
if err != nil {
|
||||
slog.Error("tagging: failed to marshal response JSON", "err", err)
|
||||
http.Error(w, awserrors.ErrorInternalError, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", JSONContentType)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
if _, err := w.Write(body); err != nil {
|
||||
slog.Error("tagging: failed to write response", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// unmarshalIfBody decodes body into out only when body is non-empty.
|
||||
func unmarshalIfBody(body []byte, out any) error {
|
||||
if len(body) == 0 {
|
||||
return nil
|
||||
}
|
||||
return json.Unmarshal(body, out)
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/awserrors"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func setupTaggingRequest(target, body string) *http.Request {
|
||||
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
|
||||
if target != "" {
|
||||
req.Header.Set("X-Amz-Target", target)
|
||||
}
|
||||
ctx := context.WithValue(req.Context(), ctxService, "tagging")
|
||||
ctx = context.WithValue(ctx, ctxAccountID, "123456789012")
|
||||
return req.WithContext(ctx)
|
||||
}
|
||||
|
||||
func TestTaggingActionFromTarget(t *testing.T) {
|
||||
assert.Equal(t, "GetResources", taggingActionFromTarget("ResourceGroupsTaggingAPI_20170126.GetResources"))
|
||||
assert.Equal(t, "GetResources", taggingActionFromTarget("GetResources"))
|
||||
assert.Equal(t, "", taggingActionFromTarget(""))
|
||||
}
|
||||
|
||||
func TestTaggingActionsMap_AllActionsRegistered(t *testing.T) {
|
||||
expected := []string{
|
||||
"GetResources",
|
||||
}
|
||||
for _, action := range expected {
|
||||
_, ok := taggingActions[action]
|
||||
assert.True(t, ok, "action %q should be registered in taggingActions", action)
|
||||
}
|
||||
assert.Len(t, taggingActions, len(expected))
|
||||
}
|
||||
|
||||
func TestTaggingRequest_MissingTarget(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true}
|
||||
w := httptest.NewRecorder()
|
||||
err := gw.Tagging_Request(w, setupTaggingRequest("", ""))
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorMissingAction, err.Error())
|
||||
}
|
||||
|
||||
func TestTaggingRequest_UnknownAction(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true}
|
||||
w := httptest.NewRecorder()
|
||||
err := gw.Tagging_Request(w, setupTaggingRequest("ResourceGroupsTaggingAPI_20170126.TagResources", "{}"))
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorInvalidAction, err.Error())
|
||||
}
|
||||
|
||||
// A known action with no NATS connection passes routing + policy and fails at
|
||||
// the NATS-availability guard.
|
||||
func TestTaggingRequest_KnownActionNoNATS(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true}
|
||||
w := httptest.NewRecorder()
|
||||
err := gw.Tagging_Request(w, setupTaggingRequest("ResourceGroupsTaggingAPI_20170126.GetResources", "{}"))
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
|
||||
}
|
||||
@@ -10,24 +10,24 @@ import (
|
||||
// --- generateNetworkConfig ---
|
||||
|
||||
func TestGenerateNetworkConfig_BothEmpty(t *testing.T) {
|
||||
cfg := generateNetworkConfig("", "", "", "", nil)
|
||||
cfg := generateNetworkConfig("", "", "", "", nil, true)
|
||||
assert.Equal(t, cloudInitNetworkConfigWildcard, cfg)
|
||||
}
|
||||
|
||||
func TestGenerateNetworkConfig_OneEmpty(t *testing.T) {
|
||||
cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "", "", "", nil)
|
||||
cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "", "", "", nil, true)
|
||||
assert.Contains(t, cfg, "vpc0:", "eniMAC alone should produce per-interface config")
|
||||
assert.NotContains(t, cfg, "dev0:", "no dev NIC without devMAC")
|
||||
// The IMDS on-link route rides vpc0 even with no dev NIC / extra ENIs — the
|
||||
// common production shape — so guard it here too, not only on the multi-NIC path.
|
||||
assert.Contains(t, cfg, "to: 169.254.169.254/32", "vpc0 must carry the IMDS on-link route")
|
||||
|
||||
cfg = generateNetworkConfig("", "02:00:00:dd:ee:ff", "", "", nil)
|
||||
cfg = generateNetworkConfig("", "02:00:00:dd:ee:ff", "", "", nil, true)
|
||||
assert.Equal(t, cloudInitNetworkConfigWildcard, cfg, "should fall back to wildcard if eniMAC empty")
|
||||
}
|
||||
|
||||
func TestGenerateNetworkConfig_DualNIC(t *testing.T) {
|
||||
cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:00:00:dd:ee:ff", "", "", nil)
|
||||
cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:00:00:dd:ee:ff", "", "", nil, true)
|
||||
assert.Contains(t, cfg, "version: 2")
|
||||
assert.Contains(t, cfg, `macaddress: "02:00:00:aa:bb:cc"`)
|
||||
assert.Contains(t, cfg, `macaddress: "02:00:00:dd:ee:ff"`)
|
||||
@@ -39,7 +39,7 @@ func TestGenerateNetworkConfig_DualNIC(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGenerateNetworkConfig_TripleNIC(t *testing.T) {
|
||||
cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:de:00:dd:ee:ff", "02:a0:00:11:22:33", "10.15.8.101", nil)
|
||||
cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:de:00:dd:ee:ff", "02:a0:00:11:22:33", "10.15.8.101", nil, true)
|
||||
assert.Contains(t, cfg, "version: 2")
|
||||
assert.Contains(t, cfg, `macaddress: "02:00:00:aa:bb:cc"`)
|
||||
assert.Contains(t, cfg, `macaddress: "02:de:00:dd:ee:ff"`)
|
||||
@@ -53,7 +53,7 @@ func TestGenerateNetworkConfig_TripleNIC(t *testing.T) {
|
||||
|
||||
func TestGenerateNetworkConfig_MgmtWithoutDev(t *testing.T) {
|
||||
// System instances: eniMAC + mgmtMAC, no devMAC — should get per-interface config with mgmt NIC
|
||||
cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "", "02:a0:00:11:22:33", "10.15.8.101", nil)
|
||||
cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "", "02:a0:00:11:22:33", "10.15.8.101", nil, true)
|
||||
assert.Contains(t, cfg, "vpc0:")
|
||||
assert.NotContains(t, cfg, "dev0:", "no dev NIC without devMAC")
|
||||
assert.Contains(t, cfg, "mgmt0:")
|
||||
@@ -62,12 +62,12 @@ func TestGenerateNetworkConfig_MgmtWithoutDev(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGenerateNetworkConfig_MgmtMACWithoutIP(t *testing.T) {
|
||||
cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:de:00:dd:ee:ff", "02:a0:00:11:22:33", "", nil)
|
||||
cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:de:00:dd:ee:ff", "02:a0:00:11:22:33", "", nil, true)
|
||||
assert.NotContains(t, cfg, "mgmt0:", "mgmt NIC should not appear without IP")
|
||||
}
|
||||
|
||||
func TestGenerateNetworkConfig_MgmtIPWithoutMAC(t *testing.T) {
|
||||
cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:de:00:dd:ee:ff", "", "10.15.8.101", nil)
|
||||
cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:de:00:dd:ee:ff", "", "10.15.8.101", nil, true)
|
||||
assert.NotContains(t, cfg, "mgmt0:", "mgmt NIC should not appear without MAC")
|
||||
}
|
||||
|
||||
@@ -75,7 +75,7 @@ func TestGenerateNetworkConfig_IMDSOnLinkRoute(t *testing.T) {
|
||||
// The primary VPC NIC carries an on-link route to the IMDS metadata IP so the
|
||||
// guest ARPs 169.254.169.254 directly; the per-subnet localport answers.
|
||||
cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:00:00:dd:ee:ff", "", "",
|
||||
[]string{"02:00:00:bb:bb:bb"})
|
||||
[]string{"02:00:00:bb:bb:bb"}, true)
|
||||
assert.Contains(t, cfg, "to: 169.254.169.254/32")
|
||||
assert.Contains(t, cfg, "scope: link")
|
||||
// Only the primary NIC (vpc0) gets it — not extra VPC NICs or the dev NIC,
|
||||
@@ -83,5 +83,41 @@ func TestGenerateNetworkConfig_IMDSOnLinkRoute(t *testing.T) {
|
||||
assert.Equal(t, 1, strings.Count(cfg, "169.254.169.254/32"))
|
||||
}
|
||||
|
||||
func TestGenerateNetworkConfig_IMDSRouteOmittedWhenDisabled(t *testing.T) {
|
||||
// Alpine's eni renderer crashes on a gateway-less route, so the route is
|
||||
// omitted from network-config and delivered out-of-band (see
|
||||
// buildAlpineCloudInit). emitIMDSRoute=false must drop it entirely.
|
||||
cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:00:00:dd:ee:ff", "", "", nil, false)
|
||||
assert.Contains(t, cfg, "vpc0:")
|
||||
assert.NotContains(t, cfg, "169.254.169.254/32", "route must not render when emitIMDSRoute=false")
|
||||
assert.NotContains(t, cfg, " routes:", "no vpc0 routes block when the only route is the IMDS one")
|
||||
}
|
||||
|
||||
// selectNetworkConfigForFamily must omit the IMDS route for Alpine (eni renderer
|
||||
// can't handle the gateway-less route) but keep it for everyone else.
|
||||
func TestSelectNetworkConfigForFamily_AlpineOmitsIMDSRoute(t *testing.T) {
|
||||
alpine := selectNetworkConfigForFamily("alpine", "02:00:00:aa:bb:cc", "02:00:00:dd:ee:ff", "", "", nil)
|
||||
assert.NotContains(t, alpine, "169.254.169.254/32", "Alpine network-config must not carry the IMDS route")
|
||||
|
||||
deb := selectNetworkConfigForFamily("debian", "02:00:00:aa:bb:cc", "02:00:00:dd:ee:ff", "", "", nil)
|
||||
assert.Contains(t, deb, "169.254.169.254/32", "non-Alpine families keep the netplan IMDS route")
|
||||
}
|
||||
|
||||
// buildAlpineCloudInit delivers the IMDS on-link route via a persistent
|
||||
// /etc/local.d script + OpenRC local service, since the eni renderer can't.
|
||||
func TestBuildAlpineCloudInit(t *testing.T) {
|
||||
wf, rc := buildAlpineCloudInit("02:00:00:aa:bb:cc")
|
||||
assert.Contains(t, wf, "/etc/local.d/imds-onlink-route.start")
|
||||
assert.Contains(t, wf, "ip route show default")
|
||||
assert.Contains(t, wf, `ip route replace 169.254.169.254/32 dev "$dev" scope link`)
|
||||
assert.Contains(t, rc, "rc-update")
|
||||
assert.Contains(t, rc, "local")
|
||||
|
||||
// No VPC NIC → nothing to add.
|
||||
wf, rc = buildAlpineCloudInit("")
|
||||
assert.Empty(t, wf)
|
||||
assert.Empty(t, rc)
|
||||
}
|
||||
|
||||
// Route for multi-node LB mgmt traffic is handled via the fw_cfg netcfg
|
||||
// blob delivered to the LB microVM, not in the network-config.
|
||||
|
||||
@@ -64,19 +64,21 @@ ca_certs:
|
||||
{{.UserDataCloudConfig}}
|
||||
{{end}}
|
||||
|
||||
{{if or .RHELWriteFiles .UserDataScript}}
|
||||
{{if or .RHELWriteFiles .AlpineWriteFiles .UserDataScript}}
|
||||
write_files:
|
||||
{{- if .RHELWriteFiles}}
|
||||
{{.RHELWriteFiles}}{{end}}{{if .UserDataScript}}
|
||||
{{.RHELWriteFiles}}{{end}}{{if .AlpineWriteFiles}}
|
||||
{{.AlpineWriteFiles}}{{end}}{{if .UserDataScript}}
|
||||
- path: /tmp/cloud-init-startup.sh
|
||||
permissions: '0755'
|
||||
content: |
|
||||
{{.UserDataScript}}{{end}}
|
||||
{{end}}
|
||||
{{if or .RHELRunCmd .UserDataScript}}
|
||||
{{if or .RHELRunCmd .AlpineRunCmd .UserDataScript}}
|
||||
runcmd:
|
||||
{{- if .RHELRunCmd}}
|
||||
{{.RHELRunCmd}}{{end}}{{if .UserDataScript}}
|
||||
{{.RHELRunCmd}}{{end}}{{if .AlpineRunCmd}}
|
||||
{{.AlpineRunCmd}}{{end}}{{if .UserDataScript}}
|
||||
- [ "/bin/bash", "/tmp/cloud-init-startup.sh" ]
|
||||
{{end}}
|
||||
{{end}}
|
||||
@@ -118,7 +120,11 @@ func selectNetworkConfigForFamily(distroFamily, eniMAC, devMAC, mgmtMAC, mgmtIP
|
||||
if distroFamily == "rhel" {
|
||||
return cloudInitNetworkConfigDisabled
|
||||
}
|
||||
return generateNetworkConfig(eniMAC, devMAC, mgmtMAC, mgmtIP, extraENIMACs)
|
||||
// Alpine renders via cloud-init's eni backend, which cannot emit a
|
||||
// gateway-less route; the IMDS on-link route is delivered via
|
||||
// buildAlpineCloudInit instead. Debian/Ubuntu render via netplan, which
|
||||
// handles the on-link route natively.
|
||||
return generateNetworkConfig(eniMAC, devMAC, mgmtMAC, mgmtIP, extraENIMACs, distroFamily != "alpine")
|
||||
}
|
||||
|
||||
// generateNetworkConfig produces the cloud-init network-config for the instance.
|
||||
@@ -142,7 +148,7 @@ func selectNetworkConfigForFamily(distroFamily, eniMAC, devMAC, mgmtMAC, mgmtIP
|
||||
// route is reached via the primary ENI, matching AWS; it is not delivered via
|
||||
// DHCP option 121 (which would force the default route into option 121 per
|
||||
// RFC 3442) and the default gateway stays option 3.
|
||||
func generateNetworkConfig(eniMAC, devMAC, mgmtMAC, mgmtIP string, extraENIMACs []string) string {
|
||||
func generateNetworkConfig(eniMAC, devMAC, mgmtMAC, mgmtIP string, extraENIMACs []string, emitIMDSRoute bool) string {
|
||||
if eniMAC == "" {
|
||||
return cloudInitNetworkConfigWildcard
|
||||
}
|
||||
@@ -154,10 +160,17 @@ func generateNetworkConfig(eniMAC, devMAC, mgmtMAC, mgmtIP string, extraENIMACs
|
||||
macaddress: "%s"
|
||||
dhcp4: true
|
||||
dhcp-identifier: mac
|
||||
routes:
|
||||
`, eniMAC)
|
||||
// The gateway-less IMDS on-link route renders only under netplan. cloud-init's
|
||||
// eni renderer (Alpine) crashes on a route with no gateway, taking the whole
|
||||
// network-config down with it, so eni-rendered guests get this route via
|
||||
// buildAlpineCloudInit instead (emitIMDSRoute=false).
|
||||
if emitIMDSRoute {
|
||||
cfg += fmt.Sprintf(` routes:
|
||||
- to: %s/32
|
||||
scope: link
|
||||
`, eniMAC, handlers_imds.MetaDataServerIP)
|
||||
`, handlers_imds.MetaDataServerIP)
|
||||
}
|
||||
|
||||
for i, mac := range extraENIMACs {
|
||||
if mac == "" {
|
||||
@@ -220,6 +233,12 @@ type CloudInitData struct {
|
||||
// + nmcli reload/up) needed to load and bring up the keyfiles. Empty on
|
||||
// non-RHEL guests.
|
||||
RHELRunCmd string
|
||||
// AlpineWriteFiles / AlpineRunCmd carry the IMDS on-link route delivery for
|
||||
// Alpine guests (a persistent /etc/local.d script + OpenRC `local` enable),
|
||||
// since the eni renderer cannot emit the gateway-less route in the
|
||||
// network-config. Empty on non-Alpine guests and on non-VPC instances.
|
||||
AlpineWriteFiles string
|
||||
AlpineRunCmd string
|
||||
}
|
||||
|
||||
// buildRHELCloudInit produces the cloud-init write_files (NM keyfiles) and
|
||||
@@ -322,6 +341,43 @@ func appendRHELStaticKeyfile(b *strings.Builder, name, mac, ip string) {
|
||||
b.WriteString(" method=disabled\n")
|
||||
}
|
||||
|
||||
// buildAlpineCloudInit produces the cloud-init write_files + runcmd blocks that
|
||||
// deliver the IMDS on-link route on Alpine guests. Returns empty strings when
|
||||
// eniMAC is empty (wildcard / non-VPC path).
|
||||
//
|
||||
// Alpine renders networking via cloud-init's eni backend, which crashes on a
|
||||
// gateway-less route (it assumes every route has a next hop), so the route is
|
||||
// omitted from the netplan network-config (see generateNetworkConfig) and added
|
||||
// here instead. A persistent /etc/local.d script keeps the on-link route present
|
||||
// across reboots — runcmd runs once per instance, so the OpenRC `local` service
|
||||
// re-applies it on every boot. cloud-init's eni renderer renames the primary
|
||||
// VPC NIC to vpc0 via a 70-persistent-net.rules udev rule, so the device name is
|
||||
// stable. `ip route replace` is idempotent.
|
||||
func buildAlpineCloudInit(eniMAC string) (writeFiles, runCmd string) {
|
||||
if eniMAC == "" {
|
||||
return "", ""
|
||||
}
|
||||
|
||||
var wf strings.Builder
|
||||
wf.WriteString(" - path: /etc/local.d/imds-onlink-route.start\n")
|
||||
wf.WriteString(" owner: root:root\n")
|
||||
wf.WriteString(" permissions: '0755'\n")
|
||||
wf.WriteString(" content: |\n")
|
||||
wf.WriteString(" #!/bin/sh\n")
|
||||
// Resolve the primary VPC NIC via its default route, not by name: the
|
||||
// persistent-net udev rename to vpc0 doesn't apply on the Alpine AMI, so the
|
||||
// kernel name is eth0 at first boot. The mgmt NIC (eth1) carries no default
|
||||
// route, so this always picks the VPC egress NIC — the one IMDS rides.
|
||||
wf.WriteString(" dev=$(ip route show default | awk '{print $5; exit}')\n")
|
||||
fmt.Fprintf(&wf, " [ -n \"$dev\" ] && ip route replace %s/32 dev \"$dev\" scope link\n", handlers_imds.MetaDataServerIP)
|
||||
|
||||
var rc strings.Builder
|
||||
rc.WriteString(" - [ rc-update, add, local, default ]\n")
|
||||
rc.WriteString(" - [ rc-service, local, start ]\n")
|
||||
|
||||
return strings.TrimRight(wf.String(), "\n"), strings.TrimRight(rc.String(), "\n")
|
||||
}
|
||||
|
||||
type CloudInitMetaData struct {
|
||||
InstanceID string
|
||||
Hostname string
|
||||
@@ -1871,7 +1927,8 @@ func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, i
|
||||
|
||||
sudoGroup := "sudo"
|
||||
var rhelWriteFiles, rhelRunCmd string
|
||||
if distroFamily == "rhel" {
|
||||
switch distroFamily {
|
||||
case "rhel":
|
||||
sudoGroup = "wheel"
|
||||
rhelWriteFiles, rhelRunCmd = buildRHELCloudInit(instance.ENIMac, instance.DevMAC, instance.MgmtMAC, instance.MgmtIP, extraMACs)
|
||||
}
|
||||
@@ -1909,6 +1966,15 @@ func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, i
|
||||
}
|
||||
}
|
||||
|
||||
// Alpine's eni renderer can't emit the gateway-less IMDS route, so deliver it
|
||||
// out-of-band via a local.d script. Only when the caller didn't supply its own
|
||||
// #cloud-config: that payload owns the write_files/runcmd keys (e.g. the EKS
|
||||
// k3s server VM, which bakes the same route into its own block), and a second
|
||||
// write_files key here would collide — last key wins, silently dropping a block.
|
||||
if distroFamily == "alpine" && userData.UserDataCloudConfig == "" {
|
||||
userData.AlpineWriteFiles, userData.AlpineRunCmd = buildAlpineCloudInit(instance.ENIMac)
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
t := template.Must(template.New("cloud-init").Parse(cloudInitUserDataTemplate))
|
||||
|
||||
@@ -2155,6 +2221,16 @@ func (s *InstanceServiceImpl) DescribeInstances(input *ec2.DescribeInstancesInpu
|
||||
if !IsInstanceVisible(accountID, instance.AccountID) {
|
||||
continue
|
||||
}
|
||||
// Platform-managed system VMs (LB HAProxy, EKS control plane) carry a
|
||||
// ManagedBy tag and must stay out of customer EC2 listings, the same
|
||||
// way they're filtered from the UI Nodes page. LB VMs are owned by the
|
||||
// system account so IsInstanceVisible already hides them, but the EKS
|
||||
// control-plane VM is owned by the customer account (its ENI lives in
|
||||
// the customer VPC), so guard on ManagedBy here. Root/operator callers
|
||||
// still see system instances.
|
||||
if instance.ManagedBy != "" && accountID != utils.GlobalAccountID {
|
||||
continue
|
||||
}
|
||||
if len(instanceIDFilter) > 0 && !instanceIDFilter[instance.ID] {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"github.com/mulgadc/spinifex/spinifex/awserrors"
|
||||
"github.com/mulgadc/spinifex/spinifex/config"
|
||||
"github.com/mulgadc/spinifex/spinifex/gpu"
|
||||
"github.com/mulgadc/spinifex/spinifex/tags"
|
||||
spxtypes "github.com/mulgadc/spinifex/spinifex/types"
|
||||
"github.com/mulgadc/spinifex/spinifex/utils"
|
||||
"github.com/mulgadc/spinifex/spinifex/vm"
|
||||
@@ -662,7 +663,7 @@ func TestParseVolumeParams_PartialEbs(t *testing.T) {
|
||||
|
||||
func TestCloudInitNetworkConfigWildcard(t *testing.T) {
|
||||
// No MACs → wildcard config (non-VPC or VPC without DEV_NETWORKING)
|
||||
cfg := generateNetworkConfig("", "", "", "", nil)
|
||||
cfg := generateNetworkConfig("", "", "", "", nil, true)
|
||||
assert.Contains(t, cfg, "version: 2")
|
||||
assert.Contains(t, cfg, "dhcp4: true")
|
||||
assert.Contains(t, cfg, "dhcp-identifier: mac")
|
||||
@@ -674,7 +675,7 @@ func TestCloudInitNetworkConfigDualNIC(t *testing.T) {
|
||||
eniMAC := "02:00:00:61:ef:c2"
|
||||
devMAC := "02:de:00:60:83:0d"
|
||||
|
||||
cfg := generateNetworkConfig(eniMAC, devMAC, "", "", nil)
|
||||
cfg := generateNetworkConfig(eniMAC, devMAC, "", "", nil, true)
|
||||
|
||||
// Both MACs present in config
|
||||
assert.Contains(t, cfg, eniMAC)
|
||||
@@ -694,12 +695,12 @@ func TestCloudInitNetworkConfigDualNIC(t *testing.T) {
|
||||
|
||||
func TestCloudInitNetworkConfigPartialMAC(t *testing.T) {
|
||||
// Only ENI MAC (VPC without dev) → per-interface config with VPC NIC only
|
||||
cfg := generateNetworkConfig("02:00:00:61:ef:c2", "", "", "", nil)
|
||||
cfg := generateNetworkConfig("02:00:00:61:ef:c2", "", "", "", nil, true)
|
||||
assert.Contains(t, cfg, "vpc0:")
|
||||
assert.NotContains(t, cfg, "dev0:")
|
||||
|
||||
// Only dev MAC (shouldn't happen, but defensive) → wildcard
|
||||
cfg = generateNetworkConfig("", "02:de:00:60:83:0d", "", "", nil)
|
||||
cfg = generateNetworkConfig("", "02:de:00:60:83:0d", "", "", nil, true)
|
||||
assert.Contains(t, cfg, `name: "e*"`)
|
||||
assert.NotContains(t, cfg, "use-routes")
|
||||
}
|
||||
@@ -710,7 +711,7 @@ func TestCloudInitNetworkConfigMultiVPCNICs(t *testing.T) {
|
||||
"02:00:00:bb:bb:bb",
|
||||
"02:00:00:cc:cc:cc",
|
||||
}
|
||||
cfg := generateNetworkConfig("02:00:00:aa:aa:aa", "", "", "", extras)
|
||||
cfg := generateNetworkConfig("02:00:00:aa:aa:aa", "", "", "", extras, true)
|
||||
|
||||
assert.Contains(t, cfg, "vpc0:")
|
||||
assert.Contains(t, cfg, "vpc1:")
|
||||
@@ -729,7 +730,7 @@ func TestCloudInitNetworkConfigMultiVPCNICs(t *testing.T) {
|
||||
func TestCloudInitNetworkConfigEmptyExtraMACSkipped(t *testing.T) {
|
||||
// Empty strings inside the extras slice are ignored rather than producing
|
||||
// a malformed ethernets block.
|
||||
cfg := generateNetworkConfig("02:00:00:aa:aa:aa", "", "", "", []string{""})
|
||||
cfg := generateNetworkConfig("02:00:00:aa:aa:aa", "", "", "", []string{""}, true)
|
||||
assert.Contains(t, cfg, "vpc0:")
|
||||
assert.NotContains(t, cfg, "vpc1:")
|
||||
}
|
||||
@@ -1081,6 +1082,70 @@ func TestDescribeInstances_AccountFilteringHidesOtherTenant(t *testing.T) {
|
||||
assert.Empty(t, out.Reservations)
|
||||
}
|
||||
|
||||
// EKS control-plane VMs are owned by the customer account (their ENI lives in
|
||||
// the customer VPC) but are platform-managed system instances, so they must be
|
||||
// hidden from the owning customer's DescribeInstances the same way LB VMs are.
|
||||
func TestDescribeInstances_HidesManagedSystemVMFromCustomer(t *testing.T) {
|
||||
owner := "111122223333"
|
||||
v := &vm.VM{
|
||||
ID: "i-ekscp",
|
||||
AccountID: owner,
|
||||
ManagedBy: tags.ManagedByEKS,
|
||||
Reservation: &ec2.Reservation{
|
||||
ReservationId: aws.String("r-ekscp"),
|
||||
OwnerId: aws.String(owner),
|
||||
},
|
||||
Instance: &ec2.Instance{InstanceId: aws.String("i-ekscp")},
|
||||
}
|
||||
svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{v.ID: v})}
|
||||
|
||||
out, err := svc.DescribeInstances(&ec2.DescribeInstancesInput{}, owner)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, out.Reservations, "managed system VM must not appear in customer listing")
|
||||
}
|
||||
|
||||
// Root/operator callers still see managed system VMs.
|
||||
func TestDescribeInstances_RootSeesManagedSystemVM(t *testing.T) {
|
||||
v := &vm.VM{
|
||||
ID: "i-lb",
|
||||
AccountID: utils.GlobalAccountID,
|
||||
ManagedBy: tags.ManagedByELBv2,
|
||||
Reservation: &ec2.Reservation{
|
||||
ReservationId: aws.String("r-lb"),
|
||||
OwnerId: aws.String(utils.GlobalAccountID),
|
||||
},
|
||||
Instance: &ec2.Instance{InstanceId: aws.String("i-lb")},
|
||||
}
|
||||
svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{v.ID: v})}
|
||||
|
||||
out, err := svc.DescribeInstances(&ec2.DescribeInstancesInput{}, utils.GlobalAccountID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, out.Reservations, 1)
|
||||
assert.Equal(t, "i-lb", *out.Reservations[0].Instances[0].InstanceId)
|
||||
}
|
||||
|
||||
// A customer's own unmanaged instance (no ManagedBy) stays visible — confirms
|
||||
// the exclusion keys on ManagedBy, not on account scope. Future EKS worker
|
||||
// nodes (customer-owned, no ManagedBy tag) follow this path.
|
||||
func TestDescribeInstances_CustomerWorkloadStaysVisible(t *testing.T) {
|
||||
owner := "111122223333"
|
||||
v := &vm.VM{
|
||||
ID: "i-worker",
|
||||
AccountID: owner,
|
||||
Reservation: &ec2.Reservation{
|
||||
ReservationId: aws.String("r-worker"),
|
||||
OwnerId: aws.String(owner),
|
||||
},
|
||||
Instance: &ec2.Instance{InstanceId: aws.String("i-worker")},
|
||||
}
|
||||
svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{v.ID: v})}
|
||||
|
||||
out, err := svc.DescribeInstances(&ec2.DescribeInstancesInput{}, owner)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, out.Reservations, 1)
|
||||
assert.Equal(t, "i-worker", *out.Reservations[0].Instances[0].InstanceId)
|
||||
}
|
||||
|
||||
func TestDescribeInstances_MalformedID(t *testing.T) {
|
||||
svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{})}
|
||||
input := &ec2.DescribeInstancesInput{
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
@@ -21,8 +22,18 @@ const (
|
||||
// (absolute), so a VM that never boots reaches a terminal state even across
|
||||
// daemon restarts. Plan Risk #1.
|
||||
defaultCreateTimeout = 15 * time.Minute
|
||||
// defaultStateStaleAfter bounds how old the control plane's last NATS state
|
||||
// report may be before health is treated as unknown/failed. The CP publishes
|
||||
// every ~30s; 3x that tolerates a missed tick without flapping.
|
||||
defaultStateStaleAfter = 90 * time.Second
|
||||
)
|
||||
|
||||
// natsSubscriber is the minimum surface the reconciler needs to consume the
|
||||
// control-plane state report; *nats.Conn satisfies it, tests stub it.
|
||||
type natsSubscriber interface {
|
||||
Subscribe(subject string, cb nats.MsgHandler) (*nats.Subscription, error)
|
||||
}
|
||||
|
||||
// ErrReconcilerLeaseLost is returned by Run when the leader lease was lost
|
||||
// mid-loop. Callers can re-Acquire and re-Run if they want to keep driving
|
||||
// reconcile from the same node.
|
||||
@@ -58,6 +69,14 @@ type ClusterReconciler struct {
|
||||
healthTimeout time.Duration
|
||||
createTimeout time.Duration
|
||||
httpClient HTTPDoer
|
||||
|
||||
// State-report health source. When stateSub is non-nil the reconciler gates
|
||||
// on the CP's NATS self-report (latest) rather than the HTTP /healthz probe —
|
||||
// the apiserver is bound to the VPC node-ip and unreachable from the host.
|
||||
stateSub natsSubscriber
|
||||
stateSubject string
|
||||
stateStaleAfter time.Duration
|
||||
latest atomic.Pointer[ServerStateReport]
|
||||
}
|
||||
|
||||
// ReconcilerOption tunes a ClusterReconciler. Tests use the With* helpers to
|
||||
@@ -91,6 +110,22 @@ func WithHTTPClient(c HTTPDoer) ReconcilerOption {
|
||||
return func(r *ClusterReconciler) { r.httpClient = c }
|
||||
}
|
||||
|
||||
// WithStateSource configures the reconciler to gate health on the control
|
||||
// plane's NATS state report (subject) consumed via sub, instead of the HTTP
|
||||
// /healthz probe. The subscription is opened in Run and closed when it returns.
|
||||
func WithStateSource(sub natsSubscriber, subject string) ReconcilerOption {
|
||||
return func(r *ClusterReconciler) {
|
||||
r.stateSub = sub
|
||||
r.stateSubject = subject
|
||||
}
|
||||
}
|
||||
|
||||
// WithStateStaleAfter overrides how old the last state report may be before
|
||||
// health is treated as failing.
|
||||
func WithStateStaleAfter(d time.Duration) ReconcilerOption {
|
||||
return func(r *ClusterReconciler) { r.stateStaleAfter = d }
|
||||
}
|
||||
|
||||
// NewClusterReconciler constructs a ClusterReconciler. healthURL is the
|
||||
// fully-qualified probe target (e.g. https://nlb-dns:443/healthz) or empty
|
||||
// to skip /healthz probing (CREATING transitions on bootstrap state alone).
|
||||
@@ -111,16 +146,17 @@ func NewClusterReconciler(leaderKV, acctKV nats.KeyValue, accountID, clusterName
|
||||
return nil, errors.New("eks: NewClusterReconciler empty holderID")
|
||||
}
|
||||
r := &ClusterReconciler{
|
||||
leaderKV: leaderKV,
|
||||
acctKV: acctKV,
|
||||
accountID: accountID,
|
||||
clusterName: clusterName,
|
||||
holderID: holderID,
|
||||
healthURL: healthURL,
|
||||
leaseRefresh: defaultReconcileLeaseRefresh,
|
||||
interval: defaultReconcileInterval,
|
||||
healthTimeout: defaultHealthzTimeout,
|
||||
createTimeout: defaultCreateTimeout,
|
||||
leaderKV: leaderKV,
|
||||
acctKV: acctKV,
|
||||
accountID: accountID,
|
||||
clusterName: clusterName,
|
||||
holderID: holderID,
|
||||
healthURL: healthURL,
|
||||
leaseRefresh: defaultReconcileLeaseRefresh,
|
||||
interval: defaultReconcileInterval,
|
||||
healthTimeout: defaultHealthzTimeout,
|
||||
createTimeout: defaultCreateTimeout,
|
||||
stateStaleAfter: defaultStateStaleAfter,
|
||||
httpClient: &http.Client{
|
||||
Timeout: defaultHealthzTimeout,
|
||||
Transport: &http.Transport{
|
||||
@@ -186,6 +222,22 @@ func (r *ClusterReconciler) RefreshLease() bool {
|
||||
// cluster reaches a terminal state. Caller is expected to AcquireLease first;
|
||||
// Run does not re-acquire.
|
||||
func (r *ClusterReconciler) Run(ctx context.Context) error {
|
||||
if r.stateSub != nil && r.stateSubject != "" {
|
||||
sub, err := r.stateSub.Subscribe(r.stateSubject, func(m *nats.Msg) {
|
||||
report, perr := unmarshalServerStateReport(m.Data)
|
||||
if perr != nil {
|
||||
slog.Warn("ClusterReconciler: bad state report",
|
||||
"cluster", r.clusterName, "subject", m.Subject, "err", perr)
|
||||
return
|
||||
}
|
||||
r.latest.Store(&report)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("subscribe state report %s: %w", r.stateSubject, err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
}
|
||||
|
||||
refreshT := time.NewTicker(r.leaseRefresh)
|
||||
defer refreshT.Stop()
|
||||
reconcileT := time.NewTicker(r.interval)
|
||||
@@ -238,24 +290,30 @@ func (r *ClusterReconciler) reconcileOnce(ctx context.Context) error {
|
||||
"cluster", r.clusterName, "reason", reason)
|
||||
return r.failIfCreateTimedOut(meta, "bootstrap not ready: "+reason)
|
||||
}
|
||||
if err := r.probeHealthz(ctx); err != nil {
|
||||
slog.Debug("ClusterReconciler: /healthz still failing",
|
||||
"cluster", r.clusterName, "err", err)
|
||||
return r.failIfCreateTimedOut(meta, "healthz not ready: "+err.Error())
|
||||
issue, nodeCount := r.observe(ctx)
|
||||
if issue != "" {
|
||||
slog.Debug("ClusterReconciler: health not ready",
|
||||
"cluster", r.clusterName, "issue", issue)
|
||||
return r.failIfCreateTimedOut(meta, "healthz not ready: "+issue)
|
||||
}
|
||||
if err := SetClusterStatus(r.acctKV, r.clusterName, ClusterStatusActive); err != nil {
|
||||
return err
|
||||
}
|
||||
slog.Info("ClusterReconciler: cluster transitioned to ACTIVE",
|
||||
"cluster", r.clusterName)
|
||||
case ClusterStatusActive:
|
||||
issue := ""
|
||||
if err := r.probeHealthz(ctx); err != nil {
|
||||
slog.Warn("ClusterReconciler: /healthz failing for ACTIVE cluster",
|
||||
"cluster", r.clusterName, "err", err)
|
||||
issue = err.Error()
|
||||
if err := SetClusterHealthState(r.acctKV, r.clusterName, "", nodeCount); err != nil {
|
||||
if errors.Is(err, ErrClusterNotFound) {
|
||||
return ErrClusterNotFound
|
||||
}
|
||||
return fmt.Errorf("record cluster health: %w", err)
|
||||
}
|
||||
if err := SetClusterHealth(r.acctKV, r.clusterName, issue); err != nil {
|
||||
slog.Info("ClusterReconciler: cluster transitioned to ACTIVE",
|
||||
"cluster", r.clusterName, "nodes", nodeCount)
|
||||
case ClusterStatusActive:
|
||||
issue, nodeCount := r.observe(ctx)
|
||||
if issue != "" {
|
||||
slog.Warn("ClusterReconciler: health failing for ACTIVE cluster",
|
||||
"cluster", r.clusterName, "issue", issue)
|
||||
}
|
||||
if err := SetClusterHealthState(r.acctKV, r.clusterName, issue, nodeCount); err != nil {
|
||||
if errors.Is(err, ErrClusterNotFound) {
|
||||
return ErrClusterNotFound
|
||||
}
|
||||
@@ -269,6 +327,31 @@ func (r *ClusterReconciler) reconcileOnce(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// observe returns the current health issue ("" when healthy) and the reported
|
||||
// node count. With a NATS state source it reads the control plane's latest
|
||||
// self-report (rejecting a stale or unhealthy one); otherwise it falls back to
|
||||
// the HTTP /healthz probe (node count is then 0, the probe carries no count).
|
||||
func (r *ClusterReconciler) observe(ctx context.Context) (issue string, nodeCount int) {
|
||||
if r.stateSub != nil && r.stateSubject != "" {
|
||||
report := r.latest.Load()
|
||||
if report == nil {
|
||||
return "no control-plane state report received yet", 0
|
||||
}
|
||||
age := time.Since(time.Unix(report.TS, 0))
|
||||
if age > r.stateStaleAfter {
|
||||
return fmt.Sprintf("control-plane state report stale (%s old)", age.Round(time.Second)), report.NodeCount
|
||||
}
|
||||
if !report.Healthy() {
|
||||
return fmt.Sprintf("apiserver healthz=%q", report.Healthz), report.NodeCount
|
||||
}
|
||||
return "", report.NodeCount
|
||||
}
|
||||
if err := r.probeHealthz(ctx); err != nil {
|
||||
return err.Error(), 0
|
||||
}
|
||||
return "", 0
|
||||
}
|
||||
|
||||
// failIfCreateTimedOut transitions a still-CREATING cluster to FAILED once it
|
||||
// has exceeded createTimeout since meta.CreatedAt, and returns
|
||||
// ErrReconcilerClusterFailed so Run exits the loop. While still within the
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
package handlers_eks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/testutil"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// newStateReconcilerHarness builds a reconciler wired to a real NATS conn as its
|
||||
// state source (so observe() reads r.latest) and returns the conn + account KV.
|
||||
// It does NOT start a subscription — tests drive r.latest directly for
|
||||
// determinism; the integration test below exercises the real Subscribe path.
|
||||
func newStateReconcilerHarness(t *testing.T, opts ...ReconcilerOption) (*ClusterReconciler, *nats.Conn, nats.KeyValue) {
|
||||
t.Helper()
|
||||
_, nc, js := testutil.StartTestJetStream(t)
|
||||
leaderKV, err := InitLeaderBucket(js)
|
||||
require.NoError(t, err)
|
||||
acctKV, err := GetOrCreateAccountBucket(js, testAccountID)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, PutClusterMeta(acctKV, sampleClusterMeta("alpha")))
|
||||
|
||||
base := []ReconcilerOption{WithStateSource(nc, StateSubject(testAccountID, "alpha"))}
|
||||
r, err := NewClusterReconciler(leaderKV, acctKV, testAccountID, "alpha", "holder-1", "", append(base, opts...)...)
|
||||
require.NoError(t, err)
|
||||
return r, nc, acctKV
|
||||
}
|
||||
|
||||
func freshReport(healthz string, nodes int) *ServerStateReport {
|
||||
return &ServerStateReport{Healthz: healthz, NodeCount: nodes, TS: time.Now().Unix()}
|
||||
}
|
||||
|
||||
func TestClusterReconciler_CreatingTransitionsToActiveOnStateReport(t *testing.T) {
|
||||
r, _, acctKV := newStateReconcilerHarness(t)
|
||||
freshenClusterCreatedAt(t, acctKV)
|
||||
seedBootstrapState(t, acctKV)
|
||||
r.latest.Store(freshReport("ok", 3))
|
||||
|
||||
require.NoError(t, r.reconcileOnce(context.Background()))
|
||||
|
||||
meta, err := GetClusterMeta(acctKV, "alpha")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, ClusterStatusActive, meta.Status)
|
||||
assert.Equal(t, 3, meta.NodeCount)
|
||||
assert.Empty(t, meta.HealthIssue)
|
||||
}
|
||||
|
||||
func TestClusterReconciler_CreatingStaysWithoutStateReport(t *testing.T) {
|
||||
r, _, acctKV := newStateReconcilerHarness(t)
|
||||
freshenClusterCreatedAt(t, acctKV)
|
||||
seedBootstrapState(t, acctKV)
|
||||
// No report stored.
|
||||
|
||||
require.NoError(t, r.reconcileOnce(context.Background()))
|
||||
|
||||
meta, err := GetClusterMeta(acctKV, "alpha")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, ClusterStatusCreating, meta.Status, "no state report yet → stays CREATING")
|
||||
}
|
||||
|
||||
func TestClusterReconciler_CreatingStaysOnUnhealthyReport(t *testing.T) {
|
||||
r, _, acctKV := newStateReconcilerHarness(t)
|
||||
freshenClusterCreatedAt(t, acctKV)
|
||||
seedBootstrapState(t, acctKV)
|
||||
r.latest.Store(freshReport("fail", 0))
|
||||
|
||||
require.NoError(t, r.reconcileOnce(context.Background()))
|
||||
|
||||
meta, err := GetClusterMeta(acctKV, "alpha")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, ClusterStatusCreating, meta.Status, "unhealthy apiserver → stays CREATING")
|
||||
}
|
||||
|
||||
func TestClusterReconciler_ActiveRecordsNodeCountAndClearsIssue(t *testing.T) {
|
||||
r, _, acctKV := newStateReconcilerHarness(t)
|
||||
require.NoError(t, SetClusterStatus(acctKV, "alpha", ClusterStatusActive))
|
||||
require.NoError(t, SetClusterHealthState(acctKV, "alpha", "stale", 0))
|
||||
r.latest.Store(freshReport("ok", 4))
|
||||
|
||||
require.NoError(t, r.reconcileOnce(context.Background()))
|
||||
|
||||
meta, err := GetClusterMeta(acctKV, "alpha")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, ClusterStatusActive, meta.Status)
|
||||
assert.Equal(t, 4, meta.NodeCount)
|
||||
assert.Empty(t, meta.HealthIssue, "healthy report clears the prior issue")
|
||||
}
|
||||
|
||||
func TestClusterReconciler_ActiveFlagsStaleReport(t *testing.T) {
|
||||
r, _, acctKV := newStateReconcilerHarness(t, WithStateStaleAfter(90*time.Second))
|
||||
require.NoError(t, SetClusterStatus(acctKV, "alpha", ClusterStatusActive))
|
||||
r.latest.Store(&ServerStateReport{Healthz: "ok", NodeCount: 2, TS: time.Now().Add(-5 * time.Minute).Unix()})
|
||||
|
||||
require.NoError(t, r.reconcileOnce(context.Background()))
|
||||
|
||||
meta, err := GetClusterMeta(acctKV, "alpha")
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, meta.HealthIssue, "stale", "report older than stale window flags an issue")
|
||||
assert.Equal(t, 2, meta.NodeCount, "last known node count still recorded")
|
||||
}
|
||||
|
||||
func TestClusterReconciler_StateReportSubscriptionDrivesActive(t *testing.T) {
|
||||
r, nc, acctKV := newStateReconcilerHarness(t,
|
||||
WithReconcileInterval(10*time.Millisecond),
|
||||
WithLeaseRefresh(10*time.Second),
|
||||
)
|
||||
freshenClusterCreatedAt(t, acctKV)
|
||||
seedBootstrapState(t, acctKV)
|
||||
|
||||
release, ok := r.AcquireLease()
|
||||
require.True(t, ok)
|
||||
defer release()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
runErr := make(chan error, 1)
|
||||
go func() { runErr <- r.Run(ctx) }()
|
||||
|
||||
// Publish healthy reports until the cluster goes ACTIVE (core NATS only
|
||||
// delivers post-subscribe, so keep emitting while Run's subscription opens).
|
||||
stop := make(chan struct{})
|
||||
defer close(stop)
|
||||
go func() {
|
||||
payload, _ := json.Marshal(ServerStateReport{Healthz: "ok", NodeCount: 3, TS: time.Now().Unix()})
|
||||
tick := time.NewTicker(20 * time.Millisecond)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
case <-tick.C:
|
||||
_ = nc.Publish(StateSubject(testAccountID, "alpha"), payload)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
require.Eventually(t, func() bool {
|
||||
meta, err := GetClusterMeta(acctKV, "alpha")
|
||||
return err == nil && meta.Status == ClusterStatusActive && meta.NodeCount == 3
|
||||
}, 1500*time.Millisecond, 10*time.Millisecond, "state reports should drive CREATING → ACTIVE")
|
||||
|
||||
cancel()
|
||||
select {
|
||||
case err := <-runErr:
|
||||
assert.ErrorIs(t, err, context.Canceled)
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("Run did not return after cancel")
|
||||
}
|
||||
}
|
||||
@@ -28,19 +28,32 @@ type ClusterVpcConfig struct {
|
||||
SubnetIds []string `json:"subnetIds"`
|
||||
SecurityGroupIds []string `json:"securityGroupIds,omitempty"`
|
||||
VpcId string `json:"vpcId,omitempty"`
|
||||
// EndpointPublicAccess / EndpointPrivateAccess control apiserver endpoint
|
||||
// exposure (AWS resourcesVpcConfig parity). Public ⇒ internet-facing cluster
|
||||
// NLB (external-pool front-end IP, LAN-reachable); private ⇒ internal NLB
|
||||
// (VPC-only). PublicAccessCidrs records the allowed source ranges for the
|
||||
// public endpoint (stored for parity + Describe; ingress enforcement is a
|
||||
// follow-up).
|
||||
EndpointPublicAccess bool `json:"endpointPublicAccess"`
|
||||
EndpointPrivateAccess bool `json:"endpointPrivateAccess"`
|
||||
PublicAccessCidrs []string `json:"publicAccessCidrs,omitempty"`
|
||||
}
|
||||
|
||||
// ClusterMeta is the persisted control-plane record for one cluster. The
|
||||
// blob lives at ClusterMetaKey(name) inside the per-account KV bucket and is
|
||||
// the source of truth for DescribeCluster.
|
||||
type ClusterMeta struct {
|
||||
Name string `json:"name"`
|
||||
Arn string `json:"arn"`
|
||||
Status ClusterStatus `json:"status"`
|
||||
StatusReason string `json:"statusReason,omitempty"`
|
||||
Version string `json:"version"`
|
||||
RoleArn string `json:"roleArn"`
|
||||
Endpoint string `json:"endpoint,omitempty"`
|
||||
Name string `json:"name"`
|
||||
Arn string `json:"arn"`
|
||||
Status ClusterStatus `json:"status"`
|
||||
StatusReason string `json:"statusReason,omitempty"`
|
||||
Version string `json:"version"`
|
||||
RoleArn string `json:"roleArn"`
|
||||
Endpoint string `json:"endpoint,omitempty"`
|
||||
// EndpointIP is the host/LAN-reachable front-end IP the cluster NLB exposes
|
||||
// (public-access ⇒ external-pool IP; private ⇒ VPC IP). The apiserver serving
|
||||
// cert SANs this IP, so kubectl reaches Endpoint with TLS verification on.
|
||||
EndpointIP string `json:"endpointIp,omitempty"`
|
||||
OIDCIssuer string `json:"oidcIssuer,omitempty"`
|
||||
CertificateAuthorityB64 string `json:"certificateAuthorityB64,omitempty"`
|
||||
ResourcesVpcConfig *ClusterVpcConfig `json:"resourcesVpcConfig,omitempty"`
|
||||
@@ -69,6 +82,15 @@ type ClusterMeta struct {
|
||||
// only) and never treats a time.Time struct as empty, so this field always
|
||||
// serializes. The tag states that honestly rather than implying it elides.
|
||||
LastHealthProbe time.Time `json:"lastHealthProbe"`
|
||||
// NodeCount is the node total from the control plane's last NATS state
|
||||
// report (server + joined agents). Surfaces nodegroup join/scale progress
|
||||
// without the host needing apiserver reachability.
|
||||
NodeCount int `json:"nodeCount,omitempty"`
|
||||
// BuiltinIngress records whether the cluster opted into K3s' bundled
|
||||
// traefik + servicelb (dev / interim in-VPC app exposure), derived from the
|
||||
// managed-ingress tag at CreateCluster. Default false = AWS parity (built-ins
|
||||
// disabled; ingress via the AWS Load Balancer Controller).
|
||||
BuiltinIngress bool `json:"builtinIngress,omitempty"`
|
||||
}
|
||||
|
||||
// ErrClusterNotFound is returned by GetClusterMeta / SetClusterStatus /
|
||||
@@ -231,6 +253,27 @@ func SetClusterHealth(kv nats.KeyValue, name, issue string) error {
|
||||
})
|
||||
}
|
||||
|
||||
// SetClusterHealthState records the latest health outcome together with the
|
||||
// node count from the control-plane state report. issue is the failure reason
|
||||
// (empty clears it / marks healthy). LastHealthProbe is stamped whenever health
|
||||
// or node count changes; a steady-state healthy cluster with an unchanging node
|
||||
// count produces no KV write. Returns ErrClusterNotFound if the meta record was
|
||||
// deleted underneath us.
|
||||
func SetClusterHealthState(kv nats.KeyValue, name, issue string, nodeCount int) error {
|
||||
if name == "" {
|
||||
return errors.New("eks: SetClusterHealthState empty name")
|
||||
}
|
||||
return casUpdateMeta(kv, name, func(m *ClusterMeta) bool {
|
||||
if m.HealthIssue == issue && m.NodeCount == nodeCount {
|
||||
return false
|
||||
}
|
||||
m.HealthIssue = issue
|
||||
m.NodeCount = nodeCount
|
||||
m.LastHealthProbe = time.Now().UTC()
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
// DeleteClusterPrefix removes every KV key under clusters/{name}/. Called
|
||||
// from DeleteCluster after the OIDC private key has been zeroized.
|
||||
// Returns the first delete error encountered but continues sweeping so
|
||||
|
||||
@@ -24,6 +24,30 @@ func createInput(name string) *eks.CreateClusterInput {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateCluster_BuiltinIngressDefaultsOff(t *testing.T) {
|
||||
f := newEKSServiceFixture(t)
|
||||
|
||||
_, err := f.svc.CreateCluster(createInput("alpha"), testAccountID, "")
|
||||
require.NoError(t, err)
|
||||
|
||||
meta, err := GetClusterMeta(f.kv, "alpha")
|
||||
require.NoError(t, err)
|
||||
assert.False(t, meta.BuiltinIngress, "parity default: built-in ingress off")
|
||||
}
|
||||
|
||||
func TestCreateCluster_BuiltinIngressFromTag(t *testing.T) {
|
||||
f := newEKSServiceFixture(t)
|
||||
|
||||
in := createInput("alpha")
|
||||
in.Tags = map[string]*string{managedIngressTagKey: aws.String("true")}
|
||||
_, err := f.svc.CreateCluster(in, testAccountID, "")
|
||||
require.NoError(t, err)
|
||||
|
||||
meta, err := GetClusterMeta(f.kv, "alpha")
|
||||
require.NoError(t, err)
|
||||
assert.True(t, meta.BuiltinIngress, "managed-ingress tag opts into built-in ingress")
|
||||
}
|
||||
|
||||
// A create that fails after the NLB is provisioned must leave the NLB ARNs
|
||||
// persisted on the (now FAILED) meta, otherwise the resources leak with no
|
||||
// owning record and DeleteCluster cannot reclaim them (bead 165.3).
|
||||
|
||||
@@ -18,6 +18,7 @@ type fakeSubnetResolver struct{}
|
||||
var _ SubnetVPCResolver = (*fakeSubnetResolver)(nil)
|
||||
|
||||
func (fakeSubnetResolver) GetSubnetVPC(_, _ string) (string, error) { return "vpc-aaa", nil }
|
||||
func (fakeSubnetResolver) GetVPCCIDR(_, _ string) (string, error) { return "10.0.0.0/16", nil }
|
||||
|
||||
func deleteInput(name string) *eks.DeleteClusterInput {
|
||||
return &eks.DeleteClusterInput{Name: aws.String(name)}
|
||||
@@ -27,13 +28,15 @@ func deleteInput(name string) *eks.DeleteClusterInput {
|
||||
// and an embedded JetStream KV. Tests poke the fakes' *Err fields to force
|
||||
// failures at specific lifecycle steps.
|
||||
type eksServiceFixture struct {
|
||||
svc *EKSServiceImpl
|
||||
kv nats.KeyValue
|
||||
nlb *fakeNLBProvisioner
|
||||
inst *fakeK3sInst
|
||||
vpc *fakeK3sVPC
|
||||
ami *fakeK3sAMI
|
||||
eip *fakeEIPProvisioner
|
||||
svc *EKSServiceImpl
|
||||
kv nats.KeyValue
|
||||
nlb *fakeNLBProvisioner
|
||||
inst *fakeK3sInst
|
||||
vpc *fakeK3sVPC
|
||||
ami *fakeK3sAMI
|
||||
eip *fakeEIPProvisioner
|
||||
sg *fakeSGProvisioner
|
||||
worker *fakeWorkerLauncher
|
||||
}
|
||||
|
||||
func newEKSServiceFixture(t *testing.T) *eksServiceFixture {
|
||||
@@ -47,28 +50,32 @@ func newEKSServiceFixture(t *testing.T) *eksServiceFixture {
|
||||
vpc := &fakeK3sVPC{}
|
||||
ami := &fakeK3sAMI{}
|
||||
eip := newFakeEIPProvisioner()
|
||||
sg := newFakeSGProvisioner()
|
||||
worker := newFakeWorkerLauncher()
|
||||
|
||||
svc, err := NewEKSServiceImpl(EKSServiceDeps{
|
||||
NATSConn: nc,
|
||||
MasterKey: bootstrapTestMasterKey,
|
||||
GatewayBaseURL: "https://gw.local:9999",
|
||||
Region: "us-east-1",
|
||||
HolderID: "node-1",
|
||||
NATSURL: "nats://gw.local:4222",
|
||||
NATSToken: "s3cr3t-token",
|
||||
NATSCACert: "-----BEGIN CERTIFICATE-----\nFAKECA\n-----END CERTIFICATE-----\n",
|
||||
VPCSG: newFakeSGProvisioner(),
|
||||
VPCK3s: vpc,
|
||||
VPCSubnet: fakeSubnetResolver{},
|
||||
NLB: nlb,
|
||||
Instance: inst,
|
||||
Image: ami,
|
||||
EIP: eip,
|
||||
NATSConn: nc,
|
||||
MasterKey: bootstrapTestMasterKey,
|
||||
GatewayBaseURL: "https://gw.local:9999",
|
||||
Region: "us-east-1",
|
||||
HolderID: "node-1",
|
||||
SystemGatewayURL: "https://gw.local:9999",
|
||||
SystemAccessKey: "AKIAEXAMPLE",
|
||||
SystemSecretKey: "s3cr3t-key",
|
||||
GatewayCACert: "-----BEGIN CERTIFICATE-----\nFAKECA\n-----END CERTIFICATE-----\n",
|
||||
VPCSG: sg,
|
||||
VPCK3s: vpc,
|
||||
VPCSubnet: fakeSubnetResolver{},
|
||||
NLB: nlb,
|
||||
Instance: inst,
|
||||
Image: ami,
|
||||
EIP: eip,
|
||||
Worker: worker,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(svc.Shutdown)
|
||||
|
||||
return &eksServiceFixture{svc: svc, kv: kv, nlb: nlb, inst: inst, vpc: vpc, ami: ami, eip: eip}
|
||||
return &eksServiceFixture{svc: svc, kv: kv, nlb: nlb, inst: inst, vpc: vpc, ami: ami, eip: eip, sg: sg, worker: worker}
|
||||
}
|
||||
|
||||
// deleteClusterFixture is an eksServiceFixture pre-seeded with a CREATING
|
||||
|
||||
@@ -15,6 +15,11 @@ import (
|
||||
"github.com/mulgadc/spinifex/spinifex/tags"
|
||||
)
|
||||
|
||||
// imdsServerIP is the link-local IMDS address (mirror of
|
||||
// handlers/imds.MetaDataServerIP, duplicated to avoid an import cycle:
|
||||
// imds → sts → eks).
|
||||
const imdsServerIP = "169.254.169.254"
|
||||
|
||||
// ErrEKSServerAMINotFound is returned by the launcher when no AMI carrying the
|
||||
// EKS managed-by tag exists in the account. CreateCluster maps it to a precise
|
||||
// client error: it signals an operator/config gap (the eks-server image was
|
||||
@@ -66,10 +71,23 @@ const (
|
||||
// (see scripts/images/eks-server/k3s-first-boot.sh ENVFILE).
|
||||
k3sFirstBootEnvPath = "/etc/spinifex-eks/first-boot.env"
|
||||
|
||||
// k3sNATSCAPath is the on-VM destination for the NATS CA cert PEM. The K3s
|
||||
// VM uses it to verify the daemon's NATS TLS when publishing bootstrap
|
||||
// messages. Path matches k3s-first-boot.sh SPINIFEX_NATS_CA.
|
||||
k3sNATSCAPath = "/etc/spinifex-eks/nats-ca.pem"
|
||||
// agentEnvPath is the env file the k3s-agent OpenRC service sources for its
|
||||
// K3S_URL/K3S_TOKEN/K3S_NODE_NAME/K3S_NODE_LABEL. Path matches the AGENT_ENVFILE
|
||||
// in scripts/images/eks-node/eks-node-role.sh and k3s-agent.initd.
|
||||
agentEnvPath = "/etc/spinifex-eks/agent.env"
|
||||
|
||||
// k3sGatewayCAPath is the on-VM destination for the AWS gateway TLS CA cert
|
||||
// PEM. The K3s VM uses it to verify the gateway's HTTPS cert when the
|
||||
// eks-gateway-publish helper POSTs bootstrap envelopes + state reports.
|
||||
// Path matches k3s-first-boot.sh EKS_GATEWAY_CA.
|
||||
k3sGatewayCAPath = "/etc/spinifex-eks/gateway-ca.pem"
|
||||
|
||||
// k3sTokenWebhookKubeconfigPath is the apiserver token-webhook config the
|
||||
// eks-token-webhook service (ordered `before k3s`) writes before the
|
||||
// apiserver starts. Wired via the authentication-token-webhook-config-file
|
||||
// apiserver arg so bearer tokens minted by `aws eks get-token` resolve
|
||||
// through the webhook. Must match the webhook's EKS_WEBHOOK_KUBECONFIG default.
|
||||
k3sTokenWebhookKubeconfigPath = "/etc/spinifex-eks/token-webhook.kubeconfig" //nolint:gosec // file path, not a credential
|
||||
|
||||
// k3sConfigPath is the K3s server config file cloud-init writes; K3s
|
||||
// reads it at startup (overrides the AMI-baked config.yaml.skel).
|
||||
@@ -94,19 +112,36 @@ const (
|
||||
// deferred behind cross-account-ENI work). Region is carried for future
|
||||
// region-aware AMI lookups but not consumed today.
|
||||
type K3sServerInput struct {
|
||||
AccountID string
|
||||
ClusterName string
|
||||
Region string
|
||||
SubnetID string
|
||||
ControlPlaneSGID string
|
||||
NLBDNS string
|
||||
AccountID string
|
||||
ClusterName string
|
||||
Region string
|
||||
SubnetID string
|
||||
ControlPlaneSGID string
|
||||
NLBDNS string
|
||||
// EndpointIP is the reachable cluster NLB front-end IP. When set it is added
|
||||
// to the apiserver serving-cert SANs (tls-san) so kubectl reaching the
|
||||
// cluster on this IP validates TLS. Empty for an internal endpoint with no
|
||||
// front-end IP read back.
|
||||
EndpointIP string
|
||||
OIDCIssuer string
|
||||
OIDCPrivateKeyPEM string
|
||||
OIDCPublicKeyPEM string
|
||||
NATSURL string
|
||||
NATSToken string
|
||||
NATSCACert string
|
||||
InstanceType string
|
||||
// Gateway broker config: the control-plane VM publishes its bootstrap
|
||||
// envelopes + state reports via SigV4-signed HTTPS POST to the AWS gateway
|
||||
// (the ELBv2 lb-agent model), not by dialing core NATS. GatewayURL is the
|
||||
// mgmt-reachable AWSGW endpoint; AccessKey/SecretKey are the system
|
||||
// (Predastore) SigV4 creds; GatewayCACert is the PEM that signs the gateway
|
||||
// server cert.
|
||||
GatewayURL string
|
||||
AccessKey string
|
||||
SecretKey string
|
||||
GatewayCACert string
|
||||
InstanceType string
|
||||
// BuiltinIngress keeps K3s' bundled traefik + servicelb enabled (dev /
|
||||
// interim in-VPC app exposure). Default false disables them for AWS parity,
|
||||
// where Service type=LoadBalancer / Ingress are satisfied by the AWS Load
|
||||
// Balancer Controller instead.
|
||||
BuiltinIngress bool
|
||||
}
|
||||
|
||||
// K3sServerOutput carries the identifiers the caller needs to persist into
|
||||
@@ -329,12 +364,14 @@ func validateK3sServerInput(in K3sServerInput) error {
|
||||
return errors.New("eks: K3sServerInput empty OIDCPrivateKeyPEM")
|
||||
case strings.TrimSpace(in.OIDCPublicKeyPEM) == "":
|
||||
return errors.New("eks: K3sServerInput empty OIDCPublicKeyPEM")
|
||||
case in.NATSURL == "":
|
||||
return errors.New("eks: K3sServerInput empty NATSURL")
|
||||
case in.NATSToken == "":
|
||||
return errors.New("eks: K3sServerInput empty NATSToken")
|
||||
case strings.TrimSpace(in.NATSCACert) == "":
|
||||
return errors.New("eks: K3sServerInput empty NATSCACert")
|
||||
case in.GatewayURL == "":
|
||||
return errors.New("eks: K3sServerInput empty GatewayURL")
|
||||
case in.AccessKey == "":
|
||||
return errors.New("eks: K3sServerInput empty AccessKey")
|
||||
case in.SecretKey == "":
|
||||
return errors.New("eks: K3sServerInput empty SecretKey")
|
||||
case strings.TrimSpace(in.GatewayCACert) == "":
|
||||
return errors.New("eks: K3sServerInput empty GatewayCACert")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -357,44 +394,86 @@ func buildK3sUserData(in K3sServerInput) string {
|
||||
// control-plane services (k3s server + token webhook + bootstrap
|
||||
// publisher). Nodegroup workers seed "agent" through their own path.
|
||||
"SPINIFEX_K3S_ROLE=server",
|
||||
"SPINIFEX_NATS_URL=" + in.NATSURL,
|
||||
"SPINIFEX_NATS_TOKEN=" + in.NATSToken,
|
||||
"SPINIFEX_NATS_CA=" + k3sNATSCAPath,
|
||||
"EKS_GATEWAY_URL=" + in.GatewayURL,
|
||||
"EKS_GATEWAY_CA=" + k3sGatewayCAPath,
|
||||
"EKS_ACCESS_KEY=" + in.AccessKey,
|
||||
"EKS_SECRET_KEY=" + in.SecretKey,
|
||||
"EKS_REGION=" + in.Region,
|
||||
"EKS_ACCOUNT_ID=" + in.AccountID,
|
||||
"EKS_CLUSTER_NAME=" + in.ClusterName,
|
||||
"EKS_NLB_ENDPOINT=" + nlbEndpoint,
|
||||
"EKS_OIDC_ISSUER=" + in.OIDCIssuer,
|
||||
}, "\n")
|
||||
|
||||
// Omitting cluster-init selects the embedded SQLite (kine) datastore, not
|
||||
// embedded etcd. etcd's per-commit fsync stalls past apiserver's 5s deadline
|
||||
// on the viperblock-backed root volume, so it never reaches /healthz; kine
|
||||
// tolerates the deferred-durability block path. Trade-off: no multi-server
|
||||
// HA (that needs etcd) — acceptable for single-node v1.
|
||||
// cluster-init selects the embedded etcd datastore (required for multi-server
|
||||
// HA). etcd-expose-metrics surfaces etcd's own wal_fsync_duration_seconds and
|
||||
// backend_commit_duration_seconds on 127.0.0.1:2381/metrics so control-plane
|
||||
// commit latency is measurable directly, not inferred.
|
||||
//
|
||||
// anonymous-auth=true: k3s hardens it off by default, which makes the
|
||||
// apiserver answer 401 to an unauthenticated /healthz. The cluster reconciler
|
||||
// probes https://<NLB>/healthz anonymously to gate ACTIVE, so it must be
|
||||
// reachable; the default RBAC binds only the health/version non-resource
|
||||
// paths to system:unauthenticated, so this exposes nothing else.
|
||||
k3sConfig := strings.Join([]string{
|
||||
"tls-san:",
|
||||
" - " + in.NLBDNS,
|
||||
// In real EKS neither traefik nor servicelb exists; Service type=LoadBalancer
|
||||
// and Ingress are reconciled by the AWS Load Balancer Controller. Disable the
|
||||
// K3s built-ins by default for parity. A cluster opted into built-in ingress
|
||||
// (dev / interim) keeps them so apps are reachable in-VPC before the
|
||||
// controller add-on lands.
|
||||
configLines := []string{
|
||||
"cluster-init: true",
|
||||
"etcd-expose-metrics: true",
|
||||
}
|
||||
if !in.BuiltinIngress {
|
||||
configLines = append(configLines, "disable:", " - traefik", " - servicelb")
|
||||
}
|
||||
configLines = append(configLines, "tls-san:", " - "+in.NLBDNS)
|
||||
// The reachable front-end IP must be a cert SAN so kubectl reaching the
|
||||
// cluster on https://<EndpointIP>:443 validates TLS. k3s accepts IP literals
|
||||
// in tls-san and classifies them as IP SANs.
|
||||
if in.EndpointIP != "" {
|
||||
configLines = append(configLines, " - "+in.EndpointIP)
|
||||
}
|
||||
configLines = append(configLines,
|
||||
"kube-apiserver-arg:",
|
||||
" - service-account-key-file=" + k3sOIDCPublicKeyPath,
|
||||
" - service-account-signing-key-file=" + k3sOIDCSigningKeyPath,
|
||||
" - service-account-issuer=" + in.OIDCIssuer,
|
||||
" - service-account-key-file="+k3sOIDCPublicKeyPath,
|
||||
" - service-account-signing-key-file="+k3sOIDCSigningKeyPath,
|
||||
" - service-account-issuer="+in.OIDCIssuer,
|
||||
" - api-audiences=sts.amazonaws.com",
|
||||
" - anonymous-auth=true",
|
||||
}, "\n")
|
||||
" - authentication-token-webhook-config-file="+k3sTokenWebhookKubeconfigPath,
|
||||
" - authentication-token-webhook-cache-ttl=5m",
|
||||
// Pin v1 so the apiserver decodes the webhook's authentication.k8s.io/v1
|
||||
// TokenReview response; the default v1beta1 rejects the GVK mismatch (401).
|
||||
" - authentication-token-webhook-version=v1",
|
||||
)
|
||||
k3sConfig := strings.Join(configLines, "\n")
|
||||
|
||||
files := []userDataFile{
|
||||
// first-boot.env carries the NATS token, so keep it root-only (0600).
|
||||
// first-boot.env carries the system SigV4 secret key, so keep it
|
||||
// root-only (0600).
|
||||
{Path: k3sFirstBootEnvPath, Perms: "0600", Body: envBody},
|
||||
{Path: k3sOIDCSigningKeyPath, Perms: "0600", Body: strings.TrimRight(in.OIDCPrivateKeyPEM, "\n")},
|
||||
{Path: k3sOIDCPublicKeyPath, Perms: "0644", Body: strings.TrimRight(in.OIDCPublicKeyPEM, "\n")},
|
||||
{Path: k3sConfigPath, Perms: "0644", Body: k3sConfig},
|
||||
{Path: k3sNATSCAPath, Perms: "0644", Body: strings.TrimRight(in.NATSCACert, "\n")},
|
||||
{Path: k3sGatewayCAPath, Perms: "0644", Body: strings.TrimRight(in.GatewayCACert, "\n")},
|
||||
// IMDS on-link route. Alpine's cloud-init eni renderer crashes on a
|
||||
// gateway-less route in network-config, so it's delivered out-of-band:
|
||||
// a persistent local.d script (re-applied every boot by the OpenRC
|
||||
// `local` service, enabled via runcmd below) ARPs 169.254.169.254
|
||||
// directly on the VPC subnet. This block owns the only write_files key,
|
||||
// so it must carry the route itself — appending a second write_files via
|
||||
// the generic cloud-init template would collide (last key wins, silently
|
||||
// dropping the k3s config). The device is resolved via the default route
|
||||
// (the VPC egress NIC), not a name: the persistent-net rename to vpc0
|
||||
// doesn't apply on the Alpine AMI, so the kernel name is eth0 at boot.
|
||||
{
|
||||
Path: "/etc/local.d/imds-onlink-route.start",
|
||||
Perms: "0755",
|
||||
Body: "#!/bin/sh\n" +
|
||||
"dev=$(ip route show default | awk '{print $5; exit}')\n" +
|
||||
"[ -n \"$dev\" ] && ip route replace " + imdsServerIP + "/32 dev \"$dev\" scope link",
|
||||
},
|
||||
}
|
||||
|
||||
var buf strings.Builder
|
||||
@@ -424,5 +503,12 @@ func buildK3sUserData(in K3sServerInput) string {
|
||||
buf.WriteString("\n")
|
||||
}
|
||||
}
|
||||
|
||||
// Enable + start the OpenRC `local` service so the IMDS route script runs on
|
||||
// first boot and is re-applied on every subsequent boot.
|
||||
buf.WriteString("runcmd:\n")
|
||||
buf.WriteString(" - [ rc-update, add, local, default ]\n")
|
||||
buf.WriteString(" - [ rc-service, local, start ]\n")
|
||||
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
@@ -118,9 +118,10 @@ func validK3sInput() K3sServerInput {
|
||||
OIDCIssuer: "https://oidc.spinifex.local/clusters/111122223333/alpha",
|
||||
OIDCPrivateKeyPEM: "-----BEGIN PRIVATE KEY-----\nFAKEKEY\n-----END PRIVATE KEY-----\n",
|
||||
OIDCPublicKeyPEM: "-----BEGIN PUBLIC KEY-----\nFAKEPUB\n-----END PUBLIC KEY-----\n",
|
||||
NATSURL: "nats://localhost:4222",
|
||||
NATSToken: "s3cr3t-token",
|
||||
NATSCACert: "-----BEGIN CERTIFICATE-----\nFAKECA\n-----END CERTIFICATE-----\n",
|
||||
GatewayURL: "https://10.15.8.1:9999",
|
||||
AccessKey: "AKIAEXAMPLE",
|
||||
SecretKey: "s3cr3t-key",
|
||||
GatewayCACert: "-----BEGIN CERTIFICATE-----\nFAKECA\n-----END CERTIFICATE-----\n",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -142,9 +143,10 @@ func TestLaunchK3sServerVM_EmptyInputsRejected(t *testing.T) {
|
||||
{"empty OIDCIssuer", mk(func(in *K3sServerInput) { in.OIDCIssuer = "" })},
|
||||
{"empty OIDCPrivateKeyPEM", mk(func(in *K3sServerInput) { in.OIDCPrivateKeyPEM = " \n " })},
|
||||
{"empty OIDCPublicKeyPEM", mk(func(in *K3sServerInput) { in.OIDCPublicKeyPEM = " \n " })},
|
||||
{"empty NATSURL", mk(func(in *K3sServerInput) { in.NATSURL = "" })},
|
||||
{"empty NATSToken", mk(func(in *K3sServerInput) { in.NATSToken = "" })},
|
||||
{"empty NATSCACert", mk(func(in *K3sServerInput) { in.NATSCACert = " \n" })},
|
||||
{"empty GatewayURL", mk(func(in *K3sServerInput) { in.GatewayURL = "" })},
|
||||
{"empty AccessKey", mk(func(in *K3sServerInput) { in.AccessKey = "" })},
|
||||
{"empty SecretKey", mk(func(in *K3sServerInput) { in.SecretKey = "" })},
|
||||
{"empty GatewayCACert", mk(func(in *K3sServerInput) { in.GatewayCACert = " \n" })},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
@@ -269,9 +271,11 @@ func TestLaunchK3sServerVM_UserDataContainsAllArtifacts(t *testing.T) {
|
||||
// The eks-node-role first-boot selector keys off this to start control-plane
|
||||
// services; without it the unified AMI boots into no role.
|
||||
assert.Contains(t, udata, "SPINIFEX_K3S_ROLE=server")
|
||||
assert.Contains(t, udata, "SPINIFEX_NATS_URL=nats://localhost:4222")
|
||||
assert.Contains(t, udata, "SPINIFEX_NATS_TOKEN=s3cr3t-token")
|
||||
assert.Contains(t, udata, "SPINIFEX_NATS_CA="+k3sNATSCAPath)
|
||||
assert.Contains(t, udata, "EKS_GATEWAY_URL=https://10.15.8.1:9999")
|
||||
assert.Contains(t, udata, "EKS_GATEWAY_CA="+k3sGatewayCAPath)
|
||||
assert.Contains(t, udata, "EKS_ACCESS_KEY=AKIAEXAMPLE")
|
||||
assert.Contains(t, udata, "EKS_SECRET_KEY=s3cr3t-key")
|
||||
assert.Contains(t, udata, "EKS_REGION=us-east-1")
|
||||
assert.Contains(t, udata, "EKS_ACCOUNT_ID=111122223333")
|
||||
assert.Contains(t, udata, "EKS_CLUSTER_NAME=alpha")
|
||||
assert.Contains(t, udata, "EKS_NLB_ENDPOINT=https://eks-alpha-lb-001.us-east-1.elb.spinifex.local:443")
|
||||
@@ -287,6 +291,11 @@ func TestLaunchK3sServerVM_UserDataContainsAllArtifacts(t *testing.T) {
|
||||
assert.Contains(t, udata, "FAKEPUB")
|
||||
|
||||
assert.Contains(t, udata, "path: "+k3sConfigPath)
|
||||
// Parity default (BuiltinIngress=false): K3s' bundled traefik + servicelb are
|
||||
// disabled; Service type=LoadBalancer / Ingress are the AWS LB Controller's job.
|
||||
assert.Contains(t, udata, "disable:")
|
||||
assert.Contains(t, udata, " - traefik")
|
||||
assert.Contains(t, udata, " - servicelb")
|
||||
assert.Contains(t, udata, "tls-san:")
|
||||
assert.Contains(t, udata, " - eks-alpha-lb-001.us-east-1.elb.spinifex.local")
|
||||
// service-account-key-file must point at the PUBLIC key; the signing key
|
||||
@@ -297,10 +306,58 @@ func TestLaunchK3sServerVM_UserDataContainsAllArtifacts(t *testing.T) {
|
||||
assert.NotContains(t, udata, "service-account-key-file="+k3sOIDCSigningKeyPath)
|
||||
assert.Contains(t, udata, "service-account-issuer=https://oidc.spinifex.local/clusters/111122223333/alpha")
|
||||
assert.Contains(t, udata, "api-audiences=sts.amazonaws.com")
|
||||
// IAM bearer-token auth: the generated config overrides the AMI skel, so the
|
||||
// token-webhook arg MUST be emitted here or `aws eks get-token` 401s with the
|
||||
// webhook never invoked.
|
||||
assert.Contains(t, udata, "authentication-token-webhook-config-file="+k3sTokenWebhookKubeconfigPath)
|
||||
|
||||
assert.Contains(t, udata, "path: "+k3sNATSCAPath)
|
||||
assert.Contains(t, udata, "path: "+k3sGatewayCAPath)
|
||||
assert.Contains(t, udata, "-----BEGIN CERTIFICATE-----")
|
||||
assert.Contains(t, udata, "FAKECA")
|
||||
|
||||
// IMDS on-link route delivered out-of-band (Alpine's eni renderer can't emit
|
||||
// the gateway-less route in network-config). It rides this payload's own
|
||||
// write_files/runcmd, enabled via the OpenRC local service.
|
||||
assert.Contains(t, udata, "path: /etc/local.d/imds-onlink-route.start")
|
||||
assert.Contains(t, udata, "ip route show default")
|
||||
assert.Contains(t, udata, `ip route replace 169.254.169.254/32 dev "$dev" scope link`)
|
||||
assert.Contains(t, udata, "runcmd:")
|
||||
assert.Contains(t, udata, "[ rc-update, add, local, default ]")
|
||||
|
||||
// Exactly one write_files / runcmd key — a second would collide and silently
|
||||
// drop a block under yaml.safe_load (last key wins).
|
||||
assert.Equal(t, 1, strings.Count(udata, "\nwrite_files:"))
|
||||
assert.Equal(t, 1, strings.Count(udata, "\nruncmd:"))
|
||||
}
|
||||
|
||||
func TestLaunchK3sServerVM_BuiltinIngressOptInKeepsK3sDefaults(t *testing.T) {
|
||||
vpc, inst, ami := &fakeK3sVPC{}, &fakeK3sInst{}, &fakeK3sAMI{}
|
||||
|
||||
in := validK3sInput()
|
||||
in.BuiltinIngress = true
|
||||
_, err := LaunchK3sServerVM(vpc, inst, ami, in)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, inst.launchCalls, 1)
|
||||
|
||||
udata := inst.launchCalls[0].UserData
|
||||
// Opted in: leave K3s' traefik + servicelb running, so no disable block.
|
||||
assert.NotContains(t, udata, "disable:")
|
||||
assert.NotContains(t, udata, "- traefik")
|
||||
assert.NotContains(t, udata, "- servicelb")
|
||||
}
|
||||
|
||||
func TestLaunchK3sServerVM_UsesEmbeddedEtcd(t *testing.T) {
|
||||
vpc, inst, ami := &fakeK3sVPC{}, &fakeK3sInst{}, &fakeK3sAMI{}
|
||||
|
||||
_, err := LaunchK3sServerVM(vpc, inst, ami, validK3sInput())
|
||||
require.NoError(t, err)
|
||||
require.Len(t, inst.launchCalls, 1)
|
||||
|
||||
udata := inst.launchCalls[0].UserData
|
||||
// cluster-init selects the embedded etcd datastore (not SQLite/kine);
|
||||
// etcd-expose-metrics surfaces wal_fsync/backend_commit on 127.0.0.1:2381.
|
||||
assert.Contains(t, udata, "cluster-init: true")
|
||||
assert.Contains(t, udata, "etcd-expose-metrics: true")
|
||||
}
|
||||
|
||||
func TestLaunchK3sServerVM_RunInstancesEmptyReservationRollsBack(t *testing.T) {
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
package handlers_eks
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/eks"
|
||||
"github.com/mulgadc/spinifex/spinifex/awserrors"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -53,3 +57,26 @@ func TestEKSServiceImpl_ListClustersMaxResultsClamped(t *testing.T) {
|
||||
assert.Nil(t, out.NextToken, "MaxResults=%d single page must not advertise a NextToken", mr)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEKSReadUnavailableClassification pins the post-restart behaviour: a
|
||||
// transient JetStream error (no leader yet) surfaces as a retryable
|
||||
// ServiceUnavailable, while a genuine fault stays wrapped (the daemon maps it
|
||||
// to InternalError). Regression guard for list-clusters returning a misleading
|
||||
// 500 during the KV warmup window.
|
||||
func TestEKSReadUnavailableClassification(t *testing.T) {
|
||||
transient := []error{
|
||||
nats.ErrNoResponders,
|
||||
nats.ErrTimeout,
|
||||
nats.ErrNoStreamResponse,
|
||||
nats.ErrConnectionClosed,
|
||||
fmt.Errorf("kv keys: %w", nats.ErrNoResponders), // wrapped, as ListClusters wraps it
|
||||
}
|
||||
for _, e := range transient {
|
||||
assert.True(t, natsTransient(e), "%v must be classified transient", e)
|
||||
assert.EqualError(t, eksReadUnavailableOr(e, "op"), awserrors.ErrorServiceUnavailable)
|
||||
}
|
||||
|
||||
genuine := errors.New("corrupt cluster meta")
|
||||
assert.False(t, natsTransient(genuine))
|
||||
assert.ErrorContains(t, eksReadUnavailableOr(genuine, "list cluster names"), "list cluster names: corrupt cluster meta")
|
||||
}
|
||||
|
||||
@@ -27,6 +27,8 @@ type nlbProvisioner interface {
|
||||
|
||||
RegisterTargets(input *elbv2.RegisterTargetsInput, accountID string) (*elbv2.RegisterTargetsOutput, error)
|
||||
DeregisterTargets(input *elbv2.DeregisterTargetsInput, accountID string) (*elbv2.DeregisterTargetsOutput, error)
|
||||
|
||||
SetLoadBalancerIngressCIDRs(lbArn string, cidrs []string, accountID string) error
|
||||
}
|
||||
|
||||
// AWS load-balancer + target-group names are bounded to 32 alphanumeric or
|
||||
@@ -51,6 +53,11 @@ type ClusterNLB struct {
|
||||
TargetGroupArn string
|
||||
ListenerArn string
|
||||
DNSName string
|
||||
// FrontendIP is the NLB's reachable front-end address: the external-pool
|
||||
// public IP for an internet-facing cluster NLB, or the VPC private IP for an
|
||||
// internal one. Used as the kubeconfig endpoint host + apiserver cert SAN.
|
||||
// Empty if the backing LB record has no address yet.
|
||||
FrontendIP string
|
||||
}
|
||||
|
||||
// ClusterNLBName returns the deterministic NLB name for a cluster. Stable so
|
||||
@@ -73,7 +80,18 @@ func ClusterTargetGroupName(clusterName string) string {
|
||||
//
|
||||
// The K3s server VM ENI IP is NOT registered here — Stage 4
|
||||
// (k3s_server_vm.go) calls RegisterClusterTarget once the VM has its ENI IP.
|
||||
func EnsureClusterNLB(nlbp nlbProvisioner, accountID, clusterName string, subnetIDs []string) (*ClusterNLB, error) {
|
||||
//
|
||||
// internetFacing selects the NLB scheme: true ⇒ internet-facing (external-pool
|
||||
// front-end IP, LAN-reachable, for a public cluster endpoint); false ⇒ internal
|
||||
// (VPC-only). After ensuring the LB, the reachable FrontendIP is read back from
|
||||
// the backing LB record so the caller can use it as the endpoint host + cert SAN.
|
||||
//
|
||||
// publicAccessCidrs narrows who may reach an internet-facing front-end below the
|
||||
// scheme default (0.0.0.0/0). It maps the cluster's publicAccessCidrs onto the
|
||||
// NLB managed-SG ingress; ignored for an internal NLB (its ingress already
|
||||
// tracks the VPC CIDR) and for the wide-open default, which the LB carries out
|
||||
// of the box.
|
||||
func EnsureClusterNLB(nlbp nlbProvisioner, accountID, clusterName string, subnetIDs []string, internetFacing bool, publicAccessCidrs []string) (*ClusterNLB, error) {
|
||||
if clusterName == "" {
|
||||
return nil, errors.New("eks: EnsureClusterNLB empty cluster name")
|
||||
}
|
||||
@@ -91,7 +109,7 @@ func EnsureClusterNLB(nlbp nlbProvisioner, accountID, clusterName string, subnet
|
||||
|
||||
out := &ClusterNLB{}
|
||||
|
||||
if err := ensureClusterLB(nlbp, accountID, clusterName, lbName, subnetIDs, out); err != nil {
|
||||
if err := ensureClusterLB(nlbp, accountID, clusterName, lbName, subnetIDs, internetFacing, out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ensureClusterTG(nlbp, accountID, clusterName, tgName, out); err != nil {
|
||||
@@ -100,9 +118,62 @@ func EnsureClusterNLB(nlbp nlbProvisioner, accountID, clusterName string, subnet
|
||||
if err := ensureClusterListener(nlbp, accountID, lbName, out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Narrow the internet-facing front-end to the cluster's publicAccessCidrs
|
||||
// when they restrict below the wide-open scheme default the LB already
|
||||
// carries. The override is idempotent, so re-running on retry converges.
|
||||
if internetFacing && narrowsPublicAccess(publicAccessCidrs) {
|
||||
if err := nlbp.SetLoadBalancerIngressCIDRs(out.LoadBalancerArn, publicAccessCidrs, accountID); err != nil {
|
||||
return nil, fmt.Errorf("eks: set NLB ingress CIDRs for %s: %w", lbName, err)
|
||||
}
|
||||
}
|
||||
// Read the front-end IP back from the LB record (DescribeLoadBalancers →
|
||||
// populated LoadBalancerAddresses). Best-effort: an empty IP is not fatal
|
||||
// here — the caller falls back to the DNS-name endpoint.
|
||||
if lb, err := lookupLBByName(nlbp, accountID, lbName); err == nil && lb != nil {
|
||||
out.FrontendIP = frontendIPFromLB(lb, internetFacing)
|
||||
} else if err != nil {
|
||||
slog.Warn("EnsureClusterNLB: front-end IP read-back failed", "name", lbName, "err", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// narrowsPublicAccess reports whether publicAccessCidrs restrict the front-end
|
||||
// below the wide-open default an internet-facing NLB already carries. Empty (no
|
||||
// override) or the lone 0.0.0.0/0 default are no-ops and skip the extra ELBv2
|
||||
// round-trip.
|
||||
func narrowsPublicAccess(cidrs []string) bool {
|
||||
if len(cidrs) == 0 {
|
||||
return false
|
||||
}
|
||||
if len(cidrs) == 1 && cidrs[0] == defaultPublicAccessCidr {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// frontendIPFromLB extracts the reachable front-end address from a described
|
||||
// load balancer: the public IpAddress for an internet-facing LB, else the VPC
|
||||
// PrivateIPv4Address. Returns the first non-empty match across AZs.
|
||||
func frontendIPFromLB(lb *elbv2.LoadBalancer, internetFacing bool) string {
|
||||
for _, az := range lb.AvailabilityZones {
|
||||
for _, addr := range az.LoadBalancerAddresses {
|
||||
if addr == nil {
|
||||
continue
|
||||
}
|
||||
if internetFacing {
|
||||
if ip := aws.StringValue(addr.IpAddress); ip != "" {
|
||||
return ip
|
||||
}
|
||||
continue
|
||||
}
|
||||
if ip := aws.StringValue(addr.PrivateIPv4Address); ip != "" {
|
||||
return ip
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// RegisterClusterTarget attaches the K3s server ENI IP to the cluster TG so
|
||||
// the NLB starts forwarding :443 traffic to :6443 on the VM. The ELBv2
|
||||
// RegisterTargets impl already dedupes on (id, port), so re-invocation by
|
||||
@@ -197,7 +268,7 @@ func deleteClusterTG(nlbp nlbProvisioner, accountID, tgName string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func ensureClusterLB(nlbp nlbProvisioner, accountID, clusterName, lbName string, subnetIDs []string, out *ClusterNLB) error {
|
||||
func ensureClusterLB(nlbp nlbProvisioner, accountID, clusterName, lbName string, subnetIDs []string, internetFacing bool, out *ClusterNLB) error {
|
||||
if lb, err := lookupLBByName(nlbp, accountID, lbName); err != nil {
|
||||
return err
|
||||
} else if lb != nil {
|
||||
@@ -209,10 +280,14 @@ func ensureClusterLB(nlbp nlbProvisioner, accountID, clusterName, lbName string,
|
||||
return nil
|
||||
}
|
||||
|
||||
scheme := elbv2.LoadBalancerSchemeEnumInternal
|
||||
if internetFacing {
|
||||
scheme = elbv2.LoadBalancerSchemeEnumInternetFacing
|
||||
}
|
||||
created, err := nlbp.CreateLoadBalancer(&elbv2.CreateLoadBalancerInput{
|
||||
Name: aws.String(lbName),
|
||||
Type: aws.String(elbv2.LoadBalancerTypeEnumNetwork),
|
||||
Scheme: aws.String(elbv2.LoadBalancerSchemeEnumInternal),
|
||||
Scheme: aws.String(scheme),
|
||||
Subnets: aws.StringSlice(subnetIDs),
|
||||
Tags: []*elbv2.Tag{
|
||||
{Key: aws.String(tags.ManagedByKey), Value: aws.String(tags.ManagedByEKS)},
|
||||
|
||||
@@ -23,6 +23,7 @@ type fakeNLBProvisioner struct {
|
||||
describeListeners []*elbv2.DescribeListenersInput
|
||||
registerCalls []*elbv2.RegisterTargetsInput
|
||||
deregisterCalls []*elbv2.DeregisterTargetsInput
|
||||
setIngressCalls []setIngressCIDRsCall
|
||||
|
||||
lbByName map[string]*elbv2.LoadBalancer
|
||||
tgByName map[string]*elbv2.TargetGroup
|
||||
@@ -38,6 +39,12 @@ type fakeNLBProvisioner struct {
|
||||
deleteTGErr error
|
||||
registerErr error
|
||||
deregisterErr error
|
||||
setIngressErr error
|
||||
}
|
||||
|
||||
type setIngressCIDRsCall struct {
|
||||
lbArn string
|
||||
cidrs []string
|
||||
}
|
||||
|
||||
var _ nlbProvisioner = (*fakeNLBProvisioner)(nil)
|
||||
@@ -214,13 +221,18 @@ func (f *fakeNLBProvisioner) DeregisterTargets(input *elbv2.DeregisterTargetsInp
|
||||
return &elbv2.DeregisterTargetsOutput{}, nil
|
||||
}
|
||||
|
||||
func (f *fakeNLBProvisioner) SetLoadBalancerIngressCIDRs(lbArn string, cidrs []string, _ string) error {
|
||||
f.setIngressCalls = append(f.setIngressCalls, setIngressCIDRsCall{lbArn: lbArn, cidrs: cidrs})
|
||||
return f.setIngressErr
|
||||
}
|
||||
|
||||
func TestEnsureClusterNLB_EmptyInputsRejected(t *testing.T) {
|
||||
nlbp := newFakeNLBProvisioner()
|
||||
|
||||
_, err := EnsureClusterNLB(nlbp, "111122223333", "", []string{"subnet-aaa"})
|
||||
_, err := EnsureClusterNLB(nlbp, "111122223333", "", []string{"subnet-aaa"}, false, nil)
|
||||
require.Error(t, err)
|
||||
|
||||
_, err = EnsureClusterNLB(nlbp, "111122223333", "alpha", nil)
|
||||
_, err = EnsureClusterNLB(nlbp, "111122223333", "alpha", nil, false, nil)
|
||||
require.Error(t, err)
|
||||
|
||||
assert.Empty(t, nlbp.createLBCalls)
|
||||
@@ -230,7 +242,7 @@ func TestEnsureClusterNLB_NameTooLongRejected(t *testing.T) {
|
||||
nlbp := newFakeNLBProvisioner()
|
||||
|
||||
longName := strings.Repeat("x", maxELBv2NameLen) // "eks-" + 32x = 36 chars
|
||||
_, err := EnsureClusterNLB(nlbp, "111122223333", longName, []string{"subnet-aaa"})
|
||||
_, err := EnsureClusterNLB(nlbp, "111122223333", longName, []string{"subnet-aaa"}, false, nil)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "exceeds")
|
||||
assert.Empty(t, nlbp.createLBCalls)
|
||||
@@ -239,7 +251,7 @@ func TestEnsureClusterNLB_NameTooLongRejected(t *testing.T) {
|
||||
func TestEnsureClusterNLB_FreshCreatesAllThree(t *testing.T) {
|
||||
nlbp := newFakeNLBProvisioner()
|
||||
|
||||
out, err := EnsureClusterNLB(nlbp, "111122223333", "alpha", []string{"subnet-aaa", "subnet-bbb"})
|
||||
out, err := EnsureClusterNLB(nlbp, "111122223333", "alpha", []string{"subnet-aaa", "subnet-bbb"}, false, nil)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, out)
|
||||
assert.NotEmpty(t, out.LoadBalancerArn)
|
||||
@@ -276,14 +288,14 @@ func TestEnsureClusterNLB_FreshCreatesAllThree(t *testing.T) {
|
||||
func TestEnsureClusterNLB_IdempotentReusesExisting(t *testing.T) {
|
||||
nlbp := newFakeNLBProvisioner()
|
||||
|
||||
first, err := EnsureClusterNLB(nlbp, "111122223333", "alpha", []string{"subnet-aaa"})
|
||||
first, err := EnsureClusterNLB(nlbp, "111122223333", "alpha", []string{"subnet-aaa"}, false, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
createLBCount := len(nlbp.createLBCalls)
|
||||
createTGCount := len(nlbp.createTGCalls)
|
||||
createListenerCount := len(nlbp.createListenerCalls)
|
||||
|
||||
second, err := EnsureClusterNLB(nlbp, "111122223333", "alpha", []string{"subnet-aaa"})
|
||||
second, err := EnsureClusterNLB(nlbp, "111122223333", "alpha", []string{"subnet-aaa"}, false, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, first.LoadBalancerArn, second.LoadBalancerArn)
|
||||
@@ -300,12 +312,113 @@ func TestEnsureClusterNLB_LBCreateErrorSurfaced(t *testing.T) {
|
||||
nlbp := newFakeNLBProvisioner()
|
||||
nlbp.createLBErr = errors.New("InsufficientCapacity")
|
||||
|
||||
_, err := EnsureClusterNLB(nlbp, "111122223333", "alpha", []string{"subnet-aaa"})
|
||||
_, err := EnsureClusterNLB(nlbp, "111122223333", "alpha", []string{"subnet-aaa"}, false, nil)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "create NLB eks-alpha")
|
||||
assert.Empty(t, nlbp.createTGCalls, "TG create should not run when LB create fails")
|
||||
}
|
||||
|
||||
func TestEnsureClusterNLB_InternetFacingSchemeAndPublicFrontendIP(t *testing.T) {
|
||||
nlbp := newFakeNLBProvisioner()
|
||||
// Internet-facing LB record exposes a public IpAddress in its AZ addresses.
|
||||
arn := "arn:aws:elasticloadbalancing:us-east-1:111122223333:loadbalancer/net/eks-alpha/lb-001"
|
||||
nlbp.createLBOut = &elbv2.CreateLoadBalancerOutput{
|
||||
LoadBalancers: []*elbv2.LoadBalancer{{
|
||||
LoadBalancerArn: aws.String(arn),
|
||||
LoadBalancerName: aws.String("eks-alpha"),
|
||||
DNSName: aws.String("eks-alpha-lb-001.us-east-1.elb.spinifex.local"),
|
||||
Type: aws.String(elbv2.LoadBalancerTypeEnumNetwork),
|
||||
Scheme: aws.String(elbv2.LoadBalancerSchemeEnumInternetFacing),
|
||||
AvailabilityZones: []*elbv2.AvailabilityZone{{
|
||||
LoadBalancerAddresses: []*elbv2.LoadBalancerAddress{
|
||||
{PrivateIPv4Address: aws.String("10.0.1.5")},
|
||||
{IpAddress: aws.String("203.0.113.10")},
|
||||
},
|
||||
}},
|
||||
}},
|
||||
}
|
||||
|
||||
out, err := EnsureClusterNLB(nlbp, "111122223333", "alpha", []string{"subnet-aaa"}, true, nil)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, nlbp.createLBCalls, 1)
|
||||
assert.Equal(t, elbv2.LoadBalancerSchemeEnumInternetFacing, aws.StringValue(nlbp.createLBCalls[0].Scheme))
|
||||
assert.Equal(t, "203.0.113.10", out.FrontendIP, "internet-facing front-end uses the public IpAddress")
|
||||
}
|
||||
|
||||
func TestEnsureClusterNLB_InternalSchemeUsesPrivateFrontendIP(t *testing.T) {
|
||||
nlbp := newFakeNLBProvisioner()
|
||||
arn := "arn:aws:elasticloadbalancing:us-east-1:111122223333:loadbalancer/net/eks-alpha/lb-001"
|
||||
nlbp.createLBOut = &elbv2.CreateLoadBalancerOutput{
|
||||
LoadBalancers: []*elbv2.LoadBalancer{{
|
||||
LoadBalancerArn: aws.String(arn),
|
||||
LoadBalancerName: aws.String("eks-alpha"),
|
||||
DNSName: aws.String("eks-alpha-lb-001.us-east-1.elb.spinifex.local"),
|
||||
Type: aws.String(elbv2.LoadBalancerTypeEnumNetwork),
|
||||
Scheme: aws.String(elbv2.LoadBalancerSchemeEnumInternal),
|
||||
AvailabilityZones: []*elbv2.AvailabilityZone{{
|
||||
LoadBalancerAddresses: []*elbv2.LoadBalancerAddress{
|
||||
{PrivateIPv4Address: aws.String("10.0.1.5")},
|
||||
},
|
||||
}},
|
||||
}},
|
||||
}
|
||||
|
||||
out, err := EnsureClusterNLB(nlbp, "111122223333", "alpha", []string{"subnet-aaa"}, false, nil)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, nlbp.createLBCalls, 1)
|
||||
assert.Equal(t, elbv2.LoadBalancerSchemeEnumInternal, aws.StringValue(nlbp.createLBCalls[0].Scheme))
|
||||
assert.Equal(t, "10.0.1.5", out.FrontendIP, "internal front-end uses the private IPv4 address")
|
||||
}
|
||||
|
||||
func TestEnsureClusterNLB_NarrowedPublicAccessSetsIngressCIDRs(t *testing.T) {
|
||||
nlbp := newFakeNLBProvisioner()
|
||||
|
||||
cidrs := []string{"203.0.113.0/24", "198.51.100.7/32"}
|
||||
out, err := EnsureClusterNLB(nlbp, "111122223333", "alpha", []string{"subnet-aaa"}, true, cidrs)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Len(t, nlbp.setIngressCalls, 1, "narrowed public access should drive SetLoadBalancerIngressCIDRs")
|
||||
assert.Equal(t, out.LoadBalancerArn, nlbp.setIngressCalls[0].lbArn)
|
||||
assert.Equal(t, cidrs, nlbp.setIngressCalls[0].cidrs)
|
||||
}
|
||||
|
||||
func TestEnsureClusterNLB_DefaultPublicAccessSkipsIngressCIDRs(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
cidrs []string
|
||||
}{
|
||||
{"nil", nil},
|
||||
{"empty", []string{}},
|
||||
{"wide-open default", []string{defaultPublicAccessCidr}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
nlbp := newFakeNLBProvisioner()
|
||||
_, err := EnsureClusterNLB(nlbp, "111122223333", "alpha", []string{"subnet-aaa"}, true, tc.cidrs)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, nlbp.setIngressCalls, "wide-open front-end needs no ingress override")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureClusterNLB_InternalSkipsIngressCIDRs(t *testing.T) {
|
||||
nlbp := newFakeNLBProvisioner()
|
||||
|
||||
// Even with narrowed CIDRs, an internal NLB ignores them — its ingress
|
||||
// already tracks the VPC CIDR, not a public front-end.
|
||||
_, err := EnsureClusterNLB(nlbp, "111122223333", "alpha", []string{"subnet-aaa"}, false, []string{"203.0.113.0/24"})
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, nlbp.setIngressCalls)
|
||||
}
|
||||
|
||||
func TestEnsureClusterNLB_SetIngressErrorSurfaced(t *testing.T) {
|
||||
nlbp := newFakeNLBProvisioner()
|
||||
nlbp.setIngressErr = errors.New("InvalidLoadBalancer")
|
||||
|
||||
_, err := EnsureClusterNLB(nlbp, "111122223333", "alpha", []string{"subnet-aaa"}, true, []string{"203.0.113.0/24"})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "set NLB ingress CIDRs for eks-alpha")
|
||||
}
|
||||
|
||||
func TestRegisterClusterTarget_PostsENIIPAndAPIPort(t *testing.T) {
|
||||
nlbp := newFakeNLBProvisioner()
|
||||
|
||||
@@ -343,7 +456,7 @@ func TestDeregisterClusterTarget_PostsENIIPAndAPIPort(t *testing.T) {
|
||||
|
||||
func TestDeleteClusterNLB_DeletesBoth(t *testing.T) {
|
||||
nlbp := newFakeNLBProvisioner()
|
||||
out, err := EnsureClusterNLB(nlbp, "111122223333", "alpha", []string{"subnet-aaa"})
|
||||
out, err := EnsureClusterNLB(nlbp, "111122223333", "alpha", []string{"subnet-aaa"}, false, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, DeleteClusterNLB(nlbp, "111122223333", "alpha"))
|
||||
@@ -363,7 +476,7 @@ func TestDeleteClusterNLB_MissingResourcesNoOp(t *testing.T) {
|
||||
|
||||
func TestDeleteClusterNLB_FirstErrorSurfacedSweepContinues(t *testing.T) {
|
||||
nlbp := newFakeNLBProvisioner()
|
||||
_, err := EnsureClusterNLB(nlbp, "111122223333", "alpha", []string{"subnet-aaa"})
|
||||
_, err := EnsureClusterNLB(nlbp, "111122223333", "alpha", []string{"subnet-aaa"}, false, nil)
|
||||
require.NoError(t, err)
|
||||
nlbp.deleteLBErr = errors.New("LoadBalancerInUse")
|
||||
|
||||
|
||||
@@ -0,0 +1,605 @@
|
||||
package handlers_eks
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"maps"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/aws/aws-sdk-go/service/eks"
|
||||
"github.com/google/uuid"
|
||||
"github.com/mulgadc/spinifex/spinifex/awserrors"
|
||||
handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// defaultNodegroupInstanceType mirrors the AWS EKS managed-nodegroup default
|
||||
// when the caller omits instanceTypes.
|
||||
const defaultNodegroupInstanceType = "t3.medium"
|
||||
|
||||
// ErrNodegroupNotFound is returned by GetNodegroupRecord when the record key is
|
||||
// absent. Callers translate to the AWS shape (ResourceNotFoundException) at the
|
||||
// service boundary.
|
||||
var ErrNodegroupNotFound = errors.New("eks: nodegroup not found")
|
||||
|
||||
// NodegroupARN composes a nodegroup ARN matching the AWS shape
|
||||
// (.../nodegroup/{cluster}/{ng}/{uuid}). The trailing UUID is the per-nodegroup
|
||||
// discriminator AWS appends; it is generated once at create time and persisted.
|
||||
func NodegroupARN(region, accountID, cluster, ng, id string) string {
|
||||
return fmt.Sprintf("arn:aws:eks:%s:%s:nodegroup/%s/%s/%s", region, accountID, cluster, ng, id)
|
||||
}
|
||||
|
||||
// PutNodegroupRecord writes the record unconditionally.
|
||||
func PutNodegroupRecord(kv nats.KeyValue, rec *NodegroupRecord) error {
|
||||
if rec == nil {
|
||||
return errors.New("eks: PutNodegroupRecord nil record")
|
||||
}
|
||||
if rec.ClusterName == "" || rec.Name == "" {
|
||||
return errors.New("eks: PutNodegroupRecord missing cluster or name")
|
||||
}
|
||||
data, err := json.Marshal(rec)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal nodegroup %s: %w", rec.Name, err)
|
||||
}
|
||||
key := NodegroupKey(rec.ClusterName, rec.Name)
|
||||
if _, err := kv.Put(key, data); err != nil {
|
||||
return fmt.Errorf("kv put %s: %w", key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetNodegroupRecord reads one record. Returns ErrNodegroupNotFound if absent.
|
||||
func GetNodegroupRecord(kv nats.KeyValue, cluster, ng string) (*NodegroupRecord, error) {
|
||||
if cluster == "" || ng == "" {
|
||||
return nil, errors.New("eks: GetNodegroupRecord empty cluster or name")
|
||||
}
|
||||
entry, err := kv.Get(NodegroupKey(cluster, ng))
|
||||
if err != nil {
|
||||
if errors.Is(err, nats.ErrKeyNotFound) {
|
||||
return nil, ErrNodegroupNotFound
|
||||
}
|
||||
return nil, fmt.Errorf("kv get nodegroup: %w", err)
|
||||
}
|
||||
var rec NodegroupRecord
|
||||
if err := json.Unmarshal(entry.Value(), &rec); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal nodegroup %s: %w", ng, err)
|
||||
}
|
||||
return &rec, nil
|
||||
}
|
||||
|
||||
// ListNodegroupRecords returns every nodegroup record under a cluster, sorted
|
||||
// by name for stable output.
|
||||
func ListNodegroupRecords(kv nats.KeyValue, cluster string) ([]*NodegroupRecord, error) {
|
||||
if cluster == "" {
|
||||
return nil, errors.New("eks: ListNodegroupRecords empty cluster")
|
||||
}
|
||||
keys, err := kv.Keys()
|
||||
if err != nil {
|
||||
if errors.Is(err, nats.ErrNoKeysFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("kv keys: %w", err)
|
||||
}
|
||||
prefix := NodegroupsPrefix(cluster)
|
||||
out := make([]*NodegroupRecord, 0)
|
||||
for _, k := range keys {
|
||||
if !strings.HasPrefix(k, prefix) {
|
||||
continue
|
||||
}
|
||||
entry, err := kv.Get(k)
|
||||
if err != nil {
|
||||
if errors.Is(err, nats.ErrKeyNotFound) {
|
||||
continue
|
||||
}
|
||||
return nil, fmt.Errorf("kv get %s: %w", k, err)
|
||||
}
|
||||
var rec NodegroupRecord
|
||||
if err := json.Unmarshal(entry.Value(), &rec); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal nodegroup %s: %w", k, err)
|
||||
}
|
||||
out = append(out, &rec)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// DeleteNodegroupRecord removes one record. A missing key is a no-op so
|
||||
// DeleteNodegroup stays idempotent.
|
||||
func DeleteNodegroupRecord(kv nats.KeyValue, cluster, ng string) error {
|
||||
key := NodegroupKey(cluster, ng)
|
||||
if err := kv.Delete(key); err != nil && !errors.Is(err, nats.ErrKeyNotFound) {
|
||||
return fmt.Errorf("kv delete %s: %w", key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// nodegroupAcctKV opens the per-account bucket for nodegroup handlers.
|
||||
func (s *EKSServiceImpl) nodegroupAcctKV(accountID string) (nats.KeyValue, error) {
|
||||
js, err := s.deps.NATSConn.JetStream()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("jetstream: %w", err)
|
||||
}
|
||||
acctKV, err := GetOrCreateAccountBucket(js, accountID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get account bucket: %w", err)
|
||||
}
|
||||
return acctKV, nil
|
||||
}
|
||||
|
||||
func (s *EKSServiceImpl) createNodegroup(acctKV nats.KeyValue, input *eks.CreateNodegroupInput, accountID string) (*eks.CreateNodegroupOutput, error) {
|
||||
if input == nil {
|
||||
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
|
||||
}
|
||||
cluster := aws.StringValue(input.ClusterName)
|
||||
ng := aws.StringValue(input.NodegroupName)
|
||||
if cluster == "" || ng == "" {
|
||||
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
|
||||
}
|
||||
subnets := aws.StringValueSlice(input.Subnets)
|
||||
if len(subnets) == 0 {
|
||||
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
|
||||
}
|
||||
|
||||
meta, err := GetClusterMeta(acctKV, cluster)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrClusterNotFound) {
|
||||
return nil, errors.New(awserrors.ErrorEKSResourceNotFound)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if meta.Status != ClusterStatusActive {
|
||||
slog.Warn("createNodegroup: cluster not ACTIVE", "cluster", cluster, "status", meta.Status)
|
||||
return nil, errors.New(awserrors.ErrorInvalidRequest)
|
||||
}
|
||||
if meta.ControlPlaneENIIP == "" {
|
||||
slog.Error("createNodegroup: cluster has no control-plane ENI IP", "cluster", cluster)
|
||||
return nil, errors.New(awserrors.ErrorInvalidRequest)
|
||||
}
|
||||
if meta.ResourcesVpcConfig == nil || meta.ResourcesVpcConfig.VpcId == "" {
|
||||
slog.Error("createNodegroup: cluster has no VPC", "cluster", cluster)
|
||||
return nil, errors.New(awserrors.ErrorInvalidRequest)
|
||||
}
|
||||
|
||||
if _, err := GetNodegroupRecord(acctKV, cluster, ng); err == nil {
|
||||
return nil, errors.New(awserrors.ErrorEKSResourceInUse)
|
||||
} else if !errors.Is(err, ErrNodegroupNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
instanceTypes := aws.StringValueSlice(input.InstanceTypes)
|
||||
if len(instanceTypes) == 0 {
|
||||
instanceTypes = []string{defaultNodegroupInstanceType}
|
||||
}
|
||||
minSize, maxSize, desired := scalingFromInput(input.ScalingConfig)
|
||||
|
||||
amiType := aws.StringValue(input.AmiType)
|
||||
if amiType == "" {
|
||||
amiType = eks.AMITypesAl2X8664
|
||||
}
|
||||
version := aws.StringValue(input.Version)
|
||||
if version == "" {
|
||||
version = meta.Version
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
rec := &NodegroupRecord{
|
||||
ClusterName: cluster,
|
||||
Name: ng,
|
||||
Arn: NodegroupARN(s.deps.Region, accountID, cluster, ng, uuid.NewString()),
|
||||
Status: eks.NodegroupStatusCreating,
|
||||
Subnets: subnets,
|
||||
InstanceTypes: instanceTypes,
|
||||
AMIType: amiType,
|
||||
DiskSize: aws.Int64Value(input.DiskSize),
|
||||
ScalingMin: minSize,
|
||||
ScalingMax: maxSize,
|
||||
ScalingDesired: desired,
|
||||
Version: version,
|
||||
NodeRole: aws.StringValue(input.NodeRole),
|
||||
Labels: aws.StringValueMap(input.Labels),
|
||||
CreatedAt: now,
|
||||
ModifiedAt: now,
|
||||
}
|
||||
|
||||
cpSGID, ngSGID, err := EnsureClusterSGs(s.deps.VPCSG, accountID, cluster, meta.ResourcesVpcConfig.VpcId)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ensure cluster SGs: %w", err)
|
||||
}
|
||||
if err := EnsureNodegroupSGRules(s.deps.VPCSG, accountID, cluster, cpSGID, ngSGID); err != nil {
|
||||
return nil, fmt.Errorf("ensure nodegroup SG rules: %w", err)
|
||||
}
|
||||
|
||||
if err := PutNodegroupRecord(acctKV, rec); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
token, err := s.decryptNodeToken(acctKV, cluster)
|
||||
if err != nil {
|
||||
s.markNodegroupFailed(acctKV, cluster, ng, "decrypt node token: "+err.Error())
|
||||
return nil, fmt.Errorf("decrypt node token: %w", err)
|
||||
}
|
||||
|
||||
amiID, err := lookupEKSServerAMI(s.deps.Image, accountID)
|
||||
if err != nil {
|
||||
s.markNodegroupFailed(acctKV, cluster, ng, "resolve eks-node AMI: "+err.Error())
|
||||
if errors.Is(err, ErrEKSServerAMINotFound) {
|
||||
return nil, errors.New(awserrors.ErrorServiceUnavailable)
|
||||
}
|
||||
return nil, fmt.Errorf("resolve eks-node AMI: %w", err)
|
||||
}
|
||||
|
||||
instanceIDs, err := s.launchWorkers(accountID, rec, meta, ngSGID, amiID, token, int(desired))
|
||||
rec.InstanceIDs = instanceIDs
|
||||
if err != nil {
|
||||
// Persist whatever launched so DeleteNodegroup can reclaim it, then fail.
|
||||
rec.Status = eks.NodegroupStatusCreateFailed
|
||||
rec.StatusReason = "launch workers: " + err.Error()
|
||||
rec.ModifiedAt = time.Now().UTC()
|
||||
if perr := PutNodegroupRecord(acctKV, rec); perr != nil {
|
||||
slog.Error("createNodegroup: persist CREATE_FAILED record", "cluster", cluster, "nodegroup", ng, "err", perr)
|
||||
}
|
||||
return nil, fmt.Errorf("launch workers: %w", err)
|
||||
}
|
||||
|
||||
// v1 marks ACTIVE on successful RunInstances (instance-level), not on
|
||||
// k8s node-Ready — node-Ready gating needs a kubectl probe from the
|
||||
// reconciler and lands in a follow-up.
|
||||
rec.Status = eks.NodegroupStatusActive
|
||||
rec.ModifiedAt = time.Now().UTC()
|
||||
if err := PutNodegroupRecord(acctKV, rec); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &eks.CreateNodegroupOutput{Nodegroup: nodegroupRecordToAWS(rec)}, nil
|
||||
}
|
||||
|
||||
// launchWorkers issues count customer-owned RunInstances calls (one per worker
|
||||
// so each gets a distinct node name + node label in its user-data) and returns
|
||||
// the launched instance IDs. On a partial failure it returns the IDs that did
|
||||
// launch plus the error so the caller can persist them for teardown.
|
||||
func (s *EKSServiceImpl) launchWorkers(accountID string, rec *NodegroupRecord, meta *ClusterMeta, ngSGID, amiID, token string, count int) ([]string, error) {
|
||||
instanceType := defaultNodegroupInstanceType
|
||||
if len(rec.InstanceTypes) > 0 {
|
||||
instanceType = rec.InstanceTypes[0]
|
||||
}
|
||||
subnet := rec.Subnets[0]
|
||||
|
||||
ids := make([]string, 0, count)
|
||||
for i := range count {
|
||||
shortID := uuid.NewString()[:8]
|
||||
userData := buildAgentUserData(agentUserDataInput{
|
||||
ClusterName: rec.ClusterName,
|
||||
NodegroupName: rec.Name,
|
||||
ControlPlaneENIIP: meta.ControlPlaneENIIP,
|
||||
JoinToken: token,
|
||||
NodeName: fmt.Sprintf("%s-%s-%s", rec.ClusterName, rec.Name, shortID),
|
||||
})
|
||||
runInput := &ec2.RunInstancesInput{
|
||||
ImageId: aws.String(amiID),
|
||||
InstanceType: aws.String(instanceType),
|
||||
MinCount: aws.Int64(1),
|
||||
MaxCount: aws.Int64(1),
|
||||
SubnetId: aws.String(subnet),
|
||||
SecurityGroupIds: aws.StringSlice([]string{ngSGID}),
|
||||
UserData: aws.String(userData),
|
||||
TagSpecifications: []*ec2.TagSpecification{{
|
||||
ResourceType: aws.String("instance"),
|
||||
Tags: []*ec2.Tag{
|
||||
{Key: aws.String(clusterEKSClusterTagKey), Value: aws.String(rec.ClusterName)},
|
||||
{Key: aws.String(clusterEKSNodegroupTagKey), Value: aws.String(rec.Name)},
|
||||
},
|
||||
}},
|
||||
}
|
||||
res, err := s.deps.Worker.RunWorkerInstance(runInput, accountID)
|
||||
if err != nil {
|
||||
return ids, fmt.Errorf("run worker %d/%d: %w", i+1, count, err)
|
||||
}
|
||||
for _, inst := range res.Instances {
|
||||
if id := aws.StringValue(inst.InstanceId); id != "" {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
func (s *EKSServiceImpl) describeNodegroup(acctKV nats.KeyValue, input *eks.DescribeNodegroupInput) (*eks.DescribeNodegroupOutput, error) {
|
||||
cluster := aws.StringValue(input.ClusterName)
|
||||
ng := aws.StringValue(input.NodegroupName)
|
||||
if cluster == "" || ng == "" {
|
||||
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
|
||||
}
|
||||
rec, err := GetNodegroupRecord(acctKV, cluster, ng)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrNodegroupNotFound) {
|
||||
return nil, errors.New(awserrors.ErrorEKSResourceNotFound)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return &eks.DescribeNodegroupOutput{Nodegroup: nodegroupRecordToAWS(rec)}, nil
|
||||
}
|
||||
|
||||
func (s *EKSServiceImpl) listNodegroups(acctKV nats.KeyValue, input *eks.ListNodegroupsInput) (*eks.ListNodegroupsOutput, error) {
|
||||
cluster := aws.StringValue(input.ClusterName)
|
||||
if cluster == "" {
|
||||
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
|
||||
}
|
||||
if _, err := GetClusterMeta(acctKV, cluster); err != nil {
|
||||
if errors.Is(err, ErrClusterNotFound) {
|
||||
return nil, errors.New(awserrors.ErrorEKSResourceNotFound)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
recs, err := ListNodegroupRecords(acctKV, cluster)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names := make([]string, 0, len(recs))
|
||||
for _, rec := range recs {
|
||||
names = append(names, rec.Name)
|
||||
}
|
||||
return &eks.ListNodegroupsOutput{Nodegroups: aws.StringSlice(names)}, nil
|
||||
}
|
||||
|
||||
func (s *EKSServiceImpl) updateNodegroupConfig(acctKV nats.KeyValue, input *eks.UpdateNodegroupConfigInput, accountID string) (*eks.UpdateNodegroupConfigOutput, error) {
|
||||
if input == nil {
|
||||
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
|
||||
}
|
||||
cluster := aws.StringValue(input.ClusterName)
|
||||
ng := aws.StringValue(input.NodegroupName)
|
||||
if cluster == "" || ng == "" {
|
||||
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
|
||||
}
|
||||
meta, err := GetClusterMeta(acctKV, cluster)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrClusterNotFound) {
|
||||
return nil, errors.New(awserrors.ErrorEKSResourceNotFound)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
rec, err := GetNodegroupRecord(acctKV, cluster, ng)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrNodegroupNotFound) {
|
||||
return nil, errors.New(awserrors.ErrorEKSResourceNotFound)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if input.ScalingConfig != nil {
|
||||
if v := input.ScalingConfig.MinSize; v != nil {
|
||||
rec.ScalingMin = *v
|
||||
}
|
||||
if v := input.ScalingConfig.MaxSize; v != nil {
|
||||
rec.ScalingMax = *v
|
||||
}
|
||||
if v := input.ScalingConfig.DesiredSize; v != nil {
|
||||
rec.ScalingDesired = *v
|
||||
}
|
||||
}
|
||||
if input.Labels != nil {
|
||||
rec.Labels = applyLabelUpdate(rec.Labels, input.Labels)
|
||||
}
|
||||
|
||||
if err := s.reconcileWorkerCount(acctKV, accountID, rec, meta); err != nil {
|
||||
// reconcileWorkerCount may have launched some workers before failing; their
|
||||
// IDs are already on rec. Persist so DeleteNodegroup can reclaim them rather
|
||||
// than leaking the partial scale-up.
|
||||
rec.ModifiedAt = time.Now().UTC()
|
||||
if perr := PutNodegroupRecord(acctKV, rec); perr != nil {
|
||||
slog.Error("updateNodegroupConfig: persist after reconcile failure",
|
||||
"cluster", cluster, "nodegroup", ng, "err", perr)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
rec.ModifiedAt = time.Now().UTC()
|
||||
if err := PutNodegroupRecord(acctKV, rec); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &eks.UpdateNodegroupConfigOutput{Update: &eks.Update{
|
||||
Id: aws.String(uuid.NewString()),
|
||||
Status: aws.String(eks.UpdateStatusSuccessful),
|
||||
Type: aws.String(eks.UpdateTypeConfigUpdate),
|
||||
CreatedAt: aws.Time(rec.ModifiedAt),
|
||||
}}, nil
|
||||
}
|
||||
|
||||
// reconcileWorkerCount launches or terminates workers so len(InstanceIDs)
|
||||
// matches ScalingDesired. Surplus removal terminates the last (highest-index)
|
||||
// instance IDs so scale-down is deterministic.
|
||||
func (s *EKSServiceImpl) reconcileWorkerCount(acctKV nats.KeyValue, accountID string, rec *NodegroupRecord, meta *ClusterMeta) error {
|
||||
current := len(rec.InstanceIDs)
|
||||
desired := int(rec.ScalingDesired)
|
||||
|
||||
switch {
|
||||
case desired > current:
|
||||
token, err := s.decryptNodeToken(acctKV, rec.ClusterName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("decrypt node token: %w", err)
|
||||
}
|
||||
amiID, err := lookupEKSServerAMI(s.deps.Image, accountID)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrEKSServerAMINotFound) {
|
||||
return errors.New(awserrors.ErrorServiceUnavailable)
|
||||
}
|
||||
return fmt.Errorf("resolve eks-node AMI: %w", err)
|
||||
}
|
||||
_, ngSGID, err := EnsureClusterSGs(s.deps.VPCSG, accountID, rec.ClusterName, meta.ResourcesVpcConfig.VpcId)
|
||||
if err != nil {
|
||||
return fmt.Errorf("ensure cluster SGs: %w", err)
|
||||
}
|
||||
newIDs, err := s.launchWorkers(accountID, rec, meta, ngSGID, amiID, token, desired-current)
|
||||
rec.InstanceIDs = append(rec.InstanceIDs, newIDs...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("launch workers: %w", err)
|
||||
}
|
||||
case desired < current:
|
||||
surplus := rec.InstanceIDs[desired:]
|
||||
if err := s.deps.Worker.TerminateWorkerInstances(surplus, accountID); err != nil {
|
||||
return fmt.Errorf("terminate surplus workers: %w", err)
|
||||
}
|
||||
rec.InstanceIDs = rec.InstanceIDs[:desired]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *EKSServiceImpl) updateNodegroupVersion(input *eks.UpdateNodegroupVersionInput) (*eks.UpdateNodegroupVersionOutput, error) {
|
||||
// Worker AMI version upgrades (drain + replace) land in a follow-up; v1
|
||||
// nodegroups are fixed at their create-time version.
|
||||
if input != nil {
|
||||
slog.Info("UpdateNodegroupVersion not implemented in v1",
|
||||
"cluster", aws.StringValue(input.ClusterName), "nodegroup", aws.StringValue(input.NodegroupName))
|
||||
}
|
||||
return nil, notImpl()
|
||||
}
|
||||
|
||||
func (s *EKSServiceImpl) deleteNodegroup(acctKV nats.KeyValue, input *eks.DeleteNodegroupInput, accountID string) (*eks.DeleteNodegroupOutput, error) {
|
||||
if input == nil {
|
||||
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
|
||||
}
|
||||
cluster := aws.StringValue(input.ClusterName)
|
||||
ng := aws.StringValue(input.NodegroupName)
|
||||
if cluster == "" || ng == "" {
|
||||
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
|
||||
}
|
||||
rec, err := GetNodegroupRecord(acctKV, cluster, ng)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrNodegroupNotFound) {
|
||||
return nil, errors.New(awserrors.ErrorEKSResourceNotFound)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(rec.InstanceIDs) > 0 {
|
||||
if err := s.deps.Worker.TerminateWorkerInstances(rec.InstanceIDs, accountID); err != nil {
|
||||
return nil, fmt.Errorf("terminate workers: %w", err)
|
||||
}
|
||||
}
|
||||
if err := DeleteNodegroupRecord(acctKV, cluster, ng); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
rec.Status = eks.NodegroupStatusDeleting
|
||||
return &eks.DeleteNodegroupOutput{Nodegroup: nodegroupRecordToAWS(rec)}, nil
|
||||
}
|
||||
|
||||
// decryptNodeToken reads + decrypts the cluster's K3s join token from KV.
|
||||
func (s *EKSServiceImpl) decryptNodeToken(kv nats.KeyValue, cluster string) (string, error) {
|
||||
if kv == nil {
|
||||
return "", errors.New("eks: nil KV for node token")
|
||||
}
|
||||
entry, err := kv.Get(NodeTokenKey(cluster))
|
||||
if err != nil {
|
||||
if errors.Is(err, nats.ErrKeyNotFound) {
|
||||
return "", errors.New("eks: cluster join token not yet provisioned")
|
||||
}
|
||||
return "", fmt.Errorf("kv get node token: %w", err)
|
||||
}
|
||||
token, err := handlers_iam.DecryptSecret(string(entry.Value()), s.deps.MasterKey)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("decrypt node token: %w", err)
|
||||
}
|
||||
if token == "" {
|
||||
return "", errors.New("eks: decrypted node token is empty")
|
||||
}
|
||||
return token, nil
|
||||
}
|
||||
|
||||
func (s *EKSServiceImpl) markNodegroupFailed(kv nats.KeyValue, cluster, ng, reason string) {
|
||||
rec, err := GetNodegroupRecord(kv, cluster, ng)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
rec.Status = eks.NodegroupStatusCreateFailed
|
||||
rec.StatusReason = reason
|
||||
rec.ModifiedAt = time.Now().UTC()
|
||||
if err := PutNodegroupRecord(kv, rec); err != nil {
|
||||
slog.Warn("markNodegroupFailed: persist failed", "cluster", cluster, "nodegroup", ng, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// scalingFromInput derives min/max/desired from a NodegroupScalingConfig,
|
||||
// defaulting to a single node when the caller omits the config or fields.
|
||||
func scalingFromInput(cfg *eks.NodegroupScalingConfig) (minSize, maxSize, desired int64) {
|
||||
minSize, maxSize, desired = 1, 1, 1
|
||||
if cfg == nil {
|
||||
return minSize, maxSize, desired
|
||||
}
|
||||
if cfg.MinSize != nil {
|
||||
minSize = *cfg.MinSize
|
||||
}
|
||||
if cfg.DesiredSize != nil {
|
||||
desired = *cfg.DesiredSize
|
||||
} else {
|
||||
desired = minSize
|
||||
}
|
||||
if cfg.MaxSize != nil {
|
||||
maxSize = *cfg.MaxSize
|
||||
} else if desired > maxSize {
|
||||
maxSize = desired
|
||||
}
|
||||
return minSize, maxSize, desired
|
||||
}
|
||||
|
||||
// applyLabelUpdate applies an UpdateLabelsPayload (addOrUpdate + removeLabels)
|
||||
// onto the record's label map.
|
||||
func applyLabelUpdate(existing map[string]string, payload *eks.UpdateLabelsPayload) map[string]string {
|
||||
if payload == nil {
|
||||
return existing
|
||||
}
|
||||
out := map[string]string{}
|
||||
maps.Copy(out, existing)
|
||||
for k, v := range payload.AddOrUpdateLabels {
|
||||
out[k] = aws.StringValue(v)
|
||||
}
|
||||
for _, k := range payload.RemoveLabels {
|
||||
delete(out, aws.StringValue(k))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func nodegroupRecordToAWS(rec *NodegroupRecord) *eks.Nodegroup {
|
||||
if rec == nil {
|
||||
return nil
|
||||
}
|
||||
out := &eks.Nodegroup{
|
||||
ClusterName: aws.String(rec.ClusterName),
|
||||
NodegroupName: aws.String(rec.Name),
|
||||
NodegroupArn: aws.String(rec.Arn),
|
||||
Status: aws.String(rec.Status),
|
||||
Subnets: aws.StringSlice(rec.Subnets),
|
||||
InstanceTypes: aws.StringSlice(rec.InstanceTypes),
|
||||
AmiType: aws.String(rec.AMIType),
|
||||
NodeRole: aws.String(rec.NodeRole),
|
||||
Version: aws.String(rec.Version),
|
||||
CreatedAt: aws.Time(rec.CreatedAt),
|
||||
ModifiedAt: aws.Time(rec.ModifiedAt),
|
||||
ScalingConfig: &eks.NodegroupScalingConfig{
|
||||
MinSize: aws.Int64(rec.ScalingMin),
|
||||
MaxSize: aws.Int64(rec.ScalingMax),
|
||||
DesiredSize: aws.Int64(rec.ScalingDesired),
|
||||
},
|
||||
Resources: &eks.NodegroupResources{},
|
||||
}
|
||||
if rec.DiskSize > 0 {
|
||||
out.DiskSize = aws.Int64(rec.DiskSize)
|
||||
}
|
||||
if len(rec.Labels) > 0 {
|
||||
out.Labels = aws.StringMap(rec.Labels)
|
||||
}
|
||||
if rec.StatusReason != "" {
|
||||
out.Health = &eks.NodegroupHealth{Issues: []*eks.Issue{{
|
||||
Code: aws.String(eks.NodegroupIssueCodeNodeCreationFailure),
|
||||
Message: aws.String(rec.StatusReason),
|
||||
ResourceIds: aws.StringSlice(rec.InstanceIDs),
|
||||
}}}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,321 @@
|
||||
package handlers_eks
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/aws/aws-sdk-go/service/eks"
|
||||
"github.com/mulgadc/spinifex/spinifex/awserrors"
|
||||
handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
|
||||
"github.com/mulgadc/spinifex/spinifex/testutil"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// fakeWorkerLauncher records RunWorkerInstance / TerminateWorkerInstances calls
|
||||
// and hands back sequential instance IDs so tests can assert on the IDs a
|
||||
// nodegroup tracks.
|
||||
type fakeWorkerLauncher struct {
|
||||
runCalls []*ec2.RunInstancesInput
|
||||
terminateCalls [][]string
|
||||
|
||||
nextID int
|
||||
runErr error
|
||||
termErr error
|
||||
}
|
||||
|
||||
var _ WorkerLauncher = (*fakeWorkerLauncher)(nil)
|
||||
|
||||
func newFakeWorkerLauncher() *fakeWorkerLauncher {
|
||||
return &fakeWorkerLauncher{}
|
||||
}
|
||||
|
||||
func (f *fakeWorkerLauncher) RunWorkerInstance(input *ec2.RunInstancesInput, _ string) (*ec2.Reservation, error) {
|
||||
f.runCalls = append(f.runCalls, input)
|
||||
if f.runErr != nil {
|
||||
return nil, f.runErr
|
||||
}
|
||||
f.nextID++
|
||||
id := "i-worker" + strconv.Itoa(f.nextID)
|
||||
return &ec2.Reservation{Instances: []*ec2.Instance{{InstanceId: aws.String(id)}}}, nil
|
||||
}
|
||||
|
||||
func (f *fakeWorkerLauncher) TerminateWorkerInstances(ids []string, _ string) error {
|
||||
f.terminateCalls = append(f.terminateCalls, ids)
|
||||
return f.termErr
|
||||
}
|
||||
|
||||
// seedActiveClusterWithToken lays down an ACTIVE cluster meta with the control-
|
||||
// plane ENI IP + VPC populated and an encrypted K3s join token, so the
|
||||
// nodegroup path can run end-to-end.
|
||||
func seedActiveClusterWithToken(t *testing.T, f *eksServiceFixture, cluster string) {
|
||||
t.Helper()
|
||||
meta := &ClusterMeta{
|
||||
Name: cluster,
|
||||
Status: ClusterStatusActive,
|
||||
Version: "1.32",
|
||||
ControlPlaneENIIP: "10.0.1.42",
|
||||
ResourcesVpcConfig: &ClusterVpcConfig{
|
||||
SubnetIds: []string{"subnet-aaa"},
|
||||
VpcId: "vpc-aaa",
|
||||
},
|
||||
}
|
||||
require.NoError(t, PutClusterMeta(f.kv, meta))
|
||||
|
||||
ct, err := handlers_iam.EncryptSecret("K10node-join-token::server:abc", bootstrapTestMasterKey)
|
||||
require.NoError(t, err)
|
||||
_, err = f.kv.Put(NodeTokenKey(cluster), []byte(ct))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func createNGInput(cluster, ng string, desired int64) *eks.CreateNodegroupInput {
|
||||
return &eks.CreateNodegroupInput{
|
||||
ClusterName: aws.String(cluster),
|
||||
NodegroupName: aws.String(ng),
|
||||
NodeRole: aws.String("arn:aws:iam::111122223333:role/eks-node"),
|
||||
Subnets: aws.StringSlice([]string{"subnet-aaa"}),
|
||||
ScalingConfig: &eks.NodegroupScalingConfig{
|
||||
MinSize: aws.Int64(1),
|
||||
MaxSize: aws.Int64(3),
|
||||
DesiredSize: aws.Int64(desired),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodegroupRecord_CRUDRoundTrip(t *testing.T) {
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
kv, err := GetOrCreateAccountBucket(js, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
rec := &NodegroupRecord{
|
||||
ClusterName: "c1",
|
||||
Name: "ng1",
|
||||
Arn: "arn:aws:eks:us-east-1:111122223333:nodegroup/c1/ng1/abc",
|
||||
Status: eks.NodegroupStatusActive,
|
||||
Subnets: []string{"subnet-aaa"},
|
||||
InstanceTypes: []string{"t3.medium"},
|
||||
ScalingMin: 1,
|
||||
ScalingMax: 3,
|
||||
ScalingDesired: 2,
|
||||
InstanceIDs: []string{"i-1", "i-2"},
|
||||
}
|
||||
require.NoError(t, PutNodegroupRecord(kv, rec))
|
||||
|
||||
got, err := GetNodegroupRecord(kv, "c1", "ng1")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, rec.InstanceIDs, got.InstanceIDs)
|
||||
assert.Equal(t, int64(2), got.ScalingDesired)
|
||||
|
||||
list, err := ListNodegroupRecords(kv, "c1")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, list, 1)
|
||||
assert.Equal(t, "ng1", list[0].Name)
|
||||
|
||||
require.NoError(t, DeleteNodegroupRecord(kv, "c1", "ng1"))
|
||||
_, err = GetNodegroupRecord(kv, "c1", "ng1")
|
||||
assert.ErrorIs(t, err, ErrNodegroupNotFound)
|
||||
|
||||
// Delete of an absent record is idempotent.
|
||||
require.NoError(t, DeleteNodegroupRecord(kv, "c1", "ng1"))
|
||||
}
|
||||
|
||||
func TestBuildAgentUserData_Shape(t *testing.T) {
|
||||
ud := buildAgentUserData(agentUserDataInput{
|
||||
ClusterName: "c1",
|
||||
NodegroupName: "ng1",
|
||||
ControlPlaneENIIP: "10.0.1.42",
|
||||
JoinToken: "K10secret::server:xyz",
|
||||
NodeName: "c1-ng1-abc123de",
|
||||
})
|
||||
|
||||
assert.True(t, strings.HasPrefix(ud, "#cloud-config\n"))
|
||||
assert.Contains(t, ud, "SPINIFEX_K3S_ROLE=agent")
|
||||
assert.Contains(t, ud, "K3S_URL=https://10.0.1.42:6443")
|
||||
assert.Contains(t, ud, "K3S_TOKEN=K10secret::server:xyz")
|
||||
assert.Contains(t, ud, "K3S_NODE_NAME=c1-ng1-abc123de")
|
||||
assert.Contains(t, ud, "eks.amazonaws.com/nodegroup=ng1")
|
||||
assert.Contains(t, ud, agentEnvPath)
|
||||
// IMDS on-link route line present.
|
||||
assert.Contains(t, ud, "ip route replace "+imdsServerIP+"/32")
|
||||
// Exactly one write_files block (single "write_files:" key).
|
||||
assert.Equal(t, 1, strings.Count(ud, "write_files:"))
|
||||
}
|
||||
|
||||
func TestEnsureNodegroupSGRules_AuthorizesExpectedRules(t *testing.T) {
|
||||
sg := newFakeSGProvisioner()
|
||||
|
||||
require.NoError(t, EnsureNodegroupSGRules(sg, testAccountID, "c1", "sg-cp", "sg-ng"))
|
||||
|
||||
require.Len(t, sg.authorizeCalls, 6)
|
||||
|
||||
// Verify the apiserver rule: 6443/tcp on cpSG sourced from ngSG.
|
||||
var found6443 bool
|
||||
for _, call := range sg.authorizeCalls {
|
||||
require.Len(t, call.IpPermissions, 1)
|
||||
perm := call.IpPermissions[0]
|
||||
if aws.StringValue(call.GroupId) == "sg-cp" &&
|
||||
aws.StringValue(perm.IpProtocol) == "tcp" &&
|
||||
aws.Int64Value(perm.FromPort) == 6443 {
|
||||
require.Len(t, perm.UserIdGroupPairs, 1)
|
||||
assert.Equal(t, "sg-ng", aws.StringValue(perm.UserIdGroupPairs[0].GroupId))
|
||||
assert.Empty(t, perm.IpRanges, "rules must be group-to-group, not CIDR")
|
||||
found6443 = true
|
||||
}
|
||||
}
|
||||
assert.True(t, found6443, "expected agent→apiserver 6443/tcp rule")
|
||||
}
|
||||
|
||||
func TestEnsureNodegroupSGRules_DuplicateRuleTolerated(t *testing.T) {
|
||||
sg := newFakeSGProvisioner()
|
||||
sg.authorizeErr = errors.New(awserrors.ErrorInvalidPermissionDuplicate)
|
||||
|
||||
require.NoError(t, EnsureNodegroupSGRules(sg, testAccountID, "c1", "sg-cp", "sg-ng"),
|
||||
"duplicate-rule error must be treated as success")
|
||||
}
|
||||
|
||||
func TestCreateNodegroup_HappyPath(t *testing.T) {
|
||||
f := newEKSServiceFixture(t)
|
||||
seedActiveClusterWithToken(t, f, "c1")
|
||||
|
||||
out, err := f.svc.CreateNodegroup(createNGInput("c1", "ng1", 2), testAccountID)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, out.Nodegroup)
|
||||
assert.Equal(t, eks.NodegroupStatusActive, aws.StringValue(out.Nodegroup.Status))
|
||||
assert.Equal(t, int64(2), aws.Int64Value(out.Nodegroup.ScalingConfig.DesiredSize))
|
||||
assert.Contains(t, aws.StringValue(out.Nodegroup.NodegroupArn), ":nodegroup/c1/ng1/")
|
||||
|
||||
// Two workers launched and tracked.
|
||||
assert.Len(t, f.worker.runCalls, 2)
|
||||
rec, err := GetNodegroupRecord(f.kv, "c1", "ng1")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, rec.InstanceIDs, 2)
|
||||
|
||||
// Workers go through the customer path: no ManagedBy tag, SG = nodegroup SG.
|
||||
for _, call := range f.worker.runCalls {
|
||||
require.NotEmpty(t, call.SecurityGroupIds)
|
||||
assert.Equal(t, "subnet-aaa", aws.StringValue(call.SubnetId))
|
||||
for _, ts := range call.TagSpecifications {
|
||||
for _, tg := range ts.Tags {
|
||||
assert.NotEqual(t, "spinifex:managed-by", aws.StringValue(tg.Key))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SG ingress rules were authorized.
|
||||
assert.NotEmpty(t, f.sg.authorizeCalls)
|
||||
|
||||
// Duplicate create → ResourceInUseException.
|
||||
_, err = f.svc.CreateNodegroup(createNGInput("c1", "ng1", 2), testAccountID)
|
||||
require.EqualError(t, err, awserrors.ErrorEKSResourceInUse)
|
||||
}
|
||||
|
||||
func TestCreateNodegroup_ClusterNotActive(t *testing.T) {
|
||||
f := newEKSServiceFixture(t)
|
||||
require.NoError(t, PutClusterMeta(f.kv, &ClusterMeta{
|
||||
Name: "c1",
|
||||
Status: ClusterStatusCreating,
|
||||
ControlPlaneENIIP: "10.0.1.42",
|
||||
ResourcesVpcConfig: &ClusterVpcConfig{VpcId: "vpc-aaa"},
|
||||
}))
|
||||
|
||||
_, err := f.svc.CreateNodegroup(createNGInput("c1", "ng1", 1), testAccountID)
|
||||
require.EqualError(t, err, awserrors.ErrorInvalidRequest)
|
||||
}
|
||||
|
||||
func TestCreateNodegroup_UnknownCluster(t *testing.T) {
|
||||
f := newEKSServiceFixture(t)
|
||||
_, err := f.svc.CreateNodegroup(createNGInput("ghost", "ng1", 1), testAccountID)
|
||||
require.EqualError(t, err, awserrors.ErrorEKSResourceNotFound)
|
||||
}
|
||||
|
||||
func TestDescribeAndListNodegroups(t *testing.T) {
|
||||
f := newEKSServiceFixture(t)
|
||||
seedActiveClusterWithToken(t, f, "c1")
|
||||
_, err := f.svc.CreateNodegroup(createNGInput("c1", "ng1", 1), testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
desc, err := f.svc.DescribeNodegroup(&eks.DescribeNodegroupInput{
|
||||
ClusterName: aws.String("c1"), NodegroupName: aws.String("ng1"),
|
||||
}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "ng1", aws.StringValue(desc.Nodegroup.NodegroupName))
|
||||
assert.Equal(t, "c1", aws.StringValue(desc.Nodegroup.ClusterName))
|
||||
|
||||
list, err := f.svc.ListNodegroups(&eks.ListNodegroupsInput{ClusterName: aws.String("c1")}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"ng1"}, aws.StringValueSlice(list.Nodegroups))
|
||||
|
||||
_, err = f.svc.DescribeNodegroup(&eks.DescribeNodegroupInput{
|
||||
ClusterName: aws.String("c1"), NodegroupName: aws.String("ghost"),
|
||||
}, testAccountID)
|
||||
require.EqualError(t, err, awserrors.ErrorEKSResourceNotFound)
|
||||
}
|
||||
|
||||
func TestUpdateNodegroupConfig_ScaleUpAndDown(t *testing.T) {
|
||||
f := newEKSServiceFixture(t)
|
||||
seedActiveClusterWithToken(t, f, "c1")
|
||||
_, err := f.svc.CreateNodegroup(createNGInput("c1", "ng1", 1), testAccountID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, f.worker.runCalls, 1)
|
||||
|
||||
// Scale up 1 → 3: two new workers launched.
|
||||
upOut, err := f.svc.UpdateNodegroupConfig(&eks.UpdateNodegroupConfigInput{
|
||||
ClusterName: aws.String("c1"),
|
||||
NodegroupName: aws.String("ng1"),
|
||||
ScalingConfig: &eks.NodegroupScalingConfig{DesiredSize: aws.Int64(3)},
|
||||
}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, eks.UpdateStatusSuccessful, aws.StringValue(upOut.Update.Status))
|
||||
assert.Len(t, f.worker.runCalls, 3)
|
||||
rec, err := GetNodegroupRecord(f.kv, "c1", "ng1")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, rec.InstanceIDs, 3)
|
||||
|
||||
// Scale down 3 → 1: surplus (last two) terminated.
|
||||
_, err = f.svc.UpdateNodegroupConfig(&eks.UpdateNodegroupConfigInput{
|
||||
ClusterName: aws.String("c1"),
|
||||
NodegroupName: aws.String("ng1"),
|
||||
ScalingConfig: &eks.NodegroupScalingConfig{DesiredSize: aws.Int64(1)},
|
||||
}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, f.worker.terminateCalls, 1)
|
||||
assert.Len(t, f.worker.terminateCalls[0], 2)
|
||||
rec, err = GetNodegroupRecord(f.kv, "c1", "ng1")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, rec.InstanceIDs, 1)
|
||||
}
|
||||
|
||||
func TestDeleteNodegroup_TerminatesAndIsIdempotent(t *testing.T) {
|
||||
f := newEKSServiceFixture(t)
|
||||
seedActiveClusterWithToken(t, f, "c1")
|
||||
_, err := f.svc.CreateNodegroup(createNGInput("c1", "ng1", 2), testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = f.svc.DeleteNodegroup(&eks.DeleteNodegroupInput{
|
||||
ClusterName: aws.String("c1"), NodegroupName: aws.String("ng1"),
|
||||
}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, f.worker.terminateCalls, 1)
|
||||
assert.Len(t, f.worker.terminateCalls[0], 2)
|
||||
|
||||
_, err = GetNodegroupRecord(f.kv, "c1", "ng1")
|
||||
assert.ErrorIs(t, err, ErrNodegroupNotFound)
|
||||
|
||||
// Second delete → ResourceNotFoundException (record already gone).
|
||||
_, err = f.svc.DeleteNodegroup(&eks.DeleteNodegroupInput{
|
||||
ClusterName: aws.String("c1"), NodegroupName: aws.String("ng1"),
|
||||
}, testAccountID)
|
||||
require.EqualError(t, err, awserrors.ErrorEKSResourceNotFound)
|
||||
}
|
||||
|
||||
func TestUpdateNodegroupVersion_NotImplemented(t *testing.T) {
|
||||
f := newEKSServiceFixture(t)
|
||||
_, err := f.svc.UpdateNodegroupVersion(&eks.UpdateNodegroupVersionInput{
|
||||
ClusterName: aws.String("c1"), NodegroupName: aws.String("ng1"),
|
||||
}, testAccountID)
|
||||
require.EqualError(t, err, awserrors.ErrorNotImplemented)
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package handlers_eks
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// agentUserDataInput is the input shape for buildAgentUserData. ControlPlaneENIIP
|
||||
// is the cluster's in-VPC control-plane ENI private IP (the K3s join target);
|
||||
// JoinToken is the decrypted K3s node token.
|
||||
type agentUserDataInput struct {
|
||||
ClusterName string
|
||||
NodegroupName string
|
||||
ControlPlaneENIIP string
|
||||
JoinToken string
|
||||
NodeName string
|
||||
}
|
||||
|
||||
// buildAgentUserData renders the cloud-config YAML a nodegroup worker consumes
|
||||
// at first boot. It mirrors buildK3sUserData: a bootcmd resolver fix, a single
|
||||
// write_files block (first-boot.env + agent.env + the IMDS on-link route
|
||||
// script), and a runcmd that enables the OpenRC `local` service. Output is
|
||||
// unencoded YAML; the caller base64-wraps it for the RunInstances UserData
|
||||
// field. The eks-node-role first-boot selector reads SPINIFEX_K3S_ROLE=agent
|
||||
// (and the presence of agent.env) to enable the k3s-agent service, which sources
|
||||
// agent.env for K3S_URL/K3S_TOKEN.
|
||||
func buildAgentUserData(in agentUserDataInput) string {
|
||||
// K3S_URL targets the control-plane ENI's in-VPC private IP on :6443
|
||||
// because the cluster NLB DNS endpoint is not resolvable in-VPC yet.
|
||||
// Workers share the VPC with the server, so this is directly reachable and
|
||||
// TLS-pinned by the join token's CA hash (SAN/hostname irrelevant for the
|
||||
// k3s agent join). Switch to the NLB endpoint on :443 once authoritative
|
||||
// in-VPC DNS is available.
|
||||
k3sURL := "https://" + net.JoinHostPort(in.ControlPlaneENIIP, "6443")
|
||||
nodeLabel := "eks.amazonaws.com/nodegroup=" + in.NodegroupName
|
||||
|
||||
envBody := strings.Join([]string{
|
||||
"SPINIFEX_K3S_ROLE=agent",
|
||||
}, "\n")
|
||||
|
||||
agentBody := strings.Join([]string{
|
||||
"K3S_URL=" + k3sURL,
|
||||
"K3S_TOKEN=" + in.JoinToken,
|
||||
"K3S_NODE_NAME=" + in.NodeName,
|
||||
"K3S_NODE_LABEL=" + nodeLabel,
|
||||
}, "\n")
|
||||
|
||||
files := []userDataFile{
|
||||
{Path: k3sFirstBootEnvPath, Perms: "0600", Body: envBody},
|
||||
{Path: agentEnvPath, Perms: "0600", Body: agentBody},
|
||||
{
|
||||
Path: "/etc/local.d/imds-onlink-route.start",
|
||||
Perms: "0755",
|
||||
Body: "#!/bin/sh\n" +
|
||||
"dev=$(ip route show default | awk '{print $5; exit}')\n" +
|
||||
"[ -n \"$dev\" ] && ip route replace " + imdsServerIP + "/32 dev \"$dev\" scope link",
|
||||
},
|
||||
}
|
||||
|
||||
var buf strings.Builder
|
||||
buf.WriteString("#cloud-config\n")
|
||||
|
||||
// Resolver fix via bootcmd (not write_files) for the same reason as the
|
||||
// server path: /etc/resolv.conf is a dangling symlink on the Alpine AMI, so
|
||||
// write_files would follow the dead link and abort the whole block.
|
||||
// Containerd needs a working resolver to pull the worker's system images.
|
||||
buf.WriteString("bootcmd:\n")
|
||||
buf.WriteString(" - rm -f " + k3sResolvConfPath + "\n")
|
||||
fmt.Fprintf(&buf, " - printf '%s\\n' > %s\n",
|
||||
strings.ReplaceAll(k3sResolvConf, "\n", "\\n"), k3sResolvConfPath)
|
||||
|
||||
buf.WriteString("write_files:\n")
|
||||
for _, f := range files {
|
||||
fmt.Fprintf(&buf, " - path: %s\n", f.Path)
|
||||
fmt.Fprintf(&buf, " permissions: '%s'\n", f.Perms)
|
||||
buf.WriteString(" content: |\n")
|
||||
for line := range strings.SplitSeq(f.Body, "\n") {
|
||||
buf.WriteString(" ")
|
||||
buf.WriteString(line)
|
||||
buf.WriteString("\n")
|
||||
}
|
||||
}
|
||||
|
||||
buf.WriteString("runcmd:\n")
|
||||
buf.WriteString(" - [ rc-update, add, local, default ]\n")
|
||||
buf.WriteString(" - [ rc-service, local, start ]\n")
|
||||
|
||||
return buf.String()
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/spinifex/spinifex/awserrors"
|
||||
"github.com/mulgadc/spinifex/spinifex/tags"
|
||||
)
|
||||
|
||||
@@ -17,6 +18,7 @@ type sgProvisioner interface {
|
||||
CreateSecurityGroup(input *ec2.CreateSecurityGroupInput, accountID string) (*ec2.CreateSecurityGroupOutput, error)
|
||||
DescribeSecurityGroups(input *ec2.DescribeSecurityGroupsInput, accountID string) (*ec2.DescribeSecurityGroupsOutput, error)
|
||||
DeleteSecurityGroup(input *ec2.DeleteSecurityGroupInput, accountID string) (*ec2.DeleteSecurityGroupOutput, error)
|
||||
AuthorizeSecurityGroupIngress(input *ec2.AuthorizeSecurityGroupIngressInput, accountID string) (*ec2.AuthorizeSecurityGroupIngressOutput, error)
|
||||
}
|
||||
|
||||
// clusterEKSClusterTagKey is stamped on every cluster-scoped resource so the
|
||||
@@ -27,6 +29,10 @@ const clusterEKSClusterTagKey = "spinifex:eks-cluster"
|
||||
// SG when both live in the same VPC.
|
||||
const clusterEKSRoleTagKey = "spinifex:eks-role"
|
||||
|
||||
// clusterEKSNodegroupTagKey is stamped on worker instances so the nodegroup
|
||||
// they belong to is recoverable from EC2 tags alone.
|
||||
const clusterEKSNodegroupTagKey = "spinifex:eks-nodegroup"
|
||||
|
||||
const (
|
||||
clusterEKSRoleControlPlane = "control-plane"
|
||||
clusterEKSRoleNodegroup = "nodegroup"
|
||||
@@ -142,6 +148,100 @@ func ensureClusterSG(sgp sgProvisioner, accountID, vpcID, clusterName, name, des
|
||||
return *out.GroupId, nil
|
||||
}
|
||||
|
||||
// EnsureNodegroupSGRules authorizes the ingress rules a nodegroup's workers
|
||||
// need to join the K3s control plane and form the flannel overlay. All rules
|
||||
// are group-to-group (UserIdGroupPairs) so they stay correct as instances come
|
||||
// and go. Idempotent: the InvalidPermission.Duplicate error a re-run triggers
|
||||
// for an already-authorized rule is treated as success.
|
||||
//
|
||||
// Rules:
|
||||
// - agent → server apiserver/supervisor: 6443/tcp (ngSG → cpSG)
|
||||
// - flannel VXLAN: 8472/udp both directions between cpSG and ngSG, and
|
||||
// ngSG ↔ ngSG
|
||||
// - kubelet: 10250/tcp (cpSG → ngSG)
|
||||
// - intra-nodegroup all traffic: ngSG self-reference
|
||||
func EnsureNodegroupSGRules(sgp sgProvisioner, accountID, clusterName, cpSGID, ngSGID string) error {
|
||||
if cpSGID == "" || ngSGID == "" {
|
||||
return errors.New("eks: EnsureNodegroupSGRules empty SG id")
|
||||
}
|
||||
|
||||
type rule struct {
|
||||
targetSG string
|
||||
sourceSG string
|
||||
proto string
|
||||
from int64
|
||||
to int64
|
||||
desc string
|
||||
}
|
||||
rules := []rule{
|
||||
{cpSGID, ngSGID, "tcp", 6443, 6443, "EKS agent to apiserver/supervisor"},
|
||||
{cpSGID, ngSGID, "udp", 8472, 8472, "EKS flannel VXLAN node to control-plane"},
|
||||
{ngSGID, cpSGID, "udp", 8472, 8472, "EKS flannel VXLAN control-plane to node"},
|
||||
{ngSGID, ngSGID, "udp", 8472, 8472, "EKS flannel VXLAN node to node"},
|
||||
{ngSGID, cpSGID, "tcp", 10250, 10250, "EKS kubelet control-plane to node"},
|
||||
{ngSGID, ngSGID, "-1", -1, -1, "EKS intra-nodegroup all traffic"},
|
||||
}
|
||||
|
||||
for _, r := range rules {
|
||||
perm := &ec2.IpPermission{
|
||||
IpProtocol: aws.String(r.proto),
|
||||
UserIdGroupPairs: []*ec2.UserIdGroupPair{{
|
||||
GroupId: aws.String(r.sourceSG),
|
||||
Description: aws.String(r.desc),
|
||||
}},
|
||||
}
|
||||
if r.proto != "-1" {
|
||||
perm.FromPort = aws.Int64(r.from)
|
||||
perm.ToPort = aws.Int64(r.to)
|
||||
}
|
||||
_, err := sgp.AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{
|
||||
GroupId: aws.String(r.targetSG),
|
||||
IpPermissions: []*ec2.IpPermission{perm},
|
||||
}, accountID)
|
||||
if err != nil {
|
||||
if awserrors.IsErrorCode(err, awserrors.ErrorInvalidPermissionDuplicate) {
|
||||
continue
|
||||
}
|
||||
return fmt.Errorf("authorize %s ingress on %s: %w", r.desc, r.targetSG, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// EnsureControlPlaneIngress authorizes the ingress the cluster NLB needs to
|
||||
// reach the K3s apiserver. The NLB's backing LB VM forwards the front-end
|
||||
// listener to the apiserver on k3sAPIServerPort from inside the VPC, so the
|
||||
// control-plane SG must admit that port from the VPC CIDR. Public exposure is
|
||||
// gated separately at the NLB front-end by publicAccessCidrs; this rule only
|
||||
// governs the in-VPC LB→apiserver hop. Idempotent: the
|
||||
// InvalidPermission.Duplicate error a re-run triggers is treated as success.
|
||||
func EnsureControlPlaneIngress(sgp sgProvisioner, accountID, cpSGID, vpcCIDR string) error {
|
||||
if cpSGID == "" {
|
||||
return errors.New("eks: EnsureControlPlaneIngress empty control-plane SG id")
|
||||
}
|
||||
if vpcCIDR == "" {
|
||||
return errors.New("eks: EnsureControlPlaneIngress empty vpc cidr")
|
||||
}
|
||||
|
||||
perm := &ec2.IpPermission{
|
||||
IpProtocol: aws.String("tcp"),
|
||||
FromPort: aws.Int64(k3sAPIServerPort),
|
||||
ToPort: aws.Int64(k3sAPIServerPort),
|
||||
IpRanges: []*ec2.IpRange{{
|
||||
CidrIp: aws.String(vpcCIDR),
|
||||
Description: aws.String("EKS NLB to apiserver"),
|
||||
}},
|
||||
}
|
||||
_, err := sgp.AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{
|
||||
GroupId: aws.String(cpSGID),
|
||||
IpPermissions: []*ec2.IpPermission{perm},
|
||||
}, accountID)
|
||||
if err != nil && !awserrors.IsErrorCode(err, awserrors.ErrorInvalidPermissionDuplicate) {
|
||||
return fmt.Errorf("authorize control-plane apiserver ingress on %s: %w", cpSGID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func lookupSGByName(sgp sgProvisioner, accountID, vpcID, name string) (string, error) {
|
||||
out, err := sgp.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{
|
||||
Filters: []*ec2.Filter{
|
||||
|
||||
@@ -6,15 +6,17 @@ import (
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/spinifex/spinifex/awserrors"
|
||||
"github.com/mulgadc/spinifex/spinifex/tags"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type fakeSGProvisioner struct {
|
||||
createCalls []*ec2.CreateSecurityGroupInput
|
||||
describeCalls []*ec2.DescribeSecurityGroupsInput
|
||||
deleteCalls []*ec2.DeleteSecurityGroupInput
|
||||
createCalls []*ec2.CreateSecurityGroupInput
|
||||
describeCalls []*ec2.DescribeSecurityGroupsInput
|
||||
deleteCalls []*ec2.DeleteSecurityGroupInput
|
||||
authorizeCalls []*ec2.AuthorizeSecurityGroupIngressInput
|
||||
|
||||
// existing maps "name|vpcId" → groupId for DescribeSecurityGroups lookup.
|
||||
existing map[string]string
|
||||
@@ -24,9 +26,10 @@ type fakeSGProvisioner struct {
|
||||
// nodegroup SG IDs.
|
||||
createIDs []string
|
||||
|
||||
createErr error
|
||||
describeErr error
|
||||
deleteErr error
|
||||
createErr error
|
||||
describeErr error
|
||||
deleteErr error
|
||||
authorizeErr error
|
||||
}
|
||||
|
||||
var _ sgProvisioner = (*fakeSGProvisioner)(nil)
|
||||
@@ -84,6 +87,14 @@ func (f *fakeSGProvisioner) DeleteSecurityGroup(input *ec2.DeleteSecurityGroupIn
|
||||
return &ec2.DeleteSecurityGroupOutput{}, nil
|
||||
}
|
||||
|
||||
func (f *fakeSGProvisioner) AuthorizeSecurityGroupIngress(input *ec2.AuthorizeSecurityGroupIngressInput, _ string) (*ec2.AuthorizeSecurityGroupIngressOutput, error) {
|
||||
f.authorizeCalls = append(f.authorizeCalls, input)
|
||||
if f.authorizeErr != nil {
|
||||
return nil, f.authorizeErr
|
||||
}
|
||||
return &ec2.AuthorizeSecurityGroupIngressOutput{}, nil
|
||||
}
|
||||
|
||||
func TestEnsureClusterSGs_EmptyInputsRejected(t *testing.T) {
|
||||
sgp := newFakeSGProvisioner()
|
||||
|
||||
@@ -180,6 +191,49 @@ func TestDeleteClusterSGs_FirstErrorSurfacedSweepContinues(t *testing.T) {
|
||||
assert.Len(t, sgp.deleteCalls, 2, "both delete calls should be attempted despite first error")
|
||||
}
|
||||
|
||||
func TestEnsureControlPlaneIngress_AuthorizesAPIServerFromVPCCIDR(t *testing.T) {
|
||||
sgp := newFakeSGProvisioner()
|
||||
|
||||
err := EnsureControlPlaneIngress(sgp, "111122223333", "sg-cp-001", "10.0.0.0/16")
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Len(t, sgp.authorizeCalls, 1)
|
||||
in := sgp.authorizeCalls[0]
|
||||
assert.Equal(t, "sg-cp-001", aws.StringValue(in.GroupId))
|
||||
require.Len(t, in.IpPermissions, 1)
|
||||
perm := in.IpPermissions[0]
|
||||
assert.Equal(t, "tcp", aws.StringValue(perm.IpProtocol))
|
||||
assert.Equal(t, k3sAPIServerPort, aws.Int64Value(perm.FromPort))
|
||||
assert.Equal(t, k3sAPIServerPort, aws.Int64Value(perm.ToPort))
|
||||
require.Len(t, perm.IpRanges, 1)
|
||||
assert.Equal(t, "10.0.0.0/16", aws.StringValue(perm.IpRanges[0].CidrIp))
|
||||
}
|
||||
|
||||
func TestEnsureControlPlaneIngress_EmptyInputsRejected(t *testing.T) {
|
||||
sgp := newFakeSGProvisioner()
|
||||
|
||||
require.Error(t, EnsureControlPlaneIngress(sgp, "111122223333", "", "10.0.0.0/16"))
|
||||
require.Error(t, EnsureControlPlaneIngress(sgp, "111122223333", "sg-cp-001", ""))
|
||||
assert.Empty(t, sgp.authorizeCalls)
|
||||
}
|
||||
|
||||
func TestEnsureControlPlaneIngress_DuplicateTolerated(t *testing.T) {
|
||||
sgp := newFakeSGProvisioner()
|
||||
sgp.authorizeErr = errors.New(awserrors.ErrorInvalidPermissionDuplicate)
|
||||
|
||||
err := EnsureControlPlaneIngress(sgp, "111122223333", "sg-cp-001", "10.0.0.0/16")
|
||||
require.NoError(t, err, "a duplicate rule on re-run must be treated as success")
|
||||
}
|
||||
|
||||
func TestEnsureControlPlaneIngress_AuthorizeErrorSurfaced(t *testing.T) {
|
||||
sgp := newFakeSGProvisioner()
|
||||
sgp.authorizeErr = errors.New("vpcd unavailable")
|
||||
|
||||
err := EnsureControlPlaneIngress(sgp, "111122223333", "sg-cp-001", "10.0.0.0/16")
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "sg-cp-001")
|
||||
}
|
||||
|
||||
func assertSGCreateTagged(t *testing.T, in *ec2.CreateSecurityGroupInput, name, vpcID, clusterName, role string) {
|
||||
t.Helper()
|
||||
require.NotNil(t, in)
|
||||
|
||||
@@ -22,11 +22,13 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// SubnetVPCResolver resolves a subnet ID to its parent VPC ID. Narrow so
|
||||
// EKSServiceImpl can stay free of the wider handlers/ec2/vpc surface; the
|
||||
// daemon adapts VPCServiceImpl.GetSubnet onto this contract.
|
||||
// SubnetVPCResolver resolves a subnet ID to its parent VPC ID and the VPC's
|
||||
// CIDR block. Narrow so EKSServiceImpl can stay free of the wider
|
||||
// handlers/ec2/vpc surface; the daemon adapts VPCServiceImpl onto this
|
||||
// contract.
|
||||
type SubnetVPCResolver interface {
|
||||
GetSubnetVPC(accountID, subnetID string) (vpcID string, err error)
|
||||
GetVPCCIDR(accountID, vpcID string) (cidr string, err error)
|
||||
}
|
||||
|
||||
// EKSServiceDeps wires every external collaborator EKSServiceImpl needs.
|
||||
@@ -41,12 +43,17 @@ type EKSServiceDeps struct {
|
||||
Region string
|
||||
HolderID string
|
||||
|
||||
// NATS auth handed to the K3s server VM so it can publish its one-shot
|
||||
// bootstrap messages. Shared token + CA (PEM content) + a VM-reachable
|
||||
// URL — spinifex has no per-principal nkeys hierarchy to scope against.
|
||||
NATSURL string
|
||||
NATSToken string
|
||||
NATSCACert string
|
||||
// Gateway broker config handed to the K3s server VM so it can publish its
|
||||
// bootstrap envelopes + state reports via SigV4-signed HTTPS POST to the AWS
|
||||
// gateway (the ELBv2 lb-agent model) instead of dialing core NATS.
|
||||
// SystemGatewayURL is the mgmt-reachable AWSGW endpoint (distinct from
|
||||
// GatewayBaseURL, which is the OIDC issuer host); SystemAccessKey/SecretKey
|
||||
// are the system (Predastore) SigV4 creds; GatewayCACert is the PEM that
|
||||
// signs the gateway server cert.
|
||||
SystemGatewayURL string
|
||||
SystemAccessKey string
|
||||
SystemSecretKey string
|
||||
GatewayCACert string
|
||||
|
||||
VPCSG sgProvisioner
|
||||
VPCK3s k3sVPCProvisioner
|
||||
@@ -55,6 +62,19 @@ type EKSServiceDeps struct {
|
||||
Instance k3sInstanceLauncher
|
||||
Image k3sAMIResolver
|
||||
EIP eipProvisioner
|
||||
Worker WorkerLauncher
|
||||
}
|
||||
|
||||
// WorkerLauncher is the narrow EC2 surface CreateNodegroup needs to launch and
|
||||
// terminate nodegroup worker instances. Workers go through the normal
|
||||
// customer-owned RunInstances path (no ManagedBy tag, no mgmt NIC) so they are
|
||||
// visible in DescribeInstances and reclaimed by TerminateInstances like any
|
||||
// other customer EC2. The daemon adapts its concrete instance service onto this.
|
||||
// Exported so the daemon can assert *Daemon satisfies it with a compile-time
|
||||
// check.
|
||||
type WorkerLauncher interface {
|
||||
RunWorkerInstance(input *ec2.RunInstancesInput, accountID string) (*ec2.Reservation, error)
|
||||
TerminateWorkerInstances(instanceIDs []string, accountID string) error
|
||||
}
|
||||
|
||||
// eipProvisioner is the narrow EIP surface CreateCluster needs to give the
|
||||
@@ -226,6 +246,9 @@ func (s *EKSServiceImpl) missingOrchestrationDeps() []string {
|
||||
if s.deps.EIP == nil {
|
||||
missing = append(missing, "EIP")
|
||||
}
|
||||
if s.deps.Worker == nil {
|
||||
missing = append(missing, "Worker")
|
||||
}
|
||||
if len(s.deps.MasterKey) == 0 {
|
||||
missing = append(missing, "MasterKey")
|
||||
}
|
||||
@@ -243,6 +266,28 @@ func (s *EKSServiceImpl) missingOrchestrationDeps() []string {
|
||||
|
||||
// --- Cluster ---
|
||||
|
||||
// logCreateErr records the precise cause of a CreateCluster step failure at
|
||||
// ERROR. The request surface collapses these wrapped errors to an opaque
|
||||
// ServerInternal ("An internal error has occurred... contact AWS re:Post"), so
|
||||
// without this the wrapped error never reaches the operator. Returns the error
|
||||
// wrapped with the stage for the caller to surface.
|
||||
func logCreateErr(name, accountID, stage string, err error) error {
|
||||
slog.Error("CreateCluster: "+stage+" failed", "cluster", name, "accountID", accountID, "err", err)
|
||||
return fmt.Errorf("%s: %w", stage, err)
|
||||
}
|
||||
|
||||
// managedIngressTagKey is the CreateCluster tag that opts a cluster into K3s'
|
||||
// bundled traefik + servicelb (dev / interim in-VPC app exposure). Value "true"
|
||||
// (case-insensitive) enables it; absent or any other value keeps AWS parity
|
||||
// (built-ins disabled, ingress via the AWS Load Balancer Controller).
|
||||
const managedIngressTagKey = "spinifex.io/managed-ingress"
|
||||
|
||||
// builtinIngressEnabled reports whether the CreateCluster tags opt into the K3s
|
||||
// built-in ingress stack.
|
||||
func builtinIngressEnabled(tags map[string]*string) bool {
|
||||
return strings.EqualFold(aws.StringValue(tags[managedIngressTagKey]), "true")
|
||||
}
|
||||
|
||||
func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID, callerPrincipalARN string) (*eks.CreateClusterOutput, error) {
|
||||
if err := s.requireOrchestrationDeps("CreateCluster"); err != nil {
|
||||
return nil, err
|
||||
@@ -255,11 +300,11 @@ func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID,
|
||||
|
||||
js, err := s.deps.NATSConn.JetStream()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("jetstream: %w", err)
|
||||
return nil, logCreateErr(name, accountID, "jetstream", err)
|
||||
}
|
||||
acctKV, err := GetOrCreateAccountBucket(js, accountID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get account bucket: %w", err)
|
||||
return nil, logCreateErr(name, accountID, "get account bucket", err)
|
||||
}
|
||||
|
||||
if existing, err := GetClusterMeta(acctKV, name); err == nil {
|
||||
@@ -277,20 +322,28 @@ func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID,
|
||||
slog.Info("CreateCluster: reclaiming FAILED cluster before recreate",
|
||||
"name", name, "accountID", accountID)
|
||||
if perr := s.purgeClusterInfra(accountID, name, existing, acctKV); perr != nil {
|
||||
return nil, fmt.Errorf("reclaim failed cluster %s: %w", name, perr)
|
||||
return nil, logCreateErr(name, accountID, "reclaim failed cluster", perr)
|
||||
}
|
||||
} else if !errors.Is(err, ErrClusterNotFound) {
|
||||
return nil, fmt.Errorf("preflight get meta: %w", err)
|
||||
return nil, logCreateErr(name, accountID, "preflight get meta", err)
|
||||
}
|
||||
|
||||
vpcID, err := s.deps.VPCSubnet.GetSubnetVPC(accountID, subnetIDs[0])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve subnet VPC: %w", err)
|
||||
// Subnet is a caller-supplied parameter — a resolve failure is a client
|
||||
// fault (bad/foreign subnet), not an internal one. Surface it as such
|
||||
// instead of collapsing to a retryable ServerInternal the SDK retries.
|
||||
slog.Error("CreateCluster: resolve subnet VPC failed",
|
||||
"cluster", name, "accountID", accountID, "subnet", subnetIDs[0], "err", err)
|
||||
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
|
||||
}
|
||||
|
||||
region := s.deps.Region
|
||||
arn := fmt.Sprintf("arn:aws:eks:%s:%s:cluster/%s", region, accountID, name)
|
||||
|
||||
publicAccess, privateAccess := endpointAccess(input.ResourcesVpcConfig)
|
||||
publicCidrs := publicAccessCidrs(input.ResourcesVpcConfig, publicAccess)
|
||||
|
||||
meta := &ClusterMeta{
|
||||
Name: name,
|
||||
Arn: arn,
|
||||
@@ -298,28 +351,55 @@ func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID,
|
||||
Version: deref(input.Version, defaultK8sVersion),
|
||||
RoleArn: aws.StringValue(input.RoleArn),
|
||||
ResourcesVpcConfig: &ClusterVpcConfig{
|
||||
SubnetIds: subnetIDs,
|
||||
VpcId: vpcID,
|
||||
SubnetIds: subnetIDs,
|
||||
VpcId: vpcID,
|
||||
EndpointPublicAccess: publicAccess,
|
||||
EndpointPrivateAccess: privateAccess,
|
||||
PublicAccessCidrs: publicCidrs,
|
||||
},
|
||||
CreatedAt: time.Now().UTC(),
|
||||
BuiltinIngress: builtinIngressEnabled(input.Tags),
|
||||
CreatedAt: time.Now().UTC(),
|
||||
}
|
||||
if err := PutClusterMeta(acctKV, meta); err != nil {
|
||||
return nil, err
|
||||
return nil, logCreateErr(name, accountID, "persist initial meta", err)
|
||||
}
|
||||
|
||||
cpSG, ngSG, err := EnsureClusterSGs(s.deps.VPCSG, accountID, name, vpcID)
|
||||
if err != nil {
|
||||
s.markFailed(acctKV, name)
|
||||
return nil, fmt.Errorf("ensure cluster SGs: %w", err)
|
||||
return nil, logCreateErr(name, accountID, "ensure cluster SGs", err)
|
||||
}
|
||||
meta.ResourcesVpcConfig.SecurityGroupIds = []string{cpSG, ngSG}
|
||||
|
||||
nlb, err := EnsureClusterNLB(s.deps.NLB, accountID, name, subnetIDs)
|
||||
// The NLB's backing LB VM forwards the published endpoint to the apiserver
|
||||
// from inside the VPC, so the control-plane SG must admit that hop or the
|
||||
// NLB target stays unhealthy and the endpoint never serves.
|
||||
vpcCIDR, err := s.deps.VPCSubnet.GetVPCCIDR(accountID, vpcID)
|
||||
if err != nil {
|
||||
s.markFailed(acctKV, name)
|
||||
return nil, fmt.Errorf("ensure cluster NLB: %w", err)
|
||||
return nil, logCreateErr(name, accountID, "resolve vpc cidr", err)
|
||||
}
|
||||
meta.Endpoint = "https://" + net.JoinHostPort(nlb.DNSName, strconv.FormatInt(clusterNLBListenPort, 10))
|
||||
if err := EnsureControlPlaneIngress(s.deps.VPCSG, accountID, cpSG, vpcCIDR); err != nil {
|
||||
s.markFailed(acctKV, name)
|
||||
return nil, logCreateErr(name, accountID, "ensure control-plane ingress", err)
|
||||
}
|
||||
|
||||
// Public access ⇒ internet-facing NLB (external-pool front-end IP, reachable
|
||||
// on the LAN/edge network); private-only ⇒ internal NLB (VPC-only).
|
||||
nlb, err := EnsureClusterNLB(s.deps.NLB, accountID, name, subnetIDs, publicAccess, publicCidrs)
|
||||
if err != nil {
|
||||
s.markFailed(acctKV, name)
|
||||
return nil, logCreateErr(name, accountID, "ensure cluster NLB", err)
|
||||
}
|
||||
// Prefer the reachable front-end IP as the kubeconfig endpoint host so the
|
||||
// apiserver serving cert (which SANs this IP) validates with TLS verification
|
||||
// on. Fall back to the synthetic NLB DNS name when no IP was read back.
|
||||
endpointHost := nlb.FrontendIP
|
||||
if endpointHost == "" {
|
||||
endpointHost = nlb.DNSName
|
||||
}
|
||||
meta.Endpoint = "https://" + net.JoinHostPort(endpointHost, strconv.FormatInt(clusterNLBListenPort, 10))
|
||||
meta.EndpointIP = nlb.FrontendIP
|
||||
meta.NLBArn = nlb.LoadBalancerArn
|
||||
meta.NLBTargetGroupArn = nlb.TargetGroupArn
|
||||
|
||||
@@ -330,25 +410,25 @@ func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID,
|
||||
// reclaim them.
|
||||
if err := PutClusterMeta(acctKV, meta); err != nil {
|
||||
s.markFailed(acctKV, name)
|
||||
return nil, fmt.Errorf("persist NLB arns: %w", err)
|
||||
return nil, logCreateErr(name, accountID, "persist NLB arns", err)
|
||||
}
|
||||
|
||||
oidcIssuer, err := ClusterOIDCIssuer(s.deps.GatewayBaseURL, region, accountID, name)
|
||||
if err != nil {
|
||||
s.markFailed(acctKV, name)
|
||||
return nil, fmt.Errorf("build OIDC issuer: %w", err)
|
||||
return nil, logCreateErr(name, accountID, "build OIDC issuer", err)
|
||||
}
|
||||
meta.OIDCIssuer = oidcIssuer
|
||||
|
||||
privPEM, _, err := GenerateClusterOIDCKeypair(acctKV, name, s.deps.MasterKey)
|
||||
if err != nil {
|
||||
s.markFailed(acctKV, name)
|
||||
return nil, fmt.Errorf("generate OIDC keypair: %w", err)
|
||||
return nil, logCreateErr(name, accountID, "generate OIDC keypair", err)
|
||||
}
|
||||
pubPEM, err := PublicKeyPEMFromPrivate(privPEM)
|
||||
if err != nil {
|
||||
s.markFailed(acctKV, name)
|
||||
return nil, fmt.Errorf("derive OIDC public key: %w", err)
|
||||
return nil, logCreateErr(name, accountID, "derive OIDC public key", err)
|
||||
}
|
||||
|
||||
k3sOut, err := LaunchK3sServerVM(s.deps.VPCK3s, s.deps.Instance, s.deps.Image, K3sServerInput{
|
||||
@@ -358,12 +438,15 @@ func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID,
|
||||
SubnetID: subnetIDs[0],
|
||||
ControlPlaneSGID: cpSG,
|
||||
NLBDNS: nlb.DNSName,
|
||||
EndpointIP: nlb.FrontendIP,
|
||||
OIDCIssuer: oidcIssuer,
|
||||
OIDCPrivateKeyPEM: privPEM,
|
||||
OIDCPublicKeyPEM: pubPEM,
|
||||
NATSURL: s.deps.NATSURL,
|
||||
NATSToken: s.deps.NATSToken,
|
||||
NATSCACert: s.deps.NATSCACert,
|
||||
GatewayURL: s.deps.SystemGatewayURL,
|
||||
AccessKey: s.deps.SystemAccessKey,
|
||||
SecretKey: s.deps.SystemSecretKey,
|
||||
GatewayCACert: s.deps.GatewayCACert,
|
||||
BuiltinIngress: meta.BuiltinIngress,
|
||||
})
|
||||
if err != nil {
|
||||
s.markFailed(acctKV, name)
|
||||
@@ -387,7 +470,7 @@ func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID,
|
||||
egressIP, egressAllocID, err := s.allocateClusterEgressIP(accountID, name)
|
||||
if err != nil {
|
||||
s.markFailed(acctKV, name)
|
||||
return nil, fmt.Errorf("allocate cluster egress IP: %w", err)
|
||||
return nil, logCreateErr(name, accountID, "allocate cluster egress IP", err)
|
||||
}
|
||||
meta.EgressEIPAllocationID = egressAllocID
|
||||
meta.EgressEIPPublicIP = egressIP
|
||||
@@ -402,16 +485,17 @@ func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID,
|
||||
// register (or anything after) leaves them recoverable by DeleteCluster.
|
||||
if err := PutClusterMeta(acctKV, meta); err != nil {
|
||||
s.markFailed(acctKV, name)
|
||||
return nil, fmt.Errorf("persist control-plane ids: %w", err)
|
||||
return nil, logCreateErr(name, accountID, "persist control-plane ids", err)
|
||||
}
|
||||
|
||||
if err := RegisterClusterTarget(s.deps.NLB, accountID, nlb.TargetGroupArn, k3sOut.ENIIP); err != nil {
|
||||
s.markFailed(acctKV, name)
|
||||
return nil, fmt.Errorf("register NLB target: %w", err)
|
||||
return nil, logCreateErr(name, accountID, "register NLB target", err)
|
||||
}
|
||||
|
||||
if err := PutClusterMeta(acctKV, meta); err != nil {
|
||||
return nil, err
|
||||
s.markFailed(acctKV, name)
|
||||
return nil, logCreateErr(name, accountID, "persist final meta", err)
|
||||
}
|
||||
|
||||
// bootstrapClusterCreatorAdminPermissions (default true): grant the caller
|
||||
@@ -501,15 +585,15 @@ func (s *EKSServiceImpl) DescribeCluster(input *eks.DescribeClusterInput, accoun
|
||||
func (s *EKSServiceImpl) ListClusters(input *eks.ListClustersInput, accountID string) (*eks.ListClustersOutput, error) {
|
||||
js, err := s.deps.NATSConn.JetStream()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("jetstream: %w", err)
|
||||
return nil, eksReadUnavailableOr(err, "jetstream")
|
||||
}
|
||||
acctKV, err := GetOrCreateAccountBucket(js, accountID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get account bucket: %w", err)
|
||||
return nil, eksReadUnavailableOr(err, "get account bucket")
|
||||
}
|
||||
names, err := listClusterNames(acctKV)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, eksReadUnavailableOr(err, "list cluster names")
|
||||
}
|
||||
sort.Strings(names)
|
||||
|
||||
@@ -666,28 +750,63 @@ func (s *EKSServiceImpl) UpdateClusterVersion(_ *eks.UpdateClusterVersionInput,
|
||||
|
||||
// --- Nodegroup ---
|
||||
|
||||
func (s *EKSServiceImpl) CreateNodegroup(_ *eks.CreateNodegroupInput, _ string) (*eks.CreateNodegroupOutput, error) {
|
||||
return nil, notImpl()
|
||||
func (s *EKSServiceImpl) CreateNodegroup(input *eks.CreateNodegroupInput, accountID string) (*eks.CreateNodegroupOutput, error) {
|
||||
if err := s.requireOrchestrationDeps("CreateNodegroup"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
acctKV, err := s.nodegroupAcctKV(accountID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.createNodegroup(acctKV, input, accountID)
|
||||
}
|
||||
|
||||
func (s *EKSServiceImpl) DescribeNodegroup(_ *eks.DescribeNodegroupInput, _ string) (*eks.DescribeNodegroupOutput, error) {
|
||||
return nil, notImpl()
|
||||
func (s *EKSServiceImpl) DescribeNodegroup(input *eks.DescribeNodegroupInput, accountID string) (*eks.DescribeNodegroupOutput, error) {
|
||||
if input == nil {
|
||||
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
|
||||
}
|
||||
acctKV, err := s.nodegroupAcctKV(accountID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.describeNodegroup(acctKV, input)
|
||||
}
|
||||
|
||||
func (s *EKSServiceImpl) ListNodegroups(_ *eks.ListNodegroupsInput, _ string) (*eks.ListNodegroupsOutput, error) {
|
||||
return nil, notImpl()
|
||||
func (s *EKSServiceImpl) ListNodegroups(input *eks.ListNodegroupsInput, accountID string) (*eks.ListNodegroupsOutput, error) {
|
||||
if input == nil {
|
||||
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
|
||||
}
|
||||
acctKV, err := s.nodegroupAcctKV(accountID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.listNodegroups(acctKV, input)
|
||||
}
|
||||
|
||||
func (s *EKSServiceImpl) UpdateNodegroupConfig(_ *eks.UpdateNodegroupConfigInput, _ string) (*eks.UpdateNodegroupConfigOutput, error) {
|
||||
return nil, notImpl()
|
||||
func (s *EKSServiceImpl) UpdateNodegroupConfig(input *eks.UpdateNodegroupConfigInput, accountID string) (*eks.UpdateNodegroupConfigOutput, error) {
|
||||
if err := s.requireOrchestrationDeps("UpdateNodegroupConfig"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
acctKV, err := s.nodegroupAcctKV(accountID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.updateNodegroupConfig(acctKV, input, accountID)
|
||||
}
|
||||
|
||||
func (s *EKSServiceImpl) UpdateNodegroupVersion(_ *eks.UpdateNodegroupVersionInput, _ string) (*eks.UpdateNodegroupVersionOutput, error) {
|
||||
return nil, notImpl()
|
||||
func (s *EKSServiceImpl) UpdateNodegroupVersion(input *eks.UpdateNodegroupVersionInput, accountID string) (*eks.UpdateNodegroupVersionOutput, error) {
|
||||
return s.updateNodegroupVersion(input)
|
||||
}
|
||||
|
||||
func (s *EKSServiceImpl) DeleteNodegroup(_ *eks.DeleteNodegroupInput, _ string) (*eks.DeleteNodegroupOutput, error) {
|
||||
return nil, notImpl()
|
||||
func (s *EKSServiceImpl) DeleteNodegroup(input *eks.DeleteNodegroupInput, accountID string) (*eks.DeleteNodegroupOutput, error) {
|
||||
if err := s.requireOrchestrationDeps("DeleteNodegroup"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
acctKV, err := s.nodegroupAcctKV(accountID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.deleteNodegroup(acctKV, input, accountID)
|
||||
}
|
||||
|
||||
// --- AccessEntry + AccessPolicy ---
|
||||
@@ -1069,9 +1188,49 @@ func validateCreateClusterInput(input *eks.CreateClusterInput) error {
|
||||
return errors.New(awserrors.ErrorInvalidParameter)
|
||||
}
|
||||
}
|
||||
// AWS rejects disabling both public and private endpoint access — the
|
||||
// control plane would be unreachable.
|
||||
if v := input.ResourcesVpcConfig; v != nil &&
|
||||
v.EndpointPublicAccess != nil && !*v.EndpointPublicAccess &&
|
||||
v.EndpointPrivateAccess != nil && !*v.EndpointPrivateAccess {
|
||||
return errors.New(awserrors.ErrorInvalidParameterValue)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// defaultPublicAccessCidr is the AWS-default allowed source range for a public
|
||||
// cluster endpoint when the caller specifies none.
|
||||
const defaultPublicAccessCidr = "0.0.0.0/0"
|
||||
|
||||
// endpointAccess resolves apiserver endpoint exposure from the request,
|
||||
// applying AWS defaults (public on, private off) to omitted fields.
|
||||
func endpointAccess(vpc *eks.VpcConfigRequest) (public, private bool) {
|
||||
public, private = true, false
|
||||
if vpc == nil {
|
||||
return public, private
|
||||
}
|
||||
if vpc.EndpointPublicAccess != nil {
|
||||
public = *vpc.EndpointPublicAccess
|
||||
}
|
||||
if vpc.EndpointPrivateAccess != nil {
|
||||
private = *vpc.EndpointPrivateAccess
|
||||
}
|
||||
return public, private
|
||||
}
|
||||
|
||||
// publicAccessCidrs returns the allowed source ranges for a public endpoint,
|
||||
// defaulting to 0.0.0.0/0 (AWS parity) when public access is on and none given.
|
||||
// Nil for a private-only endpoint.
|
||||
func publicAccessCidrs(vpc *eks.VpcConfigRequest, public bool) []string {
|
||||
if !public {
|
||||
return nil
|
||||
}
|
||||
if vpc != nil && len(vpc.PublicAccessCidrs) > 0 {
|
||||
return aws.StringValueSlice(vpc.PublicAccessCidrs)
|
||||
}
|
||||
return []string{defaultPublicAccessCidr}
|
||||
}
|
||||
|
||||
func (s *EKSServiceImpl) markFailed(kv nats.KeyValue, name string) {
|
||||
if err := SetClusterStatus(kv, name, ClusterStatusFailed); err != nil {
|
||||
slog.Warn("CreateCluster: SetClusterStatus(FAILED) failed", "cluster", name, "err", err)
|
||||
@@ -1104,21 +1263,7 @@ func (s *EKSServiceImpl) spawnBootstrap(accountID, clusterName string, kv nats.K
|
||||
}()
|
||||
}
|
||||
|
||||
func (s *EKSServiceImpl) spawnReconciler(accountID, clusterName string, meta *ClusterMeta) {
|
||||
healthURL := ""
|
||||
switch {
|
||||
case meta.ControlPlaneMgmtIP != "":
|
||||
// Workaround until authoritative DNS (Eclipso/Route53) lands: the
|
||||
// reconciler runs on the host, outside the VPC overlay, and can neither
|
||||
// resolve the NLB DNS name nor route to the VPC-internal NLB front-end.
|
||||
// The control-plane VM's br-mgmt NIC is the one apiserver address
|
||||
// reachable from the daemon, so probe :6443/healthz there directly. The
|
||||
// probe client sets InsecureSkipVerify, so the mgmt IP need not be in the
|
||||
// apiserver cert SAN.
|
||||
healthURL = "https://" + net.JoinHostPort(meta.ControlPlaneMgmtIP, "6443") + "/healthz"
|
||||
case meta.Endpoint != "":
|
||||
healthURL = strings.TrimRight(meta.Endpoint, "/") + "/healthz"
|
||||
}
|
||||
func (s *EKSServiceImpl) spawnReconciler(accountID, clusterName string, _ *ClusterMeta) {
|
||||
js, err := s.deps.NATSConn.JetStream()
|
||||
if err != nil {
|
||||
slog.Error("spawnReconciler: jetstream", "err", err)
|
||||
@@ -1129,8 +1274,13 @@ func (s *EKSServiceImpl) spawnReconciler(accountID, clusterName string, meta *Cl
|
||||
slog.Error("spawnReconciler: account bucket", "err", err)
|
||||
return
|
||||
}
|
||||
// Health gates on the control plane's NATS self-report, not an HTTP probe:
|
||||
// k3s binds the apiserver to the VPC node-ip, unreachable from the host. The
|
||||
// CP publishes {healthz,node_count} on the mgmt bus the daemon already shares.
|
||||
stateSubject := StateSubject(accountID, clusterName)
|
||||
spawn := func(ctx context.Context, _, _ string) (func(), error) {
|
||||
return RunClusterReconciler(ctx, s.leaderKV, acctKV, accountID, clusterName, s.deps.HolderID, healthURL)
|
||||
return RunClusterReconciler(ctx, s.leaderKV, acctKV, accountID, clusterName, s.deps.HolderID, "",
|
||||
WithStateSource(s.deps.NATSConn, stateSubject))
|
||||
}
|
||||
if err := s.registry.Spawn(s.bgCtx, accountID, clusterName, spawn); err != nil {
|
||||
slog.Error("spawnReconciler: registry spawn", "cluster", clusterName, "err", err)
|
||||
@@ -1160,9 +1310,12 @@ func clusterMetaToAWS(meta *ClusterMeta) *eks.Cluster {
|
||||
}
|
||||
if meta.ResourcesVpcConfig != nil {
|
||||
out.ResourcesVpcConfig = &eks.VpcConfigResponse{
|
||||
SubnetIds: aws.StringSlice(meta.ResourcesVpcConfig.SubnetIds),
|
||||
SecurityGroupIds: aws.StringSlice(meta.ResourcesVpcConfig.SecurityGroupIds),
|
||||
VpcId: aws.String(meta.ResourcesVpcConfig.VpcId),
|
||||
SubnetIds: aws.StringSlice(meta.ResourcesVpcConfig.SubnetIds),
|
||||
SecurityGroupIds: aws.StringSlice(meta.ResourcesVpcConfig.SecurityGroupIds),
|
||||
VpcId: aws.String(meta.ResourcesVpcConfig.VpcId),
|
||||
EndpointPublicAccess: aws.Bool(meta.ResourcesVpcConfig.EndpointPublicAccess),
|
||||
EndpointPrivateAccess: aws.Bool(meta.ResourcesVpcConfig.EndpointPrivateAccess),
|
||||
PublicAccessCidrs: aws.StringSlice(meta.ResourcesVpcConfig.PublicAccessCidrs),
|
||||
}
|
||||
}
|
||||
if meta.HealthIssue != "" {
|
||||
@@ -1176,6 +1329,30 @@ func clusterMetaToAWS(meta *ClusterMeta) *eks.Cluster {
|
||||
return out
|
||||
}
|
||||
|
||||
// natsTransient reports whether err is a transient NATS/JetStream condition
|
||||
// rather than a genuine fault. These occur in the post-restart window before
|
||||
// the KV stream's Raft group has elected a leader: JetStream requests get no
|
||||
// responder, time out, or the stream is briefly unreachable.
|
||||
func natsTransient(err error) bool {
|
||||
return err != nil && (errors.Is(err, nats.ErrNoResponders) ||
|
||||
errors.Is(err, nats.ErrTimeout) ||
|
||||
errors.Is(err, nats.ErrNoStreamResponse) ||
|
||||
errors.Is(err, nats.ErrConnectionClosed))
|
||||
}
|
||||
|
||||
// eksReadUnavailableOr maps a transient NATS/JetStream failure to a retryable
|
||||
// ServiceUnavailable (503) so AWS clients back off and retry, instead of the
|
||||
// daemon sanitizing an unrecognised wrapped error to a misleading 500
|
||||
// InternalError. Non-transient errors pass through wrapped so real faults stay
|
||||
// visible.
|
||||
func eksReadUnavailableOr(err error, op string) error {
|
||||
if natsTransient(err) {
|
||||
slog.Warn("EKS read temporarily unavailable", "op", op, "err", err)
|
||||
return errors.New(awserrors.ErrorServiceUnavailable)
|
||||
}
|
||||
return fmt.Errorf("%s: %w", op, err)
|
||||
}
|
||||
|
||||
func listClusterNames(kv nats.KeyValue) ([]string, error) {
|
||||
keys, err := kv.Keys()
|
||||
if err != nil {
|
||||
|
||||
@@ -64,6 +64,17 @@ func TestValidateCreateClusterInput_RejectsConfigMapAuthMode(t *testing.T) {
|
||||
require.EqualError(t, err, awserrors.ErrorInvalidParameter)
|
||||
}
|
||||
|
||||
// The API_AND_CONFIG_MAP hybrid still enables the unsupported aws-auth ConfigMap
|
||||
// path, so it must be rejected the same as plain CONFIG_MAP — the sibling test
|
||||
// only covers the CONFIG_MAP value.
|
||||
func TestValidateCreateClusterInput_RejectsAPIAndConfigMapAuthMode(t *testing.T) {
|
||||
in := createInput("alpha")
|
||||
in.AccessConfig = &eks.CreateAccessConfigRequest{
|
||||
AuthenticationMode: aws.String(eks.AuthenticationModeApiAndConfigMap),
|
||||
}
|
||||
require.EqualError(t, validateCreateClusterInput(in), awserrors.ErrorInvalidParameter)
|
||||
}
|
||||
|
||||
func TestValidateCreateClusterInput_AcceptsAPIAuthMode(t *testing.T) {
|
||||
in := createInput("alpha")
|
||||
in.AccessConfig = &eks.CreateAccessConfigRequest{
|
||||
@@ -92,6 +103,16 @@ func TestDescribeCluster_NotFoundWithFullDeps(t *testing.T) {
|
||||
require.EqualError(t, err, awserrors.ErrorEKSResourceNotFound)
|
||||
}
|
||||
|
||||
// DeleteCluster on an absent cluster must reach the KV lookup with full deps
|
||||
// wired (the shim path short-circuits to ServiceUnavailable before the meta
|
||||
// read) and surface ResourceNotFoundException, not a teardown of nothing.
|
||||
func TestDeleteCluster_NotFoundWithFullDeps(t *testing.T) {
|
||||
f := newEKSServiceFixture(t)
|
||||
|
||||
_, err := f.svc.DeleteCluster(deleteInput("ghost"), testAccountID)
|
||||
require.EqualError(t, err, awserrors.ErrorEKSResourceNotFound)
|
||||
}
|
||||
|
||||
// Security guarantee: the OIDC signing key must be zeroized BEFORE infra
|
||||
// teardown, so a teardown failure (which leaves the meta for retry) can never
|
||||
// leave recoverable key material behind. Force VM terminate to fail and assert
|
||||
@@ -111,26 +132,30 @@ func TestDeleteCluster_ZeroizesOIDCKeyBeforeTeardown(t *testing.T) {
|
||||
assert.Equal(t, ClusterStatusDeleting, meta.Status)
|
||||
}
|
||||
|
||||
func TestEKSServiceImpl_NodegroupMethodsReturnNotImplemented(t *testing.T) {
|
||||
// In shim mode (orchestration deps absent) the mutating nodegroup methods
|
||||
// short-circuit to ServiceUnavailable, the read methods reach an empty
|
||||
// per-account bucket and surface ResourceNotFoundException, and
|
||||
// UpdateNodegroupVersion stays NotImplemented (v1 doesn't do AMI upgrades).
|
||||
func TestEKSServiceImpl_NodegroupMethodsShimMode(t *testing.T) {
|
||||
svc := setupTestService(t)
|
||||
|
||||
_, err := svc.CreateNodegroup(&eks.CreateNodegroupInput{ClusterName: aws.String("c1"), NodegroupName: aws.String("ng1")}, testAccountID)
|
||||
require.EqualError(t, err, awserrors.ErrorNotImplemented)
|
||||
require.EqualError(t, err, awserrors.ErrorServiceUnavailable)
|
||||
|
||||
_, err = svc.DescribeNodegroup(&eks.DescribeNodegroupInput{ClusterName: aws.String("c1"), NodegroupName: aws.String("ng1")}, testAccountID)
|
||||
require.EqualError(t, err, awserrors.ErrorNotImplemented)
|
||||
require.EqualError(t, err, awserrors.ErrorEKSResourceNotFound)
|
||||
|
||||
_, err = svc.ListNodegroups(&eks.ListNodegroupsInput{ClusterName: aws.String("c1")}, testAccountID)
|
||||
require.EqualError(t, err, awserrors.ErrorNotImplemented)
|
||||
require.EqualError(t, err, awserrors.ErrorEKSResourceNotFound)
|
||||
|
||||
_, err = svc.UpdateNodegroupConfig(&eks.UpdateNodegroupConfigInput{ClusterName: aws.String("c1"), NodegroupName: aws.String("ng1")}, testAccountID)
|
||||
require.EqualError(t, err, awserrors.ErrorNotImplemented)
|
||||
require.EqualError(t, err, awserrors.ErrorServiceUnavailable)
|
||||
|
||||
_, err = svc.UpdateNodegroupVersion(&eks.UpdateNodegroupVersionInput{ClusterName: aws.String("c1"), NodegroupName: aws.String("ng1")}, testAccountID)
|
||||
require.EqualError(t, err, awserrors.ErrorNotImplemented)
|
||||
|
||||
_, err = svc.DeleteNodegroup(&eks.DeleteNodegroupInput{ClusterName: aws.String("c1"), NodegroupName: aws.String("ng1")}, testAccountID)
|
||||
require.EqualError(t, err, awserrors.ErrorNotImplemented)
|
||||
require.EqualError(t, err, awserrors.ErrorServiceUnavailable)
|
||||
}
|
||||
|
||||
// seedTestCluster lays down a minimal ACTIVE cluster meta in the per-account
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
package handlers_eks
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// StateSubject returns the NATS subject the control-plane VM publishes its
|
||||
// periodic health self-report to: "eks.state.{accountID}.{clusterName}.server".
|
||||
// The CP VM is on the management NATS bus, so this report reaches the host-side
|
||||
// reconciler without any route into the VPC overlay — unlike an HTTP /healthz
|
||||
// probe to the apiserver, which k3s binds to the unreachable VPC node-ip.
|
||||
func StateSubject(accountID, clusterName string) string {
|
||||
return fmt.Sprintf("eks.state.%s.%s.server", accountID, clusterName)
|
||||
}
|
||||
|
||||
// ServerStateReport is the JSON payload of a control-plane state report, emitted
|
||||
// by scripts/images/eks-node/mulga-eks-state-report.sh. Healthz mirrors the
|
||||
// apiserver's /healthz body ("ok" when serving); NodeCount is the number of
|
||||
// nodes the apiserver reports (server + joined agents); TS is the publish time
|
||||
// in unix seconds, used to reject stale reports from a CP that stopped emitting.
|
||||
type ServerStateReport struct {
|
||||
Healthz string `json:"healthz"`
|
||||
NodeCount int `json:"node_count"`
|
||||
TS int64 `json:"ts"`
|
||||
}
|
||||
|
||||
// Healthy reports whether the apiserver was serving at publish time.
|
||||
func (s ServerStateReport) Healthy() bool { return s.Healthz == "ok" }
|
||||
|
||||
func unmarshalServerStateReport(data []byte) (ServerStateReport, error) {
|
||||
var r ServerStateReport
|
||||
if err := json.Unmarshal(data, &r); err != nil {
|
||||
return ServerStateReport{}, fmt.Errorf("eks: unmarshal server state report: %w", err)
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
@@ -56,9 +56,15 @@ func ClusterMetaKey(name string) string {
|
||||
return fmt.Sprintf("clusters/%s/meta", name)
|
||||
}
|
||||
|
||||
// NodegroupsPrefix returns the KV key prefix under which all of a cluster's
|
||||
// nodegroup records live. Used by ListNodegroups to enumerate.
|
||||
func NodegroupsPrefix(cluster string) string {
|
||||
return fmt.Sprintf("clusters/%s/nodegroups/", cluster)
|
||||
}
|
||||
|
||||
// NodegroupKey returns the KV key for a nodegroup record under a cluster.
|
||||
func NodegroupKey(cluster, ng string) string {
|
||||
return fmt.Sprintf("clusters/%s/nodegroups/%s", cluster, ng)
|
||||
return NodegroupsPrefix(cluster) + ng
|
||||
}
|
||||
|
||||
// AccessEntriesPrefix returns the KV key prefix under which all of a cluster's
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
package handlers_eks
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/utils"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// WebhookTokenReviewResult is the gateway's reply to the eks-token-webhook's
|
||||
// TokenReview relay: the resolved K8s identity for an `aws eks get-token`
|
||||
// bearer token, or Authenticated=false. The webhook maps it into the
|
||||
// authentication.k8s.io/v1 TokenReview the apiserver expects. The locationName
|
||||
// tags let the gateway's restjson marshaler (jsonutil) emit the same keys the
|
||||
// json tags name, so the webhook decodes the body with encoding/json.
|
||||
type WebhookTokenReviewResult struct {
|
||||
Authenticated bool `json:"authenticated" locationName:"authenticated"`
|
||||
Username string `json:"username,omitempty" locationName:"username"`
|
||||
UID string `json:"uid,omitempty" locationName:"uid"`
|
||||
Groups []string `json:"groups,omitempty" locationName:"groups"`
|
||||
}
|
||||
|
||||
// Authenticate is the pure TokenReview decision core: decode token → STS verify
|
||||
// → AccessEntry lookup → identity. Any failure yields Authenticated=false (never
|
||||
// an error) so a forged/unknown token is a clean K8s 401, not a webhook outage.
|
||||
// verify and lookup are injected so the logic is unit-testable without NATS.
|
||||
func Authenticate(
|
||||
token string,
|
||||
verify func(presignedURL string) (*TokenVerifyResponse, error),
|
||||
lookup func(principalARN string) (*AccessEntryRecord, error),
|
||||
) WebhookTokenReviewResult {
|
||||
presignedURL, err := DecodeGetToken(token)
|
||||
if err != nil {
|
||||
return WebhookTokenReviewResult{Authenticated: false}
|
||||
}
|
||||
ident, err := verify(presignedURL)
|
||||
if err != nil {
|
||||
slog.Debug("token verify rejected", "err", err)
|
||||
return WebhookTokenReviewResult{Authenticated: false}
|
||||
}
|
||||
rec, err := lookup(ident.ARN)
|
||||
if err != nil {
|
||||
// No AccessEntry for an otherwise-valid IAM principal: authenticated to
|
||||
// AWS, but not granted any K8s identity on this cluster.
|
||||
slog.Debug("no access entry for principal", "arn", ident.ARN, "err", err)
|
||||
return WebhookTokenReviewResult{Authenticated: false}
|
||||
}
|
||||
uid := ident.UserID
|
||||
if uid == "" {
|
||||
uid = ident.ARN
|
||||
}
|
||||
return WebhookTokenReviewResult{
|
||||
Authenticated: true,
|
||||
Username: rec.KubernetesUsername,
|
||||
UID: uid,
|
||||
Groups: rec.KubernetesGroups,
|
||||
}
|
||||
}
|
||||
|
||||
// ResolveTokenReview runs the TokenReview decision host-side, wiring the real
|
||||
// STS verify (NATS request to the awsgw-hosted verify subject) and AccessEntry
|
||||
// KV read into Authenticate. The eks-token-webhook reaches this through the
|
||||
// gateway broker (POST /clusters/{name}/token-review) so the control-plane VM
|
||||
// never speaks NATS. Returns an error only for a genuine infrastructure fault
|
||||
// (no JetStream / missing account bucket); a forged token resolves to an
|
||||
// Authenticated=false result, not an error.
|
||||
func ResolveTokenReview(nc *nats.Conn, accountID, clusterName, token string, verifyTimeout time.Duration) (WebhookTokenReviewResult, error) {
|
||||
if nc == nil {
|
||||
return WebhookTokenReviewResult{}, fmt.Errorf("eks: ResolveTokenReview nil nats conn")
|
||||
}
|
||||
js, err := nc.JetStream()
|
||||
if err != nil {
|
||||
return WebhookTokenReviewResult{}, fmt.Errorf("eks: jetstream: %w", err)
|
||||
}
|
||||
kv, err := js.KeyValue(AccountBucketName(accountID))
|
||||
if err != nil {
|
||||
return WebhookTokenReviewResult{}, fmt.Errorf("eks: open account KV %s: %w", accountID, err)
|
||||
}
|
||||
|
||||
verify := func(presignedURL string) (*TokenVerifyResponse, error) {
|
||||
return utils.NATSRequest[TokenVerifyResponse](
|
||||
nc, TokenVerifySubject,
|
||||
TokenVerifyRequest{PresignedURL: presignedURL, ClusterName: clusterName},
|
||||
verifyTimeout, "")
|
||||
}
|
||||
lookup := func(principalARN string) (*AccessEntryRecord, error) {
|
||||
return GetAccessEntryRecord(kv, clusterName, principalARN)
|
||||
}
|
||||
return Authenticate(token, verify, lookup), nil
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
package handlers_eks
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/testutil"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func validToken(url string) string {
|
||||
return "k8s-aws-v1." + base64.RawURLEncoding.EncodeToString([]byte(url))
|
||||
}
|
||||
|
||||
const testARN = "arn:aws:iam::111122223333:role/admin"
|
||||
|
||||
func okVerify(_ string) (*TokenVerifyResponse, error) {
|
||||
return &TokenVerifyResponse{
|
||||
AccountID: "111122223333",
|
||||
ARN: testARN,
|
||||
UserID: "AROAEXAMPLE:session",
|
||||
PrincipalType: "AssumedRole",
|
||||
}, nil
|
||||
}
|
||||
|
||||
func TestAuthenticate_GrantsWhenEntryExists(t *testing.T) {
|
||||
lookup := func(arn string) (*AccessEntryRecord, error) {
|
||||
assert.Equal(t, testARN, arn)
|
||||
return &AccessEntryRecord{
|
||||
KubernetesUsername: testARN,
|
||||
KubernetesGroups: []string{"system:masters"},
|
||||
}, nil
|
||||
}
|
||||
|
||||
res := Authenticate(validToken("https://sts.amazonaws.com/?Action=GetCallerIdentity"), okVerify, lookup)
|
||||
|
||||
require.True(t, res.Authenticated)
|
||||
assert.Equal(t, testARN, res.Username)
|
||||
assert.Equal(t, "AROAEXAMPLE:session", res.UID)
|
||||
assert.Equal(t, []string{"system:masters"}, res.Groups)
|
||||
}
|
||||
|
||||
func TestAuthenticate_DeniesMalformedToken(t *testing.T) {
|
||||
called := false
|
||||
verify := func(string) (*TokenVerifyResponse, error) { called = true; return nil, nil }
|
||||
lookup := func(string) (*AccessEntryRecord, error) { return nil, nil }
|
||||
|
||||
res := Authenticate("not-a-k8s-aws-token", verify, lookup)
|
||||
|
||||
assert.False(t, res.Authenticated)
|
||||
assert.False(t, called, "must not call STS for a token that fails to decode")
|
||||
}
|
||||
|
||||
func TestAuthenticate_DeniesWhenVerifyFails(t *testing.T) {
|
||||
verify := func(string) (*TokenVerifyResponse, error) {
|
||||
return nil, errors.New("signature mismatch")
|
||||
}
|
||||
lookup := func(string) (*AccessEntryRecord, error) {
|
||||
t.Fatal("lookup must not run when verify fails")
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
res := Authenticate(validToken("https://sts/?x=1"), verify, lookup)
|
||||
assert.False(t, res.Authenticated)
|
||||
}
|
||||
|
||||
func TestAuthenticate_DeniesWhenNoAccessEntry(t *testing.T) {
|
||||
lookup := func(string) (*AccessEntryRecord, error) {
|
||||
return nil, ErrAccessEntryNotFound
|
||||
}
|
||||
|
||||
res := Authenticate(validToken("https://sts/?x=1"), okVerify, lookup)
|
||||
assert.False(t, res.Authenticated)
|
||||
assert.Empty(t, res.Username)
|
||||
}
|
||||
|
||||
func TestAuthenticate_FallsBackUIDToARN(t *testing.T) {
|
||||
verify := func(string) (*TokenVerifyResponse, error) {
|
||||
return &TokenVerifyResponse{ARN: testARN}, nil // no UserID
|
||||
}
|
||||
lookup := func(string) (*AccessEntryRecord, error) {
|
||||
return &AccessEntryRecord{KubernetesUsername: testARN, KubernetesGroups: []string{"system:masters"}}, nil
|
||||
}
|
||||
|
||||
res := Authenticate(validToken("https://sts/?x=1"), verify, lookup)
|
||||
require.True(t, res.Authenticated)
|
||||
assert.Equal(t, testARN, res.UID)
|
||||
}
|
||||
|
||||
func TestResolveTokenReview_NilConn(t *testing.T) {
|
||||
_, err := ResolveTokenReview(nil, "111122223333", "alpha", "tok", time.Second)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
// A genuine infra fault (account bucket never created) is an error, not a
|
||||
// silent Authenticated=false — the webhook turns it into a retryable 5xx.
|
||||
func TestResolveTokenReview_MissingBucketErrors(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
_, err := ResolveTokenReview(nc, "111122223333", "alpha", validToken("https://sts/?x=1"), time.Second)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestResolveTokenReview_AuthenticatesViaVerifyAndKV(t *testing.T) {
|
||||
_, nc, js := testutil.StartTestJetStream(t)
|
||||
kv := testutil.SeedKV(t, js, AccountBucketName("111122223333"), nil)
|
||||
require.NoError(t, PutAccessEntryRecord(kv, &AccessEntryRecord{
|
||||
ClusterName: "alpha",
|
||||
PrincipalARN: testARN,
|
||||
KubernetesUsername: testARN,
|
||||
KubernetesGroups: []string{"system:masters"},
|
||||
Type: AccessEntryTypeStandard,
|
||||
}))
|
||||
|
||||
// Stand in for the awsgw-hosted STS verify responder.
|
||||
sub, err := nc.Subscribe(TokenVerifySubject, func(m *nats.Msg) {
|
||||
resp, _ := json.Marshal(TokenVerifyResponse{
|
||||
AccountID: "111122223333",
|
||||
ARN: testARN,
|
||||
UserID: "AROAEXAMPLE:session",
|
||||
PrincipalType: "AssumedRole",
|
||||
})
|
||||
_ = m.Respond(resp)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
|
||||
res, err := ResolveTokenReview(nc, "111122223333", "alpha", validToken("https://sts/?Action=GetCallerIdentity"), 2*time.Second)
|
||||
require.NoError(t, err)
|
||||
require.True(t, res.Authenticated)
|
||||
assert.Equal(t, testARN, res.Username)
|
||||
assert.Equal(t, "AROAEXAMPLE:session", res.UID)
|
||||
assert.Equal(t, []string{"system:masters"}, res.Groups)
|
||||
}
|
||||
|
||||
// A valid IAM principal with no AccessEntry resolves to Authenticated=false
|
||||
// (a clean 401), not an error.
|
||||
func TestResolveTokenReview_NoAccessEntryDenies(t *testing.T) {
|
||||
_, nc, js := testutil.StartTestJetStream(t)
|
||||
testutil.SeedKV(t, js, AccountBucketName("111122223333"), nil)
|
||||
|
||||
sub, err := nc.Subscribe(TokenVerifySubject, func(m *nats.Msg) {
|
||||
resp, _ := json.Marshal(TokenVerifyResponse{AccountID: "111122223333", ARN: testARN})
|
||||
_ = m.Respond(resp)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
|
||||
res, err := ResolveTokenReview(nc, "111122223333", "alpha", validToken("https://sts/?x=1"), 2*time.Second)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, res.Authenticated)
|
||||
}
|
||||
@@ -27,14 +27,29 @@ type ClusterRecord struct {
|
||||
}
|
||||
|
||||
// NodegroupRecord is the persisted-state envelope for an EKS managed
|
||||
// nodegroup.
|
||||
// nodegroup. It is the source of truth for Describe/ListNodegroups and tracks
|
||||
// the worker EC2 instance IDs the nodegroup owns so Update/DeleteNodegroup can
|
||||
// scale and tear them down.
|
||||
type NodegroupRecord struct {
|
||||
ClusterName string `json:"clusterName"`
|
||||
Name string `json:"name"`
|
||||
Status string `json:"status"`
|
||||
DesiredSize int64 `json:"desiredSize"`
|
||||
InstanceTypes []string `json:"instanceTypes,omitempty"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
ClusterName string `json:"clusterName"`
|
||||
Name string `json:"name"`
|
||||
Arn string `json:"arn"`
|
||||
Status string `json:"status"`
|
||||
StatusReason string `json:"statusReason,omitempty"`
|
||||
Subnets []string `json:"subnets,omitempty"`
|
||||
InstanceTypes []string `json:"instanceTypes,omitempty"`
|
||||
AMIType string `json:"amiType,omitempty"`
|
||||
DiskSize int64 `json:"diskSize,omitempty"`
|
||||
ScalingMin int64 `json:"scalingMin"`
|
||||
ScalingMax int64 `json:"scalingMax"`
|
||||
ScalingDesired int64 `json:"scalingDesired"`
|
||||
Version string `json:"version,omitempty"`
|
||||
NodeRole string `json:"nodeRole,omitempty"`
|
||||
Labels map[string]string `json:"labels,omitempty"`
|
||||
InstanceIDs []string `json:"instanceIds,omitempty"`
|
||||
Health string `json:"health,omitempty"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
ModifiedAt time.Time `json:"modifiedAt"`
|
||||
}
|
||||
|
||||
// AccessEntryRecord is the persisted-state envelope for an EKS API-mode
|
||||
|
||||
@@ -11,6 +11,7 @@ type ELBv2Service interface {
|
||||
DescribeLoadBalancerAttributes(input *elbv2.DescribeLoadBalancerAttributesInput, accountID string) (*elbv2.DescribeLoadBalancerAttributesOutput, error)
|
||||
SetSecurityGroups(input *elbv2.SetSecurityGroupsInput, accountID string) (*elbv2.SetSecurityGroupsOutput, error)
|
||||
SetIpAddressType(input *elbv2.SetIpAddressTypeInput, accountID string) (*elbv2.SetIpAddressTypeOutput, error)
|
||||
SetSubnets(input *elbv2.SetSubnetsInput, accountID string) (*elbv2.SetSubnetsOutput, error)
|
||||
|
||||
CreateTargetGroup(input *elbv2.CreateTargetGroupInput, accountID string) (*elbv2.CreateTargetGroupOutput, error)
|
||||
ModifyTargetGroup(input *elbv2.ModifyTargetGroupInput, accountID string) (*elbv2.ModifyTargetGroupOutput, error)
|
||||
|
||||
@@ -89,6 +89,13 @@ func (s *ELBv2ServiceImpl) resolveMgmtRoute(scheme string) (gateway, target stri
|
||||
if scheme == SchemeInternal && s.MgmtBridgeIP != "" && s.AdvertiseIP != "" {
|
||||
return s.MgmtBridgeIP, s.AdvertiseIP
|
||||
}
|
||||
if scheme == SchemeInternal {
|
||||
slog.Warn("resolveMgmtRoute: internal LB has no mgmt return route; lb-agent cannot reach AWSGW and the LB will stay provisioning",
|
||||
"mgmtRouteGateway", s.MgmtRouteGateway,
|
||||
"mgmtRouteTarget", s.MgmtRouteTarget,
|
||||
"mgmtBridgeIP", s.MgmtBridgeIP,
|
||||
"advertiseIP", s.AdvertiseIP)
|
||||
}
|
||||
return "", ""
|
||||
}
|
||||
|
||||
@@ -429,6 +436,81 @@ func buildExtraENIInputs(eniIDs []string, eniDetails map[string]*ec2.NetworkInte
|
||||
return extras
|
||||
}
|
||||
|
||||
// lbVMLaunch is the outcome of booting the system VM that backs a load
|
||||
// balancer. failed is true when the VM could not be launched (missing system
|
||||
// credentials or a launcher error) so the caller can mark the LB failed.
|
||||
type lbVMLaunch struct {
|
||||
instanceID string
|
||||
vpcIP string
|
||||
publicIP string
|
||||
hostPorts map[int]int
|
||||
failed bool
|
||||
}
|
||||
|
||||
// launchLBVM boots the system VM backing a load balancer with the given ENI
|
||||
// set. The first ENI is the primary NIC; any additional ENIs are passed as
|
||||
// ExtraENIs so the daemon sets up a tap + QEMU NIC per subnet, giving the LB VM
|
||||
// data-plane presence in every subnet it spans. When no launcher is configured
|
||||
// the call is a no-op (zero value, failed=false) so launcher-less deployments
|
||||
// keep the LB Active. Shared by CreateLoadBalancer and the SetSubnets relaunch.
|
||||
func (s *ELBv2ServiceImpl) launchLBVM(lbID, scheme string, eniIDs, subnets []string, accountID string) lbVMLaunch {
|
||||
var res lbVMLaunch
|
||||
if s.InstanceLauncher == nil || len(eniIDs) == 0 || len(subnets) == 0 {
|
||||
return res
|
||||
}
|
||||
|
||||
eniDetails := s.describeENIs(eniIDs, accountID)
|
||||
primary := eniDetails[eniIDs[0]]
|
||||
primaryIP := ""
|
||||
primaryMAC := ""
|
||||
if primary != nil {
|
||||
primaryIP = aws.StringValue(primary.PrivateIpAddress)
|
||||
primaryMAC = aws.StringValue(primary.MacAddress)
|
||||
}
|
||||
|
||||
extraENIInputs := buildExtraENIInputs(eniIDs, eniDetails)
|
||||
|
||||
if s.GatewayURL == "" || s.SystemAccessKey == "" || s.SystemSecretKey == "" {
|
||||
slog.Error("launchLBVM: system credentials not configured — cannot launch LB VM", "lbId", lbID)
|
||||
res.failed = true
|
||||
return res
|
||||
}
|
||||
|
||||
nics := s.buildMicrovmNICs(primaryIP, primaryMAC, subnets[0], eniIDs[0], scheme, extraENIInputs, accountID)
|
||||
launchInput := &SystemInstanceInput{
|
||||
InstanceType: s.getSystemInstanceType(),
|
||||
SubnetID: subnets[0],
|
||||
ENIID: eniIDs[0],
|
||||
ENIMac: primaryMAC,
|
||||
ENIIP: primaryIP,
|
||||
ExtraENIs: extraENIInputs,
|
||||
Scheme: scheme,
|
||||
AccountID: accountID,
|
||||
NICs: nics,
|
||||
LBAgentEnv: s.buildLBAgentEnv(lbID),
|
||||
CACert: s.CACert,
|
||||
}
|
||||
// Dev-mode only: forward HTTP/HTTPS ports from host for local testing.
|
||||
// In production (VPC networking), traffic reaches the LB VM's VPC IP directly.
|
||||
if s.config != nil && s.config.Daemon.DevNetworking {
|
||||
launchInput.HostfwdPorts = []int{80, 443}
|
||||
}
|
||||
|
||||
out, launchErr := s.InstanceLauncher.LaunchSystemInstance(launchInput)
|
||||
if launchErr != nil {
|
||||
slog.Error("launchLBVM: failed to launch LB VM", "lbId", lbID, "err", launchErr)
|
||||
res.failed = true
|
||||
return res
|
||||
}
|
||||
|
||||
res.instanceID = out.InstanceID
|
||||
res.vpcIP = out.PrivateIP
|
||||
res.publicIP = out.PublicIP
|
||||
res.hostPorts = out.HostfwdMap
|
||||
slog.Info("launchLBVM: LB VM launched", "lbId", lbID, "instanceId", out.InstanceID, "ip", out.PrivateIP, "publicIp", out.PublicIP, "hostfwd", out.HostfwdMap)
|
||||
return res
|
||||
}
|
||||
|
||||
// loadRecoveryLBIndex returns the instanceID->LB map, populated lazily on
|
||||
// first successful call. Errors are not cached: a transient JetStream
|
||||
// failure at startup must not condemn every subsequent recovery candidate
|
||||
@@ -777,12 +859,19 @@ func (s *ELBv2ServiceImpl) LBAgentHeartbeat(input *LBAgentHeartbeatInput, accoun
|
||||
}
|
||||
|
||||
lbID := *input.LBID
|
||||
slog.Debug("LBAgentHeartbeat received", "lbId", lbID, "accountId", accountID)
|
||||
lb, err := s.store.GetLoadBalancer(lbID)
|
||||
if err != nil {
|
||||
slog.Error("LBAgentHeartbeat: failed to get LB", "lbId", lbID, "err", err)
|
||||
return nil, errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
if lb == nil || (lb.AccountID != accountID && accountID != utils.GlobalAccountID) {
|
||||
// The heartbeat reached AWSGW (so the mgmt return path works) but
|
||||
// cannot transition any LB. Without this the reject is silent and a
|
||||
// stuck-in-provisioning LB looks identical to one whose heartbeat
|
||||
// never arrived.
|
||||
slog.Warn("LBAgentHeartbeat: LB not found or account mismatch",
|
||||
"lbId", lbID, "accountId", accountID, "found", lb != nil)
|
||||
return nil, errors.New(awserrors.ErrorELBv2LoadBalancerNotFound)
|
||||
}
|
||||
|
||||
@@ -1072,7 +1161,22 @@ func (s *ELBv2ServiceImpl) CreateLoadBalancer(input *elbv2.CreateLoadBalancerInp
|
||||
var eniIDs []string
|
||||
var availZones []AvailZoneInfo
|
||||
vpcID := ""
|
||||
var nlbManagedSGID string
|
||||
if s.VPCService != nil && len(subnets) > 0 {
|
||||
// NLBs reject customer SGs (rejected above), so the ENIs would fall back
|
||||
// to the VPC default SG whose intra-SG-only ingress drops inbound
|
||||
// listener traffic. Mint a dedicated managed SG up front and attach it to
|
||||
// every LB ENI; CreateListener opens the listener ports on it.
|
||||
eniGroups := securityGroups
|
||||
if lbType == LoadBalancerTypeNetwork {
|
||||
sgID, sgErr := s.createNLBManagedSG(lbID, lbArn, subnets[0], accountID)
|
||||
if sgErr != nil {
|
||||
slog.Error("CreateLoadBalancer: failed to create managed NLB SG", "lbId", lbID, "err", sgErr)
|
||||
return nil, errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
nlbManagedSGID = sgID
|
||||
eniGroups = []string{sgID}
|
||||
}
|
||||
for _, subnetID := range subnets {
|
||||
eniIn := &ec2.CreateNetworkInterfaceInput{
|
||||
SubnetId: aws.String(subnetID),
|
||||
@@ -1088,11 +1192,12 @@ func (s *ELBv2ServiceImpl) CreateLoadBalancer(input *elbv2.CreateLoadBalancerInp
|
||||
},
|
||||
}
|
||||
// SG enforcement gates inbound traffic by the ENI's port-group
|
||||
// membership, which is set at ENI creation. Empty Groups
|
||||
// intentionally falls back to the VPC default SG inside
|
||||
// membership, which is set at ENI creation. For NLBs this is the
|
||||
// managed SG minted above; for ALBs it is the customer SGs. Empty
|
||||
// Groups intentionally falls back to the VPC default SG inside
|
||||
// eni.CreateNetworkInterface.
|
||||
if len(securityGroups) > 0 {
|
||||
eniIn.Groups = aws.StringSlice(securityGroups)
|
||||
if len(eniGroups) > 0 {
|
||||
eniIn.Groups = aws.StringSlice(eniGroups)
|
||||
}
|
||||
eniOut, eniErr := s.VPCService.CreateNetworkInterface(eniIn, accountID)
|
||||
if eniErr != nil {
|
||||
@@ -1104,6 +1209,7 @@ func (s *ELBv2ServiceImpl) CreateLoadBalancer(input *elbv2.CreateLoadBalancerInp
|
||||
slog.Error("CreateLoadBalancer: rollback failed to delete ENI", "eni", rollbackENI, "err", delErr)
|
||||
}
|
||||
}
|
||||
s.deleteNLBManagedSG(nlbManagedSGID, accountID)
|
||||
slog.Error("CreateLoadBalancer: failed to create ENI", "subnet", subnetID, "err", eniErr)
|
||||
return nil, errors.New(awserrors.ErrorELBv2SubnetNotFound)
|
||||
}
|
||||
@@ -1129,64 +1235,14 @@ func (s *ELBv2ServiceImpl) CreateLoadBalancer(input *elbv2.CreateLoadBalancerInp
|
||||
// The first ENI is the primary NIC; any additional ENIs are passed as
|
||||
// ExtraENIs so the daemon sets up a tap + QEMU NIC for each, giving the
|
||||
// ALB VM data-plane presence in every subnet the LB spans.
|
||||
var albInstanceID string
|
||||
var albVPCIP string
|
||||
var hostPorts map[int]int
|
||||
var launchFailed bool
|
||||
if s.InstanceLauncher != nil && len(eniIDs) > 0 && len(subnets) > 0 {
|
||||
eniDetails := s.describeENIs(eniIDs, accountID)
|
||||
|
||||
primary := eniDetails[eniIDs[0]]
|
||||
primaryIP := ""
|
||||
primaryMAC := ""
|
||||
if primary != nil {
|
||||
primaryIP = aws.StringValue(primary.PrivateIpAddress)
|
||||
primaryMAC = aws.StringValue(primary.MacAddress)
|
||||
}
|
||||
|
||||
extraENIInputs := buildExtraENIInputs(eniIDs, eniDetails)
|
||||
|
||||
var launchInput *SystemInstanceInput
|
||||
if s.GatewayURL == "" || s.SystemAccessKey == "" || s.SystemSecretKey == "" {
|
||||
slog.Error("CreateLoadBalancer: system credentials not configured — cannot launch ALB VM", "lbId", lbID)
|
||||
launchFailed = true
|
||||
} else {
|
||||
nics := s.buildMicrovmNICs(primaryIP, primaryMAC, subnets[0], eniIDs[0], scheme, extraENIInputs, accountID)
|
||||
launchInput = &SystemInstanceInput{
|
||||
InstanceType: s.getSystemInstanceType(),
|
||||
SubnetID: subnets[0],
|
||||
ENIID: eniIDs[0],
|
||||
ENIMac: primaryMAC,
|
||||
ENIIP: primaryIP,
|
||||
ExtraENIs: extraENIInputs,
|
||||
Scheme: scheme,
|
||||
AccountID: accountID,
|
||||
NICs: nics,
|
||||
LBAgentEnv: s.buildLBAgentEnv(lbID),
|
||||
CACert: s.CACert,
|
||||
}
|
||||
}
|
||||
if !launchFailed && launchInput != nil {
|
||||
// Dev-mode only: forward HTTP/HTTPS ports from host for local testing.
|
||||
// In production (VPC networking), traffic reaches the ALB VM's VPC IP directly.
|
||||
if s.config != nil && s.config.Daemon.DevNetworking {
|
||||
launchInput.HostfwdPorts = []int{80, 443}
|
||||
}
|
||||
out, launchErr := s.InstanceLauncher.LaunchSystemInstance(launchInput)
|
||||
if launchErr != nil {
|
||||
slog.Error("CreateLoadBalancer: failed to launch ALB VM", "lbId", lbID, "err", launchErr)
|
||||
launchFailed = true
|
||||
} else {
|
||||
albInstanceID = out.InstanceID
|
||||
albVPCIP = out.PrivateIP
|
||||
hostPorts = out.HostfwdMap
|
||||
// Record public IP on the first AZ entry for internet-facing ALBs.
|
||||
if out.PublicIP != "" && len(availZones) > 0 {
|
||||
availZones[0].PublicIP = out.PublicIP
|
||||
}
|
||||
slog.Info("CreateLoadBalancer: ALB VM launched", "lbId", lbID, "instanceId", albInstanceID, "ip", out.PrivateIP, "publicIp", out.PublicIP, "hostfwd", hostPorts)
|
||||
}
|
||||
}
|
||||
launch := s.launchLBVM(lbID, scheme, eniIDs, subnets, accountID)
|
||||
albInstanceID := launch.instanceID
|
||||
albVPCIP := launch.vpcIP
|
||||
hostPorts := launch.hostPorts
|
||||
launchFailed := launch.failed
|
||||
// Record public IP on the first AZ entry for internet-facing LBs.
|
||||
if launch.publicIP != "" && len(availZones) > 0 {
|
||||
availZones[0].PublicIP = launch.publicIP
|
||||
}
|
||||
|
||||
// ALB starts in provisioning until the agent inside the VM connects and
|
||||
@@ -1195,6 +1251,19 @@ func (s *ELBv2ServiceImpl) CreateLoadBalancer(input *elbv2.CreateLoadBalancerInp
|
||||
state := StateActive
|
||||
if albInstanceID != "" {
|
||||
state = StateProvisioning
|
||||
// Internal LBs only leave provisioning once the lb-agent heartbeats,
|
||||
// which needs the mgmt return route. If it can't be resolved the LB
|
||||
// hangs in provisioning forever — fail loud instead so the broken
|
||||
// return path is visible immediately rather than as a generic timeout.
|
||||
if scheme == SchemeInternal {
|
||||
if gw, tgt := s.resolveMgmtRoute(scheme); gw == "" || tgt == "" {
|
||||
slog.Error("CreateLoadBalancer: internal LB has no mgmt return route; marking failed (lb-agent cannot heartbeat AWSGW)",
|
||||
"lbId", lbID,
|
||||
"mgmtBridgeIP", s.MgmtBridgeIP,
|
||||
"advertiseIP", s.AdvertiseIP)
|
||||
state = StateFailed
|
||||
}
|
||||
}
|
||||
} else if launchFailed {
|
||||
state = StateFailed
|
||||
}
|
||||
@@ -1212,6 +1281,7 @@ func (s *ELBv2ServiceImpl) CreateLoadBalancer(input *elbv2.CreateLoadBalancerInp
|
||||
State: state,
|
||||
VpcId: vpcID,
|
||||
SecurityGroups: securityGroups,
|
||||
NLBManagedSGID: nlbManagedSGID,
|
||||
Subnets: subnets,
|
||||
AvailZones: availZones,
|
||||
ENIs: eniIDs,
|
||||
@@ -1297,6 +1367,8 @@ func (s *ELBv2ServiceImpl) DeleteLoadBalancer(input *elbv2.DeleteLoadBalancerInp
|
||||
slog.Warn("Failed to delete ALB ENI during cleanup", "eniId", eniID, "err", eniErr)
|
||||
}
|
||||
}
|
||||
// The managed NLB SG can only be removed once its ENIs are gone.
|
||||
s.deleteNLBManagedSG(lb.NLBManagedSGID, accountID)
|
||||
}
|
||||
|
||||
if err := s.store.DeleteLoadBalancer(lb.LoadBalancerID); err != nil {
|
||||
@@ -2089,6 +2161,21 @@ func (s *ELBv2ServiceImpl) CreateListener(input *elbv2.CreateListenerInput, acco
|
||||
return nil, errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
|
||||
// Open the listener port on the NLB's managed front-end SG so inbound
|
||||
// traffic from the configured client CIDRs is admitted by the OVN ACL
|
||||
// (NLB ENIs would otherwise sit in the intra-SG-only default SG).
|
||||
if lb.Type == LoadBalancerTypeNetwork && lb.NLBManagedSGID != "" && s.VPCService != nil {
|
||||
cidrs, cidrErr := s.resolveNLBIngressCIDRs(lb)
|
||||
if cidrErr != nil {
|
||||
slog.Error("CreateListener: resolve ingress CIDRs failed", "lbArn", lb.LoadBalancerArn, "err", cidrErr)
|
||||
return nil, errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
if authErr := s.authorizeNLBListenerPort(lb, protocol, port, cidrs, accountID); authErr != nil {
|
||||
slog.Error("CreateListener: authorize listener port failed", "lbArn", lb.LoadBalancerArn, "port", port, "err", authErr)
|
||||
return nil, errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
}
|
||||
|
||||
// Start or reload HAProxy now that a listener exists
|
||||
if err := s.updateStoredConfig(lb); err != nil {
|
||||
slog.Error("CreateListener: failed to update config", "listenerId", listenerID, "err", err)
|
||||
@@ -2146,6 +2233,16 @@ func (s *ELBv2ServiceImpl) DeleteListener(input *elbv2.DeleteListenerInput, acco
|
||||
// Reload or stop HAProxy after listener removal
|
||||
lb, lbErr := s.store.GetLoadBalancerByArn(listener.LoadBalancerArn)
|
||||
if lbErr == nil && lb != nil {
|
||||
// Close the listener port on the NLB's managed front-end SG.
|
||||
if lb.Type == LoadBalancerTypeNetwork && lb.NLBManagedSGID != "" && s.VPCService != nil {
|
||||
if cidrs, cidrErr := s.resolveNLBIngressCIDRs(lb); cidrErr == nil {
|
||||
if revokeErr := s.revokeNLBListenerPort(lb, listener.Protocol, listener.Port, cidrs, accountID); revokeErr != nil {
|
||||
slog.Warn("DeleteListener: revoke listener port failed", "lbArn", lb.LoadBalancerArn, "port", listener.Port, "err", revokeErr)
|
||||
}
|
||||
} else {
|
||||
slog.Warn("DeleteListener: resolve ingress CIDRs failed", "lbArn", lb.LoadBalancerArn, "err", cidrErr)
|
||||
}
|
||||
}
|
||||
if err := s.updateStoredConfig(lb); err != nil {
|
||||
slog.Error("DeleteListener: failed to update config", "listenerArn", *input.ListenerArn, "err", err)
|
||||
return nil, errors.New(awserrors.ErrorServerInternal)
|
||||
|
||||
@@ -2,6 +2,7 @@ package handlers_elbv2
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
@@ -107,3 +108,266 @@ func (s *ELBv2ServiceImpl) SetSecurityGroups(input *elbv2.SetSecurityGroupsInput
|
||||
SecurityGroupIds: aws.StringSlice(sgs),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// SetSubnets enables the subnets (and their backing ENIs) for a load balancer.
|
||||
// It is a full add+remove: a subnet in the request but not on the LB gets a new
|
||||
// managed ENI; a subnet on the LB but absent from the request has its ENI
|
||||
// removed. The system VM binds its data-plane taps at boot and the launcher
|
||||
// exposes no live ENI hotplug, so the new ENI set is applied by relaunching the
|
||||
// LB VM — a brief data-plane interruption. The record is mutated only after the
|
||||
// new ENIs exist and the VM has been relaunched.
|
||||
func (s *ELBv2ServiceImpl) SetSubnets(input *elbv2.SetSubnetsInput, accountID string) (*elbv2.SetSubnetsOutput, error) {
|
||||
if input == nil || input.LoadBalancerArn == nil || *input.LoadBalancerArn == "" {
|
||||
return nil, errors.New(awserrors.ErrorMissingParameter)
|
||||
}
|
||||
|
||||
desired := desiredSubnetSet(input)
|
||||
if len(desired) == 0 {
|
||||
return nil, errors.New(awserrors.ErrorMissingParameter)
|
||||
}
|
||||
|
||||
lb, err := s.store.GetLoadBalancerByArn(*input.LoadBalancerArn)
|
||||
if err != nil {
|
||||
slog.Error("SetSubnets: failed to get LB", "arn", *input.LoadBalancerArn, "err", err)
|
||||
return nil, errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
if lb == nil || lb.AccountID != accountID {
|
||||
return nil, errors.New(awserrors.ErrorELBv2LoadBalancerNotFound)
|
||||
}
|
||||
|
||||
current := subnetENIMap(lb)
|
||||
desiredSet := make(map[string]bool, len(desired))
|
||||
for _, sn := range desired {
|
||||
desiredSet[sn] = true
|
||||
}
|
||||
|
||||
var toAdd, toRemove []string
|
||||
for _, sn := range desired {
|
||||
if _, ok := current[sn]; !ok {
|
||||
toAdd = append(toAdd, sn)
|
||||
}
|
||||
}
|
||||
for sn := range current {
|
||||
if !desiredSet[sn] {
|
||||
toRemove = append(toRemove, sn)
|
||||
}
|
||||
}
|
||||
|
||||
if len(toAdd) == 0 && len(toRemove) == 0 {
|
||||
return s.setSubnetsOutput(lb), nil // idempotent — no change
|
||||
}
|
||||
|
||||
// Without a VPC service we cannot manage ENIs; record the requested subnet
|
||||
// set so the API stays consistent for launcher-less / test deployments.
|
||||
if s.VPCService == nil {
|
||||
lb.Subnets = desired
|
||||
lb.AvailZones = rebuildAvailZones(desired, lb.AvailZones, nil)
|
||||
if err := s.store.PutLoadBalancer(lb); err != nil {
|
||||
slog.Error("SetSubnets: failed to persist LB", "arn", *input.LoadBalancerArn, "err", err)
|
||||
return nil, errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
return s.setSubnetsOutput(lb), nil
|
||||
}
|
||||
|
||||
// Create ENIs for the added subnets first; roll them back on any failure so
|
||||
// a partial apply never leaks ENIs or mutates the live LB.
|
||||
newENIBySubnet := make(map[string]string, len(toAdd))
|
||||
newAZBySubnet := make(map[string]string, len(toAdd))
|
||||
rollbackNewENIs := func() {
|
||||
for _, created := range newENIBySubnet {
|
||||
if _, delErr := s.VPCService.DeleteNetworkInterface(&ec2.DeleteNetworkInterfaceInput{
|
||||
NetworkInterfaceId: aws.String(created),
|
||||
}, accountID); delErr != nil {
|
||||
slog.Error("SetSubnets: rollback failed to delete ENI", "eni", created, "err", delErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, subnetID := range toAdd {
|
||||
eniID, az, eniErr := s.createLBENI(subnetID, lb, accountID)
|
||||
if eniErr != nil {
|
||||
rollbackNewENIs()
|
||||
slog.Error("SetSubnets: failed to create ENI", "subnet", subnetID, "err", eniErr)
|
||||
return nil, errors.New(awserrors.ErrorELBv2SubnetNotFound)
|
||||
}
|
||||
newENIBySubnet[subnetID] = eniID
|
||||
newAZBySubnet[subnetID] = az
|
||||
}
|
||||
|
||||
// Assemble the new ENI set in desired-subnet order (primary = first subnet).
|
||||
newENIs := make([]string, 0, len(desired))
|
||||
for _, sn := range desired {
|
||||
if eniID, ok := current[sn]; ok {
|
||||
newENIs = append(newENIs, eniID)
|
||||
} else {
|
||||
newENIs = append(newENIs, newENIBySubnet[sn])
|
||||
}
|
||||
}
|
||||
|
||||
// Terminate the LB VM before reshaping its ENI set, then relaunch it on the
|
||||
// new set.
|
||||
if lb.InstanceID != "" && s.InstanceLauncher != nil {
|
||||
if err := s.InstanceLauncher.TerminateSystemInstance(lb.InstanceID); err != nil {
|
||||
rollbackNewENIs()
|
||||
slog.Error("SetSubnets: failed to terminate LB VM for relaunch", "arn", *input.LoadBalancerArn, "instanceId", lb.InstanceID, "err", err)
|
||||
return nil, errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
}
|
||||
|
||||
// TerminateSystemInstance tears down the VM but leaves its ENIs marked
|
||||
// in-use, so detach the existing ENIs explicitly: retained ones must be free
|
||||
// to re-attach to the relaunched VM, and removed ones must be detached before
|
||||
// they can be deleted.
|
||||
for _, eniID := range current {
|
||||
if detachErr := s.VPCService.DetachENI(accountID, eniID); detachErr != nil {
|
||||
slog.Warn("SetSubnets: failed to detach ENI before relaunch", "eni", eniID, "err", detachErr)
|
||||
}
|
||||
}
|
||||
|
||||
// Delete ENIs for removed subnets now that they are detached.
|
||||
for _, sn := range toRemove {
|
||||
eniID := current[sn]
|
||||
if _, delErr := s.VPCService.DeleteNetworkInterface(&ec2.DeleteNetworkInterfaceInput{
|
||||
NetworkInterfaceId: aws.String(eniID),
|
||||
}, accountID); delErr != nil {
|
||||
slog.Error("SetSubnets: failed to delete removed ENI", "subnet", sn, "eni", eniID, "err", delErr)
|
||||
}
|
||||
}
|
||||
|
||||
launch := s.launchLBVM(lb.LoadBalancerID, lb.Scheme, newENIs, desired, accountID)
|
||||
availZones := rebuildAvailZones(desired, lb.AvailZones, newAZBySubnet)
|
||||
if launch.publicIP != "" && len(availZones) > 0 {
|
||||
availZones[0].PublicIP = launch.publicIP
|
||||
}
|
||||
|
||||
lb.Subnets = desired
|
||||
lb.ENIs = newENIs
|
||||
lb.AvailZones = availZones
|
||||
lb.InstanceID = launch.instanceID
|
||||
lb.VPCIP = launch.vpcIP
|
||||
lb.HostPorts = launch.hostPorts
|
||||
lb.State = s.lbStateAfterLaunch(launch, lb.Scheme)
|
||||
|
||||
if err := s.store.PutLoadBalancer(lb); err != nil {
|
||||
slog.Error("SetSubnets: failed to persist LB", "arn", *input.LoadBalancerArn, "err", err)
|
||||
return nil, errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
|
||||
slog.Info("SetSubnets completed", "arn", *input.LoadBalancerArn, "subnets", len(desired), "added", len(toAdd), "removed", len(toRemove), "state", lb.State)
|
||||
return s.setSubnetsOutput(lb), nil
|
||||
}
|
||||
|
||||
// desiredSubnetSet flattens the request's Subnets and SubnetMappings into a
|
||||
// de-duplicated, order-preserving subnet-ID list.
|
||||
func desiredSubnetSet(input *elbv2.SetSubnetsInput) []string {
|
||||
seen := make(map[string]bool)
|
||||
var out []string
|
||||
add := func(sn string) {
|
||||
if sn != "" && !seen[sn] {
|
||||
seen[sn] = true
|
||||
out = append(out, sn)
|
||||
}
|
||||
}
|
||||
for _, sn := range input.Subnets {
|
||||
if sn != nil {
|
||||
add(*sn)
|
||||
}
|
||||
}
|
||||
for _, m := range input.SubnetMappings {
|
||||
if m != nil && m.SubnetId != nil {
|
||||
add(*m.SubnetId)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// subnetENIMap pairs each of the LB's current subnets with its backing ENI.
|
||||
// CreateLoadBalancer and SetSubnets write Subnets and ENIs in lockstep, so the
|
||||
// parallel arrays are authoritative for which ENI lives in which subnet.
|
||||
func subnetENIMap(lb *LoadBalancerRecord) map[string]string {
|
||||
m := make(map[string]string, len(lb.Subnets))
|
||||
for i, sn := range lb.Subnets {
|
||||
if i < len(lb.ENIs) {
|
||||
m[sn] = lb.ENIs[i]
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// createLBENI creates a single managed ENI for the LB in the given subnet,
|
||||
// tagged and security-grouped like the create-time ENIs. Returns the ENI ID and
|
||||
// its availability zone.
|
||||
func (s *ELBv2ServiceImpl) createLBENI(subnetID string, lb *LoadBalancerRecord, accountID string) (eniID, az string, err error) {
|
||||
eniIn := &ec2.CreateNetworkInterfaceInput{
|
||||
SubnetId: aws.String(subnetID),
|
||||
Description: aws.String(fmt.Sprintf("ELB %s/%s", lb.Name, lb.LoadBalancerID)),
|
||||
TagSpecifications: []*ec2.TagSpecification{
|
||||
{
|
||||
ResourceType: aws.String("network-interface"),
|
||||
Tags: []*ec2.Tag{
|
||||
{Key: aws.String(elbv2ManagedByTag), Value: aws.String(elbv2ManagedByValue)},
|
||||
{Key: aws.String(elbv2LBTag), Value: aws.String(lb.LoadBalancerArn)},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
if groups := lbENIGroups(lb); len(groups) > 0 {
|
||||
eniIn.Groups = aws.StringSlice(groups)
|
||||
}
|
||||
out, err := s.VPCService.CreateNetworkInterface(eniIn, accountID)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
eni := out.NetworkInterface
|
||||
return aws.StringValue(eni.NetworkInterfaceId), aws.StringValue(eni.AvailabilityZone), nil
|
||||
}
|
||||
|
||||
// rebuildAvailZones produces the AvailZoneInfo list for the new subnet set in
|
||||
// order, preserving the existing zone name for retained subnets and using
|
||||
// newAZBySubnet for added ones. PublicIP is cleared — it is re-derived from the
|
||||
// relaunched VM by the caller.
|
||||
func rebuildAvailZones(subnets []string, existing []AvailZoneInfo, newAZBySubnet map[string]string) []AvailZoneInfo {
|
||||
bySubnet := make(map[string]AvailZoneInfo, len(existing))
|
||||
for _, az := range existing {
|
||||
bySubnet[az.SubnetId] = az
|
||||
}
|
||||
out := make([]AvailZoneInfo, 0, len(subnets))
|
||||
for _, sn := range subnets {
|
||||
if az, ok := bySubnet[sn]; ok {
|
||||
out = append(out, AvailZoneInfo{ZoneName: az.ZoneName, SubnetId: sn})
|
||||
continue
|
||||
}
|
||||
out = append(out, AvailZoneInfo{ZoneName: newAZBySubnet[sn], SubnetId: sn})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// lbStateAfterLaunch mirrors CreateLoadBalancer's post-launch state gate: a VM
|
||||
// that came up enters provisioning until the lb-agent heartbeats; an internal
|
||||
// LB with no mgmt return route is failed loud; a failed launch is failed.
|
||||
func (s *ELBv2ServiceImpl) lbStateAfterLaunch(launch lbVMLaunch, scheme string) string {
|
||||
if launch.instanceID == "" {
|
||||
if launch.failed {
|
||||
return StateFailed
|
||||
}
|
||||
return StateActive
|
||||
}
|
||||
if scheme == SchemeInternal {
|
||||
if gw, tgt := s.resolveMgmtRoute(scheme); gw == "" || tgt == "" {
|
||||
slog.Error("SetSubnets: internal LB has no mgmt return route; marking failed (lb-agent cannot heartbeat AWSGW)",
|
||||
"mgmtBridgeIP", s.MgmtBridgeIP, "advertiseIP", s.AdvertiseIP)
|
||||
return StateFailed
|
||||
}
|
||||
}
|
||||
return StateProvisioning
|
||||
}
|
||||
|
||||
// setSubnetsOutput builds the SetSubnets response from the persisted record,
|
||||
// reusing the SDK availability-zone projection (which surfaces per-AZ private
|
||||
// IPs) so the response matches DescribeLoadBalancers.
|
||||
func (s *ELBv2ServiceImpl) setSubnetsOutput(lb *LoadBalancerRecord) *elbv2.SetSubnetsOutput {
|
||||
sdk := s.lbRecordToSDK(lb)
|
||||
return &elbv2.SetSubnetsOutput{
|
||||
AvailabilityZones: sdk.AvailabilityZones,
|
||||
IpAddressType: aws.String(lb.IPAddressType),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
package handlers_elbv2
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/aws/aws-sdk-go/service/elbv2"
|
||||
"github.com/mulgadc/spinifex/spinifex/awserrors"
|
||||
handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -134,3 +137,263 @@ func TestSetSecurityGroups_CrossAccount(t *testing.T) {
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), awserrors.ErrorELBv2LoadBalancerNotFound)
|
||||
}
|
||||
|
||||
// setupSubnetTestService wires a VPC-backed ELBv2 service with a launcher mock
|
||||
// so SetSubnets can exercise ENI create/delete plus the LB-VM relaunch.
|
||||
func setupSubnetTestService(t *testing.T) (*ELBv2ServiceImpl, *handlers_ec2_vpc.VPCServiceImpl, *mockSystemInstanceLauncher) {
|
||||
t.Helper()
|
||||
svc, vpcSvc := setupTestServiceWithVPC(t)
|
||||
mock := &mockSystemInstanceLauncher{
|
||||
launchResult: &SystemInstanceOutput{InstanceID: "i-lb", PrivateIP: "10.0.1.5"},
|
||||
}
|
||||
svc.InstanceLauncher = mock
|
||||
svc.GatewayURL = "https://10.0.0.1:9999"
|
||||
svc.SystemAccessKey = "AKID"
|
||||
svc.SystemSecretKey = "SECRET"
|
||||
return svc, vpcSvc, mock
|
||||
}
|
||||
|
||||
func countManagedENIs(t *testing.T, vpcSvc *handlers_ec2_vpc.VPCServiceImpl) int {
|
||||
t.Helper()
|
||||
out, err := vpcSvc.DescribeNetworkInterfaces(&ec2.DescribeNetworkInterfacesInput{}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
n := 0
|
||||
for _, eni := range out.NetworkInterfaces {
|
||||
if eni.RequesterManaged != nil && *eni.RequesterManaged {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func vpcID(t *testing.T, vpcSvc *handlers_ec2_vpc.VPCServiceImpl) string {
|
||||
t.Helper()
|
||||
vpcs, err := vpcSvc.DescribeVpcs(&ec2.DescribeVpcsInput{}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, vpcs.Vpcs)
|
||||
return *vpcs.Vpcs[0].VpcId
|
||||
}
|
||||
|
||||
func TestSetSubnets_AddSubnet(t *testing.T) {
|
||||
svc, vpcSvc, mock := setupSubnetTestService(t)
|
||||
vid := vpcID(t, vpcSvc)
|
||||
sub1 := getTestSubnetID(t, vpcSvc, vid, "10.0.20.0/24", "us-east-1a")
|
||||
sub2 := getTestSubnetID(t, vpcSvc, vid, "10.0.21.0/24", "us-east-1b")
|
||||
|
||||
out, err := svc.CreateLoadBalancer(&elbv2.CreateLoadBalancerInput{
|
||||
Name: aws.String("add-lb"),
|
||||
Subnets: []*string{aws.String(sub1)},
|
||||
}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
arn := *out.LoadBalancers[0].LoadBalancerArn
|
||||
require.Equal(t, 1, countManagedENIs(t, vpcSvc))
|
||||
require.Len(t, mock.launchCalls, 1)
|
||||
|
||||
res, err := svc.SetSubnets(&elbv2.SetSubnetsInput{
|
||||
LoadBalancerArn: aws.String(arn),
|
||||
Subnets: []*string{aws.String(sub1), aws.String(sub2)},
|
||||
}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, res.AvailabilityZones, 2)
|
||||
|
||||
// New ENI created in sub2; LB VM relaunched on the new set.
|
||||
assert.Equal(t, 2, countManagedENIs(t, vpcSvc))
|
||||
assert.Len(t, mock.terminateCalls, 1)
|
||||
require.Len(t, mock.launchCalls, 2)
|
||||
assert.Len(t, mock.launchCalls[1].ExtraENIs, 1, "relaunch must carry the added subnet as an ExtraENI")
|
||||
|
||||
lb := describeLB(t, svc, arn)
|
||||
assert.ElementsMatch(t, []string{sub1, sub2}, subnetIDsOfLB(lb))
|
||||
}
|
||||
|
||||
func TestSetSubnets_RemoveSubnet(t *testing.T) {
|
||||
svc, vpcSvc, mock := setupSubnetTestService(t)
|
||||
vid := vpcID(t, vpcSvc)
|
||||
sub1 := getTestSubnetID(t, vpcSvc, vid, "10.0.22.0/24", "us-east-1a")
|
||||
sub2 := getTestSubnetID(t, vpcSvc, vid, "10.0.23.0/24", "us-east-1b")
|
||||
|
||||
out, err := svc.CreateLoadBalancer(&elbv2.CreateLoadBalancerInput{
|
||||
Name: aws.String("rm-lb"),
|
||||
Subnets: []*string{aws.String(sub1), aws.String(sub2)},
|
||||
}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
arn := *out.LoadBalancers[0].LoadBalancerArn
|
||||
require.Equal(t, 2, countManagedENIs(t, vpcSvc))
|
||||
|
||||
res, err := svc.SetSubnets(&elbv2.SetSubnetsInput{
|
||||
LoadBalancerArn: aws.String(arn),
|
||||
Subnets: []*string{aws.String(sub1)},
|
||||
}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, res.AvailabilityZones, 1)
|
||||
|
||||
// sub2's ENI is deleted; only sub1's remains.
|
||||
assert.Equal(t, 1, countManagedENIs(t, vpcSvc))
|
||||
assert.Len(t, mock.terminateCalls, 1)
|
||||
lb := describeLB(t, svc, arn)
|
||||
assert.Equal(t, []string{sub1}, subnetIDsOfLB(lb))
|
||||
}
|
||||
|
||||
func TestSetSubnets_Replace(t *testing.T) {
|
||||
svc, vpcSvc, _ := setupSubnetTestService(t)
|
||||
vid := vpcID(t, vpcSvc)
|
||||
sub1 := getTestSubnetID(t, vpcSvc, vid, "10.0.24.0/24", "us-east-1a")
|
||||
sub2 := getTestSubnetID(t, vpcSvc, vid, "10.0.25.0/24", "us-east-1b")
|
||||
|
||||
out, err := svc.CreateLoadBalancer(&elbv2.CreateLoadBalancerInput{
|
||||
Name: aws.String("swap-lb"),
|
||||
Subnets: []*string{aws.String(sub1)},
|
||||
}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
arn := *out.LoadBalancers[0].LoadBalancerArn
|
||||
|
||||
_, err = svc.SetSubnets(&elbv2.SetSubnetsInput{
|
||||
LoadBalancerArn: aws.String(arn),
|
||||
Subnets: []*string{aws.String(sub2)},
|
||||
}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
// One ENI swapped for another — still a single managed ENI, now in sub2.
|
||||
assert.Equal(t, 1, countManagedENIs(t, vpcSvc))
|
||||
lb := describeLB(t, svc, arn)
|
||||
assert.Equal(t, []string{sub2}, subnetIDsOfLB(lb))
|
||||
}
|
||||
|
||||
func TestSetSubnets_TerminateFailureRollsBackNewENIs(t *testing.T) {
|
||||
svc, vpcSvc, mock := setupSubnetTestService(t)
|
||||
vid := vpcID(t, vpcSvc)
|
||||
sub1 := getTestSubnetID(t, vpcSvc, vid, "10.0.26.0/24", "us-east-1a")
|
||||
sub2 := getTestSubnetID(t, vpcSvc, vid, "10.0.27.0/24", "us-east-1b")
|
||||
|
||||
out, err := svc.CreateLoadBalancer(&elbv2.CreateLoadBalancerInput{
|
||||
Name: aws.String("rollback-lb"),
|
||||
Subnets: []*string{aws.String(sub1)},
|
||||
}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
arn := *out.LoadBalancers[0].LoadBalancerArn
|
||||
require.Equal(t, 1, countManagedENIs(t, vpcSvc))
|
||||
|
||||
// Terminate fails mid-relaunch (e.g. owning daemon unreachable). The ENI
|
||||
// created for the added subnet must be rolled back, not leaked.
|
||||
mock.terminateErr = errors.New("no responders available")
|
||||
|
||||
_, err = svc.SetSubnets(&elbv2.SetSubnetsInput{
|
||||
LoadBalancerArn: aws.String(arn),
|
||||
Subnets: []*string{aws.String(sub1), aws.String(sub2)},
|
||||
}, testAccountID)
|
||||
require.Error(t, err)
|
||||
|
||||
// No leak: still the single original ENI; LB record unchanged; no relaunch.
|
||||
assert.Equal(t, 1, countManagedENIs(t, vpcSvc))
|
||||
assert.Len(t, mock.launchCalls, 1)
|
||||
lb := describeLB(t, svc, arn)
|
||||
assert.Equal(t, []string{sub1}, subnetIDsOfLB(lb))
|
||||
}
|
||||
|
||||
func TestSetSubnets_Idempotent(t *testing.T) {
|
||||
svc, vpcSvc, mock := setupSubnetTestService(t)
|
||||
vid := vpcID(t, vpcSvc)
|
||||
sub1 := getTestSubnetID(t, vpcSvc, vid, "10.0.26.0/24", "us-east-1a")
|
||||
|
||||
out, err := svc.CreateLoadBalancer(&elbv2.CreateLoadBalancerInput{
|
||||
Name: aws.String("idem-lb"),
|
||||
Subnets: []*string{aws.String(sub1)},
|
||||
}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
arn := *out.LoadBalancers[0].LoadBalancerArn
|
||||
|
||||
_, err = svc.SetSubnets(&elbv2.SetSubnetsInput{
|
||||
LoadBalancerArn: aws.String(arn),
|
||||
Subnets: []*string{aws.String(sub1)},
|
||||
}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
// No change: no relaunch, no terminate, ENI count unchanged.
|
||||
assert.Equal(t, 1, countManagedENIs(t, vpcSvc))
|
||||
assert.Empty(t, mock.terminateCalls)
|
||||
assert.Len(t, mock.launchCalls, 1)
|
||||
}
|
||||
|
||||
func TestSetSubnets_SubnetMappings(t *testing.T) {
|
||||
svc, vpcSvc, _ := setupSubnetTestService(t)
|
||||
vid := vpcID(t, vpcSvc)
|
||||
sub1 := getTestSubnetID(t, vpcSvc, vid, "10.0.27.0/24", "us-east-1a")
|
||||
sub2 := getTestSubnetID(t, vpcSvc, vid, "10.0.28.0/24", "us-east-1b")
|
||||
|
||||
out, err := svc.CreateLoadBalancer(&elbv2.CreateLoadBalancerInput{
|
||||
Name: aws.String("mapping-lb"),
|
||||
Subnets: []*string{aws.String(sub1)},
|
||||
}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
arn := *out.LoadBalancers[0].LoadBalancerArn
|
||||
|
||||
_, err = svc.SetSubnets(&elbv2.SetSubnetsInput{
|
||||
LoadBalancerArn: aws.String(arn),
|
||||
SubnetMappings: []*elbv2.SubnetMapping{
|
||||
{SubnetId: aws.String(sub1)},
|
||||
{SubnetId: aws.String(sub2)},
|
||||
},
|
||||
}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 2, countManagedENIs(t, vpcSvc))
|
||||
}
|
||||
|
||||
func TestSetSubnets_WithoutVPCService(t *testing.T) {
|
||||
svc := setupTestService(t)
|
||||
arn := createLBArn(t, svc, "novpc-subnets")
|
||||
|
||||
res, err := svc.SetSubnets(&elbv2.SetSubnetsInput{
|
||||
LoadBalancerArn: aws.String(arn),
|
||||
Subnets: []*string{aws.String("subnet-aaa"), aws.String("subnet-bbb")},
|
||||
}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, res.AvailabilityZones, 2)
|
||||
assert.ElementsMatch(t, []string{"subnet-aaa", "subnet-bbb"}, subnetIDsOfLB(describeLB(t, svc, arn)))
|
||||
}
|
||||
|
||||
func TestSetSubnets_MissingParams(t *testing.T) {
|
||||
svc := setupTestService(t)
|
||||
arn := createLBArn(t, svc, "subnets-mp")
|
||||
|
||||
_, err := svc.SetSubnets(&elbv2.SetSubnetsInput{
|
||||
LoadBalancerArn: aws.String(arn),
|
||||
}, testAccountID)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), awserrors.ErrorMissingParameter)
|
||||
|
||||
_, err = svc.SetSubnets(&elbv2.SetSubnetsInput{
|
||||
Subnets: []*string{aws.String("subnet-aaa")},
|
||||
}, testAccountID)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), awserrors.ErrorMissingParameter)
|
||||
}
|
||||
|
||||
func TestSetSubnets_NotFound(t *testing.T) {
|
||||
svc := setupTestService(t)
|
||||
_, err := svc.SetSubnets(&elbv2.SetSubnetsInput{
|
||||
LoadBalancerArn: aws.String("arn:aws:elasticloadbalancing:us-east-1:123456789012:loadbalancer/app/missing/lb-deadbeef"),
|
||||
Subnets: []*string{aws.String("subnet-aaa")},
|
||||
}, testAccountID)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), awserrors.ErrorELBv2LoadBalancerNotFound)
|
||||
}
|
||||
|
||||
func TestSetSubnets_CrossAccount(t *testing.T) {
|
||||
svc := setupTestService(t)
|
||||
arn := createLBArn(t, svc, "subnets-xacct")
|
||||
|
||||
_, err := svc.SetSubnets(&elbv2.SetSubnetsInput{
|
||||
LoadBalancerArn: aws.String(arn),
|
||||
Subnets: []*string{aws.String("subnet-aaa")},
|
||||
}, "999999999999")
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), awserrors.ErrorELBv2LoadBalancerNotFound)
|
||||
}
|
||||
|
||||
func subnetIDsOfLB(lb *elbv2.LoadBalancer) []string {
|
||||
out := make([]string, 0, len(lb.AvailabilityZones))
|
||||
for _, az := range lb.AvailabilityZones {
|
||||
out = append(out, aws.StringValue(az.SubnetId))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -0,0 +1,252 @@
|
||||
package handlers_elbv2
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/spinifex/spinifex/awserrors"
|
||||
)
|
||||
|
||||
// nlbManagedSGName returns the deterministic name of the internal security
|
||||
// group minted for an NLB. Stable so recovery paths can re-resolve it.
|
||||
func nlbManagedSGName(lbID string) string {
|
||||
return "spinifex-nlb-" + lbID
|
||||
}
|
||||
|
||||
// lbENIGroups returns the security groups to attach to an LB's data-plane ENIs.
|
||||
// NLBs use the internally-minted managed SG (customer SGs are rejected at
|
||||
// create time); ALBs use the customer-supplied SGs. Shared by the create-time
|
||||
// ENI loop and the SetSubnets relaunch path so both attach the same SG.
|
||||
func lbENIGroups(lb *LoadBalancerRecord) []string {
|
||||
if lb.Type == LoadBalancerTypeNetwork {
|
||||
if lb.NLBManagedSGID != "" {
|
||||
return []string{lb.NLBManagedSGID}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return lb.SecurityGroups
|
||||
}
|
||||
|
||||
// createNLBManagedSG mints the internal front-end SG for an NLB in the LB's VPC
|
||||
// and returns its ID. The VPC is resolved from the LB's first subnet. The SG is
|
||||
// tagged like the LB's other managed resources so the ownership sweep can find
|
||||
// it.
|
||||
func (s *ELBv2ServiceImpl) createNLBManagedSG(lbID, lbArn, subnetID, accountID string) (string, error) {
|
||||
subnet, err := s.VPCService.GetSubnet(accountID, subnetID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("resolve subnet %s vpc: %w", subnetID, err)
|
||||
}
|
||||
if subnet == nil || subnet.VpcId == "" {
|
||||
return "", fmt.Errorf("subnet %s has no vpc", subnetID)
|
||||
}
|
||||
|
||||
out, err := s.VPCService.CreateSecurityGroup(&ec2.CreateSecurityGroupInput{
|
||||
GroupName: aws.String(nlbManagedSGName(lbID)),
|
||||
Description: aws.String(fmt.Sprintf("Managed front-end SG for NLB %s", lbID)),
|
||||
VpcId: aws.String(subnet.VpcId),
|
||||
TagSpecifications: []*ec2.TagSpecification{{
|
||||
ResourceType: aws.String("security-group"),
|
||||
Tags: []*ec2.Tag{
|
||||
{Key: aws.String(elbv2ManagedByTag), Value: aws.String(elbv2ManagedByValue)},
|
||||
{Key: aws.String(elbv2LBTag), Value: aws.String(lbArn)},
|
||||
},
|
||||
}},
|
||||
}, accountID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create NLB SG: %w", err)
|
||||
}
|
||||
if out == nil || out.GroupId == nil || *out.GroupId == "" {
|
||||
return "", errors.New("CreateSecurityGroup returned empty GroupId")
|
||||
}
|
||||
return *out.GroupId, nil
|
||||
}
|
||||
|
||||
// deleteNLBManagedSG best-effort deletes an NLB's managed SG. Callable on the
|
||||
// create-rollback and delete paths; a nil VPC service or empty ID is a no-op,
|
||||
// and a delete failure is logged but not surfaced (the SG can only be deleted
|
||||
// once its ENIs are gone, which the delete path guarantees).
|
||||
func (s *ELBv2ServiceImpl) deleteNLBManagedSG(sgID, accountID string) {
|
||||
if s.VPCService == nil || sgID == "" {
|
||||
return
|
||||
}
|
||||
if _, err := s.VPCService.DeleteSecurityGroup(&ec2.DeleteSecurityGroupInput{
|
||||
GroupId: aws.String(sgID),
|
||||
}, accountID); err != nil {
|
||||
slog.Warn("deleteNLBManagedSG: delete failed", "sg", sgID, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// resolveNLBIngressCIDRs returns the client CIDRs a listener port is opened to.
|
||||
// An explicit override wins; otherwise the default is scheme-based:
|
||||
// internet-facing → 0.0.0.0/0, internal → the LB's VPC CIDR.
|
||||
func (s *ELBv2ServiceImpl) resolveNLBIngressCIDRs(lb *LoadBalancerRecord) ([]string, error) {
|
||||
if len(lb.NLBIngressCIDRs) > 0 {
|
||||
return lb.NLBIngressCIDRs, nil
|
||||
}
|
||||
if lb.Scheme == SchemeInternal {
|
||||
cidr, err := s.vpcCIDR(lb.VpcId, lb.AccountID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []string{cidr}, nil
|
||||
}
|
||||
return []string{"0.0.0.0/0"}, nil
|
||||
}
|
||||
|
||||
// vpcCIDR returns the primary CIDR block of a VPC.
|
||||
func (s *ELBv2ServiceImpl) vpcCIDR(vpcID, accountID string) (string, error) {
|
||||
if s.VPCService == nil {
|
||||
return "", errors.New("vpc service unavailable")
|
||||
}
|
||||
if vpcID == "" {
|
||||
return "", errors.New("empty vpc id")
|
||||
}
|
||||
out, err := s.VPCService.DescribeVpcs(&ec2.DescribeVpcsInput{
|
||||
VpcIds: aws.StringSlice([]string{vpcID}),
|
||||
}, accountID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("describe vpc %s: %w", vpcID, err)
|
||||
}
|
||||
for _, v := range out.Vpcs {
|
||||
if v != nil && aws.StringValue(v.CidrBlock) != "" {
|
||||
return *v.CidrBlock, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("vpc %s has no cidr", vpcID)
|
||||
}
|
||||
|
||||
// listenerIPProtocols maps an ELBv2 listener protocol to the IP protocols its
|
||||
// SG ingress rule(s) cover. TCP/TLS ride tcp; UDP rides udp; TCP_UDP opens both.
|
||||
func listenerIPProtocols(protocol string) []string {
|
||||
switch protocol {
|
||||
case ProtocolUDP:
|
||||
return []string{"udp"}
|
||||
case ProtocolTCPUDP:
|
||||
return []string{"tcp", "udp"}
|
||||
default: // TCP, TLS
|
||||
return []string{"tcp"}
|
||||
}
|
||||
}
|
||||
|
||||
// authorizeNLBListenerPort opens a listener port on the NLB's managed SG for
|
||||
// each (protocol, CIDR) pair. Each rule is authorized in its own call so an
|
||||
// already-present rule (Duplicate, swallowed) never blocks a sibling rule —
|
||||
// keeping re-create / reconcile idempotent. No-op when there is no managed SG.
|
||||
func (s *ELBv2ServiceImpl) authorizeNLBListenerPort(lb *LoadBalancerRecord, protocol string, port int64, cidrs []string, accountID string) error {
|
||||
if s.VPCService == nil || lb.NLBManagedSGID == "" {
|
||||
return nil
|
||||
}
|
||||
for _, proto := range listenerIPProtocols(protocol) {
|
||||
for _, cidr := range cidrs {
|
||||
_, err := s.VPCService.AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{
|
||||
GroupId: aws.String(lb.NLBManagedSGID),
|
||||
IpPermissions: []*ec2.IpPermission{{
|
||||
IpProtocol: aws.String(proto),
|
||||
FromPort: aws.Int64(port),
|
||||
ToPort: aws.Int64(port),
|
||||
IpRanges: []*ec2.IpRange{{
|
||||
CidrIp: aws.String(cidr),
|
||||
Description: aws.String(fmt.Sprintf("NLB %s listener :%d", lb.LoadBalancerID, port)),
|
||||
}},
|
||||
}},
|
||||
}, accountID)
|
||||
if err != nil && !awserrors.IsErrorCode(err, awserrors.ErrorInvalidPermissionDuplicate) {
|
||||
return fmt.Errorf("authorize :%d/%s from %s: %w", port, proto, cidr, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// revokeNLBListenerPort removes the listener-port rules authorizeNLBListenerPort
|
||||
// added. An absent rule (NotFound, swallowed) is fine so delete / re-CIDR paths
|
||||
// stay idempotent. No-op when there is no managed SG.
|
||||
func (s *ELBv2ServiceImpl) revokeNLBListenerPort(lb *LoadBalancerRecord, protocol string, port int64, cidrs []string, accountID string) error {
|
||||
if s.VPCService == nil || lb.NLBManagedSGID == "" {
|
||||
return nil
|
||||
}
|
||||
for _, proto := range listenerIPProtocols(protocol) {
|
||||
for _, cidr := range cidrs {
|
||||
_, err := s.VPCService.RevokeSecurityGroupIngress(&ec2.RevokeSecurityGroupIngressInput{
|
||||
GroupId: aws.String(lb.NLBManagedSGID),
|
||||
IpPermissions: []*ec2.IpPermission{{
|
||||
IpProtocol: aws.String(proto),
|
||||
FromPort: aws.Int64(port),
|
||||
ToPort: aws.Int64(port),
|
||||
IpRanges: []*ec2.IpRange{{CidrIp: aws.String(cidr)}},
|
||||
}},
|
||||
}, accountID)
|
||||
if err != nil && !awserrors.IsErrorCode(err, awserrors.ErrorInvalidPermissionNotFound) {
|
||||
return fmt.Errorf("revoke :%d/%s from %s: %w", port, proto, cidr, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetLoadBalancerIngressCIDRs overrides the client CIDRs the NLB's listener
|
||||
// ports are opened to (e.g. an EKS public endpoint's publicAccessCidrs) and
|
||||
// re-authorizes every existing listener accordingly. Passing nil/empty reverts
|
||||
// to the scheme-based default. Idempotent: the old rules are revoked and the
|
||||
// new ones authorized, both swallowing the benign duplicate / not-found errors.
|
||||
func (s *ELBv2ServiceImpl) SetLoadBalancerIngressCIDRs(lbArn string, cidrs []string, accountID string) error {
|
||||
if lbArn == "" {
|
||||
return errors.New(awserrors.ErrorMissingParameter)
|
||||
}
|
||||
lb, err := s.store.GetLoadBalancerByArn(lbArn)
|
||||
if err != nil {
|
||||
slog.Error("SetLoadBalancerIngressCIDRs: failed to get LB", "arn", lbArn, "err", err)
|
||||
return errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
if lb == nil {
|
||||
return errors.New(awserrors.ErrorELBv2LoadBalancerNotFound)
|
||||
}
|
||||
if lb.Type != LoadBalancerTypeNetwork {
|
||||
return errors.New(awserrors.ErrorELBv2InvalidConfigurationRequest)
|
||||
}
|
||||
for _, c := range cidrs {
|
||||
if _, _, perr := net.ParseCIDR(c); perr != nil {
|
||||
return errors.New(awserrors.ErrorInvalidParameterValue)
|
||||
}
|
||||
}
|
||||
|
||||
oldCIDRs, err := s.resolveNLBIngressCIDRs(lb)
|
||||
if err != nil {
|
||||
slog.Error("SetLoadBalancerIngressCIDRs: resolve old CIDRs failed", "arn", lbArn, "err", err)
|
||||
return errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
listeners, err := s.store.ListListenersByLB(lb.LoadBalancerArn)
|
||||
if err != nil {
|
||||
slog.Error("SetLoadBalancerIngressCIDRs: list listeners failed", "arn", lbArn, "err", err)
|
||||
return errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
|
||||
lb.NLBIngressCIDRs = cidrs
|
||||
newCIDRs, err := s.resolveNLBIngressCIDRs(lb)
|
||||
if err != nil {
|
||||
slog.Error("SetLoadBalancerIngressCIDRs: resolve new CIDRs failed", "arn", lbArn, "err", err)
|
||||
return errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
|
||||
for _, l := range listeners {
|
||||
if err := s.revokeNLBListenerPort(lb, l.Protocol, l.Port, oldCIDRs, accountID); err != nil {
|
||||
slog.Error("SetLoadBalancerIngressCIDRs: revoke failed", "arn", lbArn, "port", l.Port, "err", err)
|
||||
return errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
if err := s.authorizeNLBListenerPort(lb, l.Protocol, l.Port, newCIDRs, accountID); err != nil {
|
||||
slog.Error("SetLoadBalancerIngressCIDRs: authorize failed", "arn", lbArn, "port", l.Port, "err", err)
|
||||
return errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
}
|
||||
|
||||
if err := s.store.PutLoadBalancer(lb); err != nil {
|
||||
slog.Error("SetLoadBalancerIngressCIDRs: persist failed", "arn", lbArn, "err", err)
|
||||
return errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
slog.Info("SetLoadBalancerIngressCIDRs completed", "arn", lbArn, "cidrs", cidrs, "listeners", len(listeners), "accountID", accountID)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,298 @@
|
||||
package handlers_elbv2
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/aws/aws-sdk-go/service/elbv2"
|
||||
handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// firstSubnet returns the ID + VPC ID of the subnet setupTestServiceWithVPC
|
||||
// pre-creates.
|
||||
func firstSubnet(t *testing.T, vpcSvc *handlers_ec2_vpc.VPCServiceImpl) (subnetID, vpcID string) {
|
||||
t.Helper()
|
||||
subnets, err := vpcSvc.DescribeSubnets(&ec2.DescribeSubnetsInput{}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, subnets.Subnets)
|
||||
return *subnets.Subnets[0].SubnetId, *subnets.Subnets[0].VpcId
|
||||
}
|
||||
|
||||
// describeSG returns the security group record by ID.
|
||||
func describeSG(t *testing.T, vpcSvc *handlers_ec2_vpc.VPCServiceImpl, sgID string) *ec2.SecurityGroup {
|
||||
t.Helper()
|
||||
out, err := vpcSvc.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{
|
||||
GroupIds: aws.StringSlice([]string{sgID}),
|
||||
}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, out.SecurityGroups, 1)
|
||||
return out.SecurityGroups[0]
|
||||
}
|
||||
|
||||
// sgHasRule reports whether the SG has an ingress rule for the proto/port/cidr.
|
||||
func sgHasRule(sg *ec2.SecurityGroup, proto string, port int64, cidr string) bool {
|
||||
for _, p := range sg.IpPermissions {
|
||||
if aws.StringValue(p.IpProtocol) != proto {
|
||||
continue
|
||||
}
|
||||
if aws.Int64Value(p.FromPort) != port || aws.Int64Value(p.ToPort) != port {
|
||||
continue
|
||||
}
|
||||
for _, r := range p.IpRanges {
|
||||
if aws.StringValue(r.CidrIp) == cidr {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func managedENI(t *testing.T, vpcSvc *handlers_ec2_vpc.VPCServiceImpl) *ec2.NetworkInterface {
|
||||
t.Helper()
|
||||
out, err := vpcSvc.DescribeNetworkInterfaces(&ec2.DescribeNetworkInterfacesInput{}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
for _, eni := range out.NetworkInterfaces {
|
||||
if eni.RequesterManaged != nil && *eni.RequesterManaged {
|
||||
return eni
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func createNLB(t *testing.T, svc *ELBv2ServiceImpl, name, scheme, subnetID string) *elbv2.LoadBalancer {
|
||||
t.Helper()
|
||||
in := &elbv2.CreateLoadBalancerInput{
|
||||
Name: aws.String(name),
|
||||
Type: aws.String("network"),
|
||||
Subnets: []*string{aws.String(subnetID)},
|
||||
}
|
||||
if scheme != "" {
|
||||
in.Scheme = aws.String(scheme)
|
||||
}
|
||||
out, err := svc.CreateLoadBalancer(in, testAccountID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, out.LoadBalancers, 1)
|
||||
return out.LoadBalancers[0]
|
||||
}
|
||||
|
||||
func createTCPListener(t *testing.T, svc *ELBv2ServiceImpl, lbArn *string, port int64) {
|
||||
t.Helper()
|
||||
tg, err := svc.CreateTargetGroup(&elbv2.CreateTargetGroupInput{
|
||||
Name: aws.String("tg-nlb-sg"),
|
||||
Protocol: aws.String(elbv2.ProtocolEnumTcp),
|
||||
Port: aws.Int64(port),
|
||||
}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
_, err = svc.CreateListener(&elbv2.CreateListenerInput{
|
||||
LoadBalancerArn: lbArn,
|
||||
Protocol: aws.String(elbv2.ProtocolEnumTcp),
|
||||
Port: aws.Int64(port),
|
||||
DefaultActions: []*elbv2.Action{{Type: aws.String("forward"), TargetGroupArn: tg.TargetGroups[0].TargetGroupArn}},
|
||||
}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// TestCreateNLB_MintsManagedSGAttachedToENI verifies a network LB gets a
|
||||
// dedicated internal SG (not the VPC default) attached to its ENI, recorded in
|
||||
// NLBManagedSGID, while DescribeLoadBalancers still reports no customer SGs.
|
||||
func TestCreateNLB_MintsManagedSGAttachedToENI(t *testing.T) {
|
||||
svc, vpcSvc := setupTestServiceWithVPC(t)
|
||||
subnetID, vpcID := firstSubnet(t, vpcSvc)
|
||||
|
||||
defaultSGID, err := vpcSvc.FindDefaultSGForVPC(testAccountID, vpcID)
|
||||
require.NoError(t, err)
|
||||
|
||||
lb := createNLB(t, svc, "nlb-sg", "internal", subnetID)
|
||||
assert.Empty(t, lb.SecurityGroups, "NLB must not surface the managed SG as a customer SG")
|
||||
|
||||
rec, err := svc.store.GetLoadBalancerByArn(*lb.LoadBalancerArn)
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, rec.NLBManagedSGID, "NLB record must carry the managed SG id")
|
||||
assert.NotEqual(t, defaultSGID, rec.NLBManagedSGID, "managed SG must be distinct from the VPC default SG")
|
||||
|
||||
eni := managedENI(t, vpcSvc)
|
||||
require.NotNil(t, eni)
|
||||
require.Len(t, eni.Groups, 1, "NLB ENI must join exactly the managed SG, not the default SG")
|
||||
assert.Equal(t, rec.NLBManagedSGID, *eni.Groups[0].GroupId)
|
||||
}
|
||||
|
||||
// TestCreateALB_NoManagedSG verifies the managed-SG behavior is NLB-only: an ALB
|
||||
// keeps the existing customer-SG / default-SG semantics.
|
||||
func TestCreateALB_NoManagedSG(t *testing.T) {
|
||||
svc, vpcSvc := setupTestServiceWithVPC(t)
|
||||
subnetID, _ := firstSubnet(t, vpcSvc)
|
||||
|
||||
out, err := svc.CreateLoadBalancer(&elbv2.CreateLoadBalancerInput{
|
||||
Name: aws.String("alb-no-sg"),
|
||||
Subnets: []*string{aws.String(subnetID)},
|
||||
}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
rec, err := svc.store.GetLoadBalancerByArn(*out.LoadBalancers[0].LoadBalancerArn)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, rec.NLBManagedSGID, "ALB must not mint a managed NLB SG")
|
||||
}
|
||||
|
||||
// TestCreateListener_Internal_OpensPortFromVPCCIDR verifies an internal NLB's
|
||||
// listener port is authorized on the managed SG from the VPC CIDR.
|
||||
func TestCreateListener_Internal_OpensPortFromVPCCIDR(t *testing.T) {
|
||||
svc, vpcSvc := setupTestServiceWithVPC(t)
|
||||
subnetID, _ := firstSubnet(t, vpcSvc)
|
||||
|
||||
lb := createNLB(t, svc, "nlb-int", "internal", subnetID)
|
||||
createTCPListener(t, svc, lb.LoadBalancerArn, 443)
|
||||
|
||||
rec, err := svc.store.GetLoadBalancerByArn(*lb.LoadBalancerArn)
|
||||
require.NoError(t, err)
|
||||
sg := describeSG(t, vpcSvc, rec.NLBManagedSGID)
|
||||
assert.True(t, sgHasRule(sg, "tcp", 443, "10.0.0.0/16"),
|
||||
"internal NLB listener :443 must be open from the VPC CIDR")
|
||||
assert.False(t, sgHasRule(sg, "tcp", 443, "0.0.0.0/0"))
|
||||
}
|
||||
|
||||
// TestCreateListener_InternetFacing_OpensPortFromAnywhere verifies an
|
||||
// internet-facing NLB's listener port is authorized from 0.0.0.0/0.
|
||||
func TestCreateListener_InternetFacing_OpensPortFromAnywhere(t *testing.T) {
|
||||
svc, vpcSvc := setupTestServiceWithVPC(t)
|
||||
subnetID, _ := firstSubnet(t, vpcSvc)
|
||||
|
||||
lb := createNLB(t, svc, "nlb-pub", "internet-facing", subnetID)
|
||||
createTCPListener(t, svc, lb.LoadBalancerArn, 443)
|
||||
|
||||
rec, err := svc.store.GetLoadBalancerByArn(*lb.LoadBalancerArn)
|
||||
require.NoError(t, err)
|
||||
sg := describeSG(t, vpcSvc, rec.NLBManagedSGID)
|
||||
assert.True(t, sgHasRule(sg, "tcp", 443, "0.0.0.0/0"),
|
||||
"internet-facing NLB listener :443 must be open from 0.0.0.0/0")
|
||||
}
|
||||
|
||||
// TestCreateListener_TCPUDP_OpensBothProtocols verifies a TCP_UDP listener opens
|
||||
// both tcp and udp on the listener port.
|
||||
func TestCreateListener_TCPUDP_OpensBothProtocols(t *testing.T) {
|
||||
svc, vpcSvc := setupTestServiceWithVPC(t)
|
||||
subnetID, _ := firstSubnet(t, vpcSvc)
|
||||
|
||||
lb := createNLB(t, svc, "nlb-tcpudp", "internet-facing", subnetID)
|
||||
tg, err := svc.CreateTargetGroup(&elbv2.CreateTargetGroupInput{
|
||||
Name: aws.String("tg-tcpudp"),
|
||||
Protocol: aws.String(elbv2.ProtocolEnumTcpUdp),
|
||||
Port: aws.Int64(53),
|
||||
}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
_, err = svc.CreateListener(&elbv2.CreateListenerInput{
|
||||
LoadBalancerArn: lb.LoadBalancerArn,
|
||||
Protocol: aws.String(elbv2.ProtocolEnumTcpUdp),
|
||||
Port: aws.Int64(53),
|
||||
DefaultActions: []*elbv2.Action{{Type: aws.String("forward"), TargetGroupArn: tg.TargetGroups[0].TargetGroupArn}},
|
||||
}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
rec, err := svc.store.GetLoadBalancerByArn(*lb.LoadBalancerArn)
|
||||
require.NoError(t, err)
|
||||
sg := describeSG(t, vpcSvc, rec.NLBManagedSGID)
|
||||
assert.True(t, sgHasRule(sg, "tcp", 53, "0.0.0.0/0"))
|
||||
assert.True(t, sgHasRule(sg, "udp", 53, "0.0.0.0/0"))
|
||||
}
|
||||
|
||||
// TestSetLoadBalancerIngressCIDRs_RewritesListenerRules verifies the override API
|
||||
// swaps the scheme-default rule for the supplied CIDRs across existing listeners
|
||||
// and is idempotent on re-invocation.
|
||||
func TestSetLoadBalancerIngressCIDRs_RewritesListenerRules(t *testing.T) {
|
||||
svc, vpcSvc := setupTestServiceWithVPC(t)
|
||||
subnetID, _ := firstSubnet(t, vpcSvc)
|
||||
|
||||
lb := createNLB(t, svc, "nlb-cidr", "internet-facing", subnetID)
|
||||
createTCPListener(t, svc, lb.LoadBalancerArn, 443)
|
||||
|
||||
rec, err := svc.store.GetLoadBalancerByArn(*lb.LoadBalancerArn)
|
||||
require.NoError(t, err)
|
||||
sgID := rec.NLBManagedSGID
|
||||
|
||||
// Default opened 0.0.0.0/0.
|
||||
assert.True(t, sgHasRule(describeSG(t, vpcSvc, sgID), "tcp", 443, "0.0.0.0/0"))
|
||||
|
||||
require.NoError(t, svc.SetLoadBalancerIngressCIDRs(*lb.LoadBalancerArn, []string{"203.0.113.0/24"}, testAccountID))
|
||||
|
||||
sg := describeSG(t, vpcSvc, sgID)
|
||||
assert.True(t, sgHasRule(sg, "tcp", 443, "203.0.113.0/24"), "new CIDR must be authorized")
|
||||
assert.False(t, sgHasRule(sg, "tcp", 443, "0.0.0.0/0"), "old default rule must be revoked")
|
||||
|
||||
// Idempotent re-apply.
|
||||
require.NoError(t, svc.SetLoadBalancerIngressCIDRs(*lb.LoadBalancerArn, []string{"203.0.113.0/24"}, testAccountID))
|
||||
sg = describeSG(t, vpcSvc, sgID)
|
||||
assert.True(t, sgHasRule(sg, "tcp", 443, "203.0.113.0/24"))
|
||||
|
||||
// Persisted override drives the resolved CIDRs.
|
||||
rec, err = svc.store.GetLoadBalancerByArn(*lb.LoadBalancerArn)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"203.0.113.0/24"}, rec.NLBIngressCIDRs)
|
||||
}
|
||||
|
||||
func TestSetLoadBalancerIngressCIDRs_RejectsBadInput(t *testing.T) {
|
||||
svc, vpcSvc := setupTestServiceWithVPC(t)
|
||||
subnetID, _ := firstSubnet(t, vpcSvc)
|
||||
|
||||
lb := createNLB(t, svc, "nlb-badcidr", "internet-facing", subnetID)
|
||||
err := svc.SetLoadBalancerIngressCIDRs(*lb.LoadBalancerArn, []string{"not-a-cidr"}, testAccountID)
|
||||
require.Error(t, err)
|
||||
|
||||
// ALB is rejected — managed SG is NLB-only.
|
||||
albOut, err := svc.CreateLoadBalancer(&elbv2.CreateLoadBalancerInput{
|
||||
Name: aws.String("alb-reject"),
|
||||
Subnets: []*string{aws.String(subnetID)},
|
||||
}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
err = svc.SetLoadBalancerIngressCIDRs(*albOut.LoadBalancers[0].LoadBalancerArn, []string{"10.0.0.0/8"}, testAccountID)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
// TestDeleteListener_RevokesPort verifies removing a listener closes its port on
|
||||
// the managed SG.
|
||||
func TestDeleteListener_RevokesPort(t *testing.T) {
|
||||
svc, vpcSvc := setupTestServiceWithVPC(t)
|
||||
subnetID, _ := firstSubnet(t, vpcSvc)
|
||||
|
||||
lb := createNLB(t, svc, "nlb-del-lst", "internet-facing", subnetID)
|
||||
createTCPListener(t, svc, lb.LoadBalancerArn, 443)
|
||||
|
||||
lst, err := svc.DescribeListeners(&elbv2.DescribeListenersInput{LoadBalancerArn: lb.LoadBalancerArn}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, lst.Listeners, 1)
|
||||
|
||||
rec, err := svc.store.GetLoadBalancerByArn(*lb.LoadBalancerArn)
|
||||
require.NoError(t, err)
|
||||
require.True(t, sgHasRule(describeSG(t, vpcSvc, rec.NLBManagedSGID), "tcp", 443, "0.0.0.0/0"))
|
||||
|
||||
_, err = svc.DeleteListener(&elbv2.DeleteListenerInput{ListenerArn: lst.Listeners[0].ListenerArn}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.False(t, sgHasRule(describeSG(t, vpcSvc, rec.NLBManagedSGID), "tcp", 443, "0.0.0.0/0"),
|
||||
"deleting the listener must close its port")
|
||||
}
|
||||
|
||||
// TestDeleteNLB_DeletesManagedSG verifies the managed SG is removed on LB
|
||||
// teardown.
|
||||
func TestDeleteNLB_DeletesManagedSG(t *testing.T) {
|
||||
svc, vpcSvc := setupTestServiceWithVPC(t)
|
||||
subnetID, _ := firstSubnet(t, vpcSvc)
|
||||
|
||||
lb := createNLB(t, svc, "nlb-del", "internal", subnetID)
|
||||
rec, err := svc.store.GetLoadBalancerByArn(*lb.LoadBalancerArn)
|
||||
require.NoError(t, err)
|
||||
sgID := rec.NLBManagedSGID
|
||||
require.NotEmpty(t, sgID)
|
||||
|
||||
_, err = svc.DeleteLoadBalancer(&elbv2.DeleteLoadBalancerInput{LoadBalancerArn: lb.LoadBalancerArn}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
out, err := vpcSvc.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{
|
||||
GroupIds: aws.StringSlice([]string{sgID}),
|
||||
}, testAccountID)
|
||||
// Deleted SG: either an empty result or a not-found error is acceptable.
|
||||
if err == nil {
|
||||
assert.Empty(t, out.SecurityGroups, "managed SG must be deleted with the NLB")
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user