Merge remote-tracking branch 'origin/feat/eks-node-lifecycle' into e2e/integration

WorkerNodeReady accepted any Ready Node, the tolerate-all DaemonSet check
dropped pods with no Ready condition, and nodegroup cleanup returned before
the nodegroup was gone. Scope the Ready check to the CPIsolation nodegroup,
count a pod with no Ready condition as not Ready, wait for each nodegroup
delete, and assert that deleted nodegroups leave no Node behind.

cmd/eks-gateway-fetch/main.go

@@ -6,6 +6,7 @@
 //
 //	eks-gateway-fetch -resource addons   # GET /clusters/{cluster}/internal-addons/{accountId}
 //	eks-gateway-fetch -resource recovery -instance-id i-… # GET /clusters/{cluster}/internal-recovery/{accountId}/{instanceId}
+//	eks-gateway-fetch -resource workers  # GET /clusters/{cluster}/internal-workers/{accountId}
 //
 // Flags default to environment variables seeded by cloud-init:
 // EKS_GATEWAY_URL, EKS_GATEWAY_CA, EKS_ACCESS_KEY, EKS_SECRET_KEY, EKS_REGION,
@@ -33,7 +34,7 @@ func main() {
 	flag.StringVar(&cfg.AccountID, "account-id", os.Getenv("EKS_ACCOUNT_ID"), "Cluster account ID")
 	flag.StringVar(&cfg.ClusterName, "cluster", os.Getenv("EKS_CLUSTER_NAME"), "Cluster name")
 	flag.StringVar(&cfg.InstanceID, "instance-id", os.Getenv("EKS_INSTANCE_ID"), "This member's EC2 instance ID (recovery resource)")
-	flag.StringVar(&cfg.Resource, "resource", "addons", "Resource to fetch: addons | recovery")
+	flag.StringVar(&cfg.Resource, "resource", "addons", "Resource to fetch: addons | recovery | workers")
 	flag.Parse()
 
 	if err := gatewayfetch.Run(context.Background(), cfg); err != nil {

docs/install/install-oracle-linux/README.md

@@ -164,7 +164,7 @@ On Oracle Linux the installer makes these changes, which Debian hosts do not nee
 - **Repositories.** It enables `ol10_UEKR8` and `ol10_kvm_utils`, and adds `qemu-kvm*` and `qemu-img*` to the `excludepkgs` of `ol10_appstream`, keeping any excludes already set, so a later AppStream build can never replace the KVM Utilities QEMU.
 - **NFV SIG.** It writes `/etc/yum.repos.d/spinifex-nfv-sig.repo` with an `includepkgs=` list of the OVS and OVN packages, and imports the SIG key to `/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-NFV` only when its fingerprint is `35154228174901BEFA8E69A621465E289D2A76A7`.
 - **OVS and OVN user.** Both run as root, as on Debian, through `OVS_USER_ID` and `OVN_USER_ID` in `/etc/sysconfig/openvswitch` and `/etc/sysconfig/ovn`.
-- **DNS.** Where systemd-resolved is installed but not running, it saves `/etc/resolv.conf` to `/var/lib/spinifex-reset/resolv.conf.orig`, starts resolved behind NetworkManager and links `/etc/resolv.conf` to the resolved stub. It then checks that names still resolve and stops, printing the commands that put DNS back, if they do not.
+- **DNS.** Where systemd-resolved is installed but not running, it saves `/etc/resolv.conf` to `/var/lib/spinifex-reset/resolv.conf.orig`, starts resolved behind NetworkManager, tells NetworkManager in `/etc/NetworkManager/conf.d/90-spinifex-dns.conf` to leave `/etc/resolv.conf` alone, and links `/etc/resolv.conf` to the resolved stub. It then checks that names still resolve and stops, printing the commands that put DNS back, if they do not.
 - **chrony.** It appends `confdir /etc/chrony.d` to `/etc/chrony.conf` once, and writes its step policy to `/etc/chrony.d/spinifex.conf`.
 - **`spx` under `sudo`.** Oracle Linux's `sudo` path leaves out `/usr/local/bin`, so the installer links `/usr/bin/spx` to `/usr/local/bin/spx`.
 

scripts/README.md

@@ -6,7 +6,7 @@ Node installation, dev-environment lifecycle, guest image builds, and verificati
 
 | Script | What it does |
 |---|---|
-| `setup.sh` | The production binary installer, served from `install.mulgadc.com`. Installs deps, users, sudoers, the host firewall policy, files, systemd units, logrotate and udev rules, and loads `openvswitch`, `geneve`, `vhost_net` and `tun`, listing them in `/etc/modules-load.d/spinifex.conf` for later boots. It masks firewalld wherever it is installed, since the Spinifex nftables policy is the only supported node firewall. It writes a needrestart drop-in so an apt upgrade never restarts the Spinifex, OVS or OVN units; restarting them after an upgrade is left to the operator. An unset `--firewall` keeps the mode recorded in `/etc/spinifex/firewall/mode`, and otherwise arms the policy when firewalld was active. Where systemd-resolved is installed but not running, resolvconf is absent and NetworkManager runs, it hands host DNS to resolved: it saves `/etc/resolv.conf` once to `/var/lib/spinifex-reset/resolv.conf.orig`, links `/etc/resolv.conf` to the resolved stub, and stops unless resolved has upstream servers and the package repository host still resolves. It links `/usr/bin/spx` to `/usr/local/bin/spx` where sudo's PATH leaves out `/usr/local/bin`, and takes the nbdkit plugin directory from `nbdkit --dump-config`, stopping when nbdkit cannot say. `SETUP_STAGES` runs a subset; `ISO_BUILD=1` runs it inside the ISO builder's chroot. |
+| `setup.sh` | The production binary installer, served from `install.mulgadc.com`. Installs deps, users, sudoers, the host firewall policy, files, systemd units, logrotate and udev rules, and loads `openvswitch`, `geneve`, `vhost_net` and `tun`, listing them in `/etc/modules-load.d/spinifex.conf` for later boots. It masks firewalld wherever it is installed, since the Spinifex nftables policy is the only supported node firewall. It writes a needrestart drop-in so an apt upgrade never restarts the Spinifex, OVS or OVN units; restarting them after an upgrade is left to the operator. An unset `--firewall` keeps the mode recorded in `/etc/spinifex/firewall/mode`, and otherwise arms the policy when firewalld was active. Where systemd-resolved is installed but not running, resolvconf is absent and NetworkManager runs, it hands host DNS to resolved: it saves `/etc/resolv.conf` once to `/var/lib/spinifex-reset/resolv.conf.orig`, writes `/etc/NetworkManager/conf.d/90-spinifex-dns.conf` (`dns=systemd-resolved`, `rc-manager=unmanaged`) so NetworkManager never rewrites `/etc/resolv.conf`, links `/etc/resolv.conf` to the resolved stub, and stops unless the link holds, resolved has upstream servers and the package repository host still resolves. A rerun after that handover adds a missing drop-in, reloads NetworkManager and relinks the stub. It links `/usr/bin/spx` to `/usr/local/bin/spx` where sudo's PATH leaves out `/usr/local/bin`, and takes the nbdkit plugin directory from `nbdkit --dump-config`, stopping when nbdkit cannot say. `SETUP_STAGES` runs a subset; `ISO_BUILD=1` runs it inside the ISO builder's chroot. |
 | `setup-ovn.sh` | Bootstraps a compute node for OVN VPC networking: installs OVN/OVS, creates `br-int`, configures the WAN bridge (bridged, DHCP, or routed-NAT uplink), sets the chassis identity, and applies overlay sysctls. It runs OVS and OVN as root, setting `OVS_USER_ID` and `OVN_USER_ID` where `/etc/sysconfig` holds them, and hooks the socket-permission helper onto `ovs-vswitchd`, `ovn-controller`, `openvswitch-ipsec` and, on EL, `ovn-northd`. `--management` also starts the OVN central services, writes their `ovn-ctl` flags to `/etc/default/ovn-central` (Debian) or `/etc/sysconfig/ovn-northd` (EL), and records the role in `/etc/spinifex/ovn-role`. With clustered databases it returns only once this node's NB and SB members are back in their RAFT clusters under a known leader, and fails after `OVN_RAFT_REJOIN_WAIT` seconds (default 60) otherwise, so reruns one node at a time never take two members of a cluster down. A later run without `--management` refuses while that file, or the `OVN_CTL_OPTS` line older nodes carry in `/etc/default/ovn-central`, records the role. It adds `maxsize 100M` to the OVS and OVN logrotate files, plus `rotate 5` where the file sets no count, and rotates as root, matching the daemons. On a host whose IPsec stack is libreswan it enables, stops and masks `openvswitch-ipsec.service` and `ipsec.service` for the daemon to unmask when IPsec is on, unless `openvswitch-ipsec.service` is already running; elsewhere it enables and starts `openvswitch-ipsec.service`, unless the daemon has masked it to keep IPsec off. The links it creates persist through systemd-networkd where that is enabled, and otherwise through `/etc/spinifex/hostnet.json`, which `spinifex-hostnet.service` (`spx admin hostnet apply`) replays at boot and on every OVS start, starting `ovn-controller` again when an OVS stop took it down; the first run records the choice in `/etc/spinifex/hostnet-backend`. Where NetworkManager runs, it leaves the links Spinifex owns unmanaged. |
 | `install-node.sh` | Forms a multi-node cluster across hosts that are already running Spinifex: rebuilds the OVN database in clustered form, runs `spx admin init` on the first host and `spx admin join` on the rest, then verifies. Before touching any host it refuses hosts of different OS families (`distro_os_family`) or `spx` versions, and `--ipsec on` when any host uses libreswan. The scripted form of [install-multi-node](../docs/install/install-multi-node/README.md). Destroys the joining nodes' logical network state and crypto — see its `--help`. |
 | `dev-install.sh` | Full local dev setup through the production installer: builds from source, packs a tarball, runs `setup.sh`, initialises the cluster, and starts services under systemd. |

scripts/images/eks-node/eks-node-role.sh

@@ -12,11 +12,12 @@ set -eu
 # once.
 #
 #   server      → eks-token-webhook, k3s (server), k3s-first-boot (bootstrap
-#                 publish), mulga-eks-state-report
+#                 publish), mulga-eks-state-report, mulga-eks-node-gc
 #   server-join → eks-token-webhook, k3s (server, joins the first server's etcd
-#                 quorum), mulga-eks-state-report. NO k3s-first-boot: the first
-#                 server already publishes the cluster-identical bootstrap
-#                 artifacts; a join re-publish only races the bootstrap bus.
+#                 quorum), mulga-eks-state-report, mulga-eks-node-gc. NO
+#                 k3s-first-boot: the first server already publishes the
+#                 cluster-identical bootstrap artifacts; a join re-publish only
+#                 races the bootstrap bus.
 #   agent       → k3s-agent
 #
 # Paths are overridable via env so the selection logic is unit-testable (bats)
@@ -140,6 +141,7 @@ case "${ROLE}" in
         # writer owns. HA multi-server addon delivery is tracked separately
         # Server-join nodes do not run this setup.
         svc_enable mulga-eks-addon-sync
+        svc_enable mulga-eks-node-gc
         svc_start eks-token-webhook
         svc_start mulga-eks-k3s-recovery
         svc_start k3s
@@ -147,6 +149,7 @@ case "${ROLE}" in
         svc_start k3s-first-boot
         svc_start mulga-eks-state-report
         svc_start mulga-eks-addon-sync
+        svc_start mulga-eks-node-gc
         ;;
     server-join)
         log "configuring server-join role"
@@ -158,11 +161,13 @@ case "${ROLE}" in
         svc_enable k3s
         svc_enable konnectivity-server
         svc_enable mulga-eks-state-report
+        svc_enable mulga-eks-node-gc
         svc_start eks-token-webhook
         svc_start mulga-eks-k3s-recovery
         svc_start k3s
         svc_start konnectivity-server
         svc_start mulga-eks-state-report
+        svc_start mulga-eks-node-gc
         ;;
     agent)
         log "configuring agent role"

scripts/images/eks-node/eks-node-role_test.sh

@@ -141,6 +141,8 @@ for INIT in openrc systemd; do
             END { print (rn > 0 && kn > 0 && rn < kn) ? "yes" : "no" }
         ' "${CALLS}"
     )"
+    check "server(${INIT}): enables node-gc" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-node-gc)")"
+    check "server(${INIT}): starts node-gc" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-node-gc)")"
     check "server(${INIT}): no k3s-agent" "no" "$(present "${CALLS}" "$(enable_cmd "${INIT}" k3s-agent)")"
     check "server(${INIT}): self-disables" "yes" "$(present "${CALLS}" "$(disable_cmd "${INIT}" eks-node-role)")"
 
@@ -153,6 +155,8 @@ for INIT in openrc systemd; do
     check "server-join(${INIT}): enables webhook" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" eks-token-webhook)")"
     check "server-join(${INIT}): enables k3s" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" k3s)")"
     check "server-join(${INIT}): enables state-report" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-state-report)")"
+    check "server-join(${INIT}): enables node-gc" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-node-gc)")"
+    check "server-join(${INIT}): starts node-gc" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-node-gc)")"
     check "server-join(${INIT}): enables konnectivity-server" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" konnectivity-server)")"
     check "server-join(${INIT}): enables k3s-recovery" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-k3s-recovery)")"
     check "server-join(${INIT}): starts k3s-recovery" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-k3s-recovery)")"
@@ -170,7 +174,7 @@ for INIT in openrc systemd; do
     check "agent(${INIT}): exit 0" 0 "${rc}"
     check "agent(${INIT}): role file" "agent" "$(cat "${ROLE_FILE}")"
     check "agent(${INIT}): enables k3s-agent" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" k3s-agent)")"
-    check "agent(${INIT}): no server services" "yes" "$(absent_all "${CALLS}" "eks-token-webhook" "k3s-first-boot" "$(enable_cmd "${INIT}" k3s)")"
+    check "agent(${INIT}): no server services" "yes" "$(absent_all "${CALLS}" "eks-token-webhook" "k3s-first-boot" "mulga-eks-node-gc" "$(enable_cmd "${INIT}" k3s)")"
 
     # --- agent (inferred from agent.env, no explicit role) ---
     run_case "${INIT}" agent_inferred

scripts/images/eks-node/manifest.conf

@@ -42,6 +42,8 @@ scripts/images/eks-node/mulga-eks-state-report.initd:/etc/init.d/mulga-eks-state
 scripts/images/eks-node/mulga-eks-state-report.sh:/usr/local/sbin/mulga-eks-state-report \
 scripts/images/eks-node/mulga-eks-addon-sync.initd:/etc/init.d/mulga-eks-addon-sync \
 scripts/images/eks-node/mulga-eks-addon-sync.sh:/usr/local/sbin/mulga-eks-addon-sync \
+scripts/images/eks-node/mulga-eks-node-gc.initd:/etc/init.d/mulga-eks-node-gc \
+scripts/images/eks-node/mulga-eks-node-gc.sh:/usr/local/sbin/mulga-eks-node-gc \
 scripts/images/eks-node/konnectivity-server.initd:/etc/init.d/konnectivity-server \
 scripts/images/eks-node/konnectivity-server-prestart.sh:/usr/local/sbin/konnectivity-server-prestart \
 scripts/images/eks-node/konnectivity/konnectivity-agent.yaml:/usr/share/spinifex-eks/konnectivity/konnectivity-agent.yaml \

scripts/images/eks-node/mulga-eks-node-gc.initd

@@ -0,0 +1,27 @@
+#!/sbin/openrc-run
+# mulga-eks-node-gc — server-role service that deletes the Node of every
+# nodegroup worker the nodegroup has terminated, every NODE_GC_INTERVAL
+# seconds. Enabled on every server role by eks-node-role.sh.
+
+description="Mulga EKS terminated-worker Node cleanup"
+command="/usr/local/sbin/mulga-eks-node-gc"
+command_background="yes"
+pidfile="/run/mulga-eks-node-gc.pid"
+output_log="/var/log/mulga-eks-node-gc.log"
+error_log="/var/log/mulga-eks-node-gc.log"
+
+# A Node goes NotReady about 40s after its worker stops, so 30s deletes it
+# within a couple of minutes of the terminate.
+export NODE_GC_INTERVAL="${NODE_GC_INTERVAL:-30}"
+
+depend() {
+    need net
+    after k3s
+}
+
+start_pre() {
+    if [ ! -f /etc/spinifex-eks/first-boot.env ]; then
+        eerror "/etc/spinifex-eks/first-boot.env not present — cloud-init did not seed gateway creds"
+        return 1
+    fi
+}

scripts/images/eks-node/mulga-eks-node-gc.sh

@@ -0,0 +1,89 @@
+#!/bin/sh
+set -eu
+
+# mulga-eks-node-gc — deletes the Node of a nodegroup worker whose instance the
+# nodegroup has terminated (DeleteNodegroup, scale-down, reclaim), as the AWS
+# cloud controller deletes a Node whose instance no longer exists. K3s runs no
+# AWS cloud provider: its own reports every instance as existing, so without
+# this a terminated worker stays a NotReady Node forever, its pods stay bound to
+# it, and once most Nodes are NotReady the node lifecycle controller stops
+# evicting at all.
+#
+# Pull (host→VM): GET /clusters/{cluster}/internal-workers/{accountId}, the
+# instance ID of every worker the cluster's nodegroups still own.
+#
+# A Node is deleted only when all of these hold, so a live worker is never
+# touched: it carries the eks.amazonaws.com/nodegroup label, its providerID is
+# aws:///<az>/<instance-id>, it is not Ready, and its instance is not in the
+# fetched set. A failed fetch deletes nothing.
+#
+# Runs on every server; deletes are idempotent. Loops every NODE_GC_INTERVAL
+# seconds in service mode, one-shot otherwise.
+
+ENVFILE=${ENVFILE:-/etc/spinifex-eks/first-boot.env}
+[ -f "${ENVFILE}" ] || { logger -t mulga-eks-node-gc "${ENVFILE} missing — exiting"; exit 0; }
+# set -a so the sourced creds are exported to the eks-gateway-fetch child.
+set -a
+# shellcheck disable=SC1090
+. "${ENVFILE}"
+set +a
+
+: "${EKS_GATEWAY_URL:?}"
+: "${EKS_ACCOUNT_ID:?}"
+: "${EKS_CLUSTER_NAME:?}"
+
+KUBECONFIG=${KUBECONFIG:-/etc/rancher/k3s/k3s.yaml}
+export KUBECONFIG
+
+# One "name|providerID|nodegroup|Ready" row per Node.
+NODE_ROWS='jsonpath={range .items[*]}{.metadata.name}{"|"}{.spec.providerID}{"|"}{.metadata.labels.eks\.amazonaws\.com/nodegroup}{"|"}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}'
+
+log() {
+    logger -t mulga-eks-node-gc "$*"
+    echo "[mulga-eks-node-gc] $*" > "${CONSOLE:-/dev/console}" 2>/dev/null || true
+}
+
+# stale_nodes LIVE_FILE — reads Node rows on stdin and prints the name of each
+# Node to delete: a nodegroup worker, not Ready, whose instance is not listed
+# in LIVE_FILE.
+stale_nodes() {
+    awk -F'|' -v live="$1" '
+        BEGIN { while ((getline id < live) > 0) keep[id] = 1 }
+        $3 == "" || $4 == "True" { next }
+        $2 !~ /^aws:\/\/\/[^\/]+\/i-[0-9a-f]+$/ { next }
+        { n = split($2, p, "/"); if (!(p[n] in keep)) print $1 }
+    '
+}
+
+sync_once() {
+    # Nodes are listed before the workers are fetched. A worker is recorded on
+    # its nodegroup before it boots, so every Node listed here is already in a
+    # set fetched after.
+    if ! rows=$(kubectl get nodes -o "${NODE_ROWS}" 2>/dev/null); then
+        log "kubectl get nodes failed; skipping this pass"
+        return 0
+    fi
+    live=$(mktemp)
+    if ! eks-gateway-fetch -resource workers > "${live}" 2>/dev/null; then
+        log "fetch of the cluster's nodegroup workers failed; deleting nothing"
+        rm -f "${live}"
+        return 0
+    fi
+    printf '%s\n' "${rows}" | stale_nodes "${live}" | while read -r node; do
+        if kubectl delete node "${node}" --ignore-not-found --wait=false >/dev/null 2>&1; then
+            log "deleted Node ${node}: its instance is no longer a nodegroup worker"
+        else
+            log "delete of Node ${node} failed; retrying next pass"
+        fi
+    done
+    rm -f "${live}"
+}
+
+if [ -n "${NODE_GC_INTERVAL:-}" ] && [ "${NODE_GC_INTERVAL}" -gt 0 ] 2>/dev/null; then
+    while true; do
+        sync_once
+        sleep "${NODE_GC_INTERVAL}"
+    done
+else
+    sync_once
+fi

scripts/images/eks-node/mulga-eks-node-gc_test.sh

@@ -0,0 +1,107 @@
+#!/bin/sh
+# Self-contained POSIX test for mulga-eks-node-gc.sh. kubectl, eks-gateway-fetch
+# and logger are stubbed on PATH: kubectl serves canned Node rows and records
+# every delete, and eks-gateway-fetch serves a canned worker set or fails.
+#
+# Run: sh scripts/images/eks-node/mulga-eks-node-gc_test.sh
+set -eu
+
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
+AGENT="${SCRIPT_DIR}/mulga-eks-node-gc.sh"
+WORK=$(mktemp -d)
+trap 'rm -rf "${WORK}"' EXIT
+
+STUBBIN="${WORK}/bin"
+mkdir -p "${STUBBIN}"
+
+cat > "${STUBBIN}/kubectl" <<'EOF'
+#!/bin/sh
+case "$1 $2" in
+    "get nodes")
+        [ "${NODES_RC:-0}" -eq 0 ] || exit "${NODES_RC}"
+        printf '%s\n' "${NODES_BODY:-}" ;;
+    "delete node") echo "$3" >> "${DELETED}" ;;
+esac
+EOF
+
+cat > "${STUBBIN}/eks-gateway-fetch" <<'EOF'
+#!/bin/sh
+echo "$*" >> "${FETCHED}"
+[ "${FETCH_RC:-0}" -eq 0 ] || exit "${FETCH_RC}"
+printf '%s' "${LIVE_BODY:-}"
+EOF
+
+cat > "${STUBBIN}/logger" <<'EOF'
+#!/bin/sh
+:
+EOF
+
+chmod +x "${STUBBIN}"/*
+PATH="${STUBBIN}:${PATH}"
+export PATH
+
+ENVFILE="${WORK}/first-boot.env"
+cat > "${ENVFILE}" <<'EOF'
+EKS_GATEWAY_URL=https://gw.invalid:9999
+EKS_ACCOUNT_ID=000000000001
+EKS_CLUSTER_NAME=demo3
+EOF
+CONSOLE="${WORK}/console"
+DELETED="${WORK}/deleted"
+FETCHED="${WORK}/fetched"
+export ENVFILE CONSOLE DELETED FETCHED
+
+PASS=0
+FAIL=0
+check() {
+    if [ "$2" = "$3" ]; then
+        PASS=$((PASS + 1))
+    else
+        FAIL=$((FAIL + 1))
+        printf 'FAIL: %s\n  want: %s\n  got:  %s\n' "$1" "$2" "$3"
+    fi
+}
+
+run_agent() {
+    : > "${DELETED}"
+    : > "${FETCHED}"
+    sh "${AGENT}"
+}
+
+deleted() { tr '\n' ' ' < "${DELETED}" | sed 's/ $//'; }
+
+# Every way a Node can look: only a NotReady nodegroup worker whose instance the
+# cluster's nodegroups no longer own is deleted.
+NODES_BODY='live|aws:///z1/i-0aaa|ng1|True
+gone|aws:///z1/i-0bbb|ng1|Unknown
+gone-never-reported|aws:///z1/i-0ddd|ng2|
+owned-notready|aws:///z1/i-0ccc|ng1|False
+gone-but-ready|aws:///z1/i-0eee|ng1|True
+self-managed|aws:///z1/i-0fff||Unknown
+k3s-provider|k3s://k3s-node|ng1|Unknown'
+LIVE_BODY='i-0aaa
+i-0ccc
+'
+export NODES_BODY LIVE_BODY
+
+run_agent
+check "deletes only NotReady nodegroup Nodes whose instance is gone" "gone gone-never-reported" "$(deleted)"
+check "fetches the workers resource" "-resource workers" "$(cat "${FETCHED}")"
+
+# A cluster with no nodegroups left: every NotReady nodegroup Node goes.
+LIVE_BODY='' run_agent
+check "empty worker set deletes every NotReady nodegroup Node" "gone gone-never-reported owned-notready" "$(deleted)"
+
+# A failed fetch is not an empty set: nothing is deleted.
+FETCH_RC=1 run_agent
+check "failed fetch deletes nothing" "" "$(deleted)"
+
+# Without a Node listing the workers are not fetched, so a set fetched before a
+# Node registered can never be compared against it.
+NODES_RC=1 run_agent
+check "failed node listing deletes nothing" "" "$(deleted)"
+check "failed node listing does not fetch" "" "$(cat "${FETCHED}")"
+
+echo "---"
+echo "PASS=${PASS} FAIL=${FAIL}"
+[ "${FAIL}" -eq 0 ]

scripts/images/eks-node/setup.sh

@@ -44,9 +44,10 @@ ln -sf /usr/local/bin/k3s /usr/local/bin/ctr
 # role's services are baked; the selector enables the right ones at first boot.
 chmod 0755 /etc/init.d/eks-node-role /etc/init.d/k3s /etc/init.d/k3s-agent \
     /etc/init.d/eks-token-webhook /etc/init.d/k3s-first-boot /etc/init.d/mulga-eks-state-report \
-    /etc/init.d/mulga-eks-addon-sync /etc/init.d/konnectivity-server
+    /etc/init.d/mulga-eks-addon-sync /etc/init.d/mulga-eks-node-gc /etc/init.d/konnectivity-server
 chmod 0755 /usr/local/sbin/eks-node-role /usr/local/sbin/k3s-first-boot \
-    /usr/local/sbin/mulga-eks-state-report /usr/local/sbin/mulga-eks-addon-sync
+    /usr/local/sbin/mulga-eks-state-report /usr/local/sbin/mulga-eks-addon-sync \
+    /usr/local/sbin/mulga-eks-node-gc
 # start_pre helpers shared with the systemd wrappers that will eventually
 # replace these runscripts.
 chmod 0755 /usr/local/sbin/k3s-prestart /usr/local/sbin/k3s-agent-prestart \

scripts/setup.sh

@@ -1014,6 +1014,9 @@ RESOLV_CONF="${SPINIFEX_RESOLV_CONF:-/etc/resolv.conf}"
 RESOLVED_STUB=/run/systemd/resolve/stub-resolv.conf
 # node-reset.sh keeps /var/lib/spinifex-reset, so the copy survives until uninstall.
 RESOLV_CONF_SAVED="${SPINIFEX_RESOLV_CONF_SAVED:-/var/lib/spinifex-reset/resolv.conf.orig}"
+# NetworkManager started while resolv.conf is a plain file rewrites it on its
+# next DNS update, replacing the stub link; this drop-in stops that.
+NM_DNS_DROPIN="${SPINIFEX_NM_DNS_DROPIN:-/etc/NetworkManager/conf.d/90-spinifex-dns.conf}"
 
 # systemd-resolved ships resolvconf as a link to resolvectl, which only works
 # while resolved runs, so that one is not a resolvconf backend.
@@ -1066,10 +1069,52 @@ dns_handover_failed() {
     fatal "Host DNS through systemd-resolved does not work: $1.
     To put host DNS back as it was before this installer:
       sudo systemctl disable --now systemd-resolved
+      sudo rm -f $NM_DNS_DROPIN
       sudo cp -a --remove-destination $RESOLV_CONF_SAVED $RESOLV_CONF
       sudo systemctl restart NetworkManager"
 }
 
+# Prints the rc-manager NetworkManager would use if it is not unmanaged, which
+# means a file read after the Spinifex drop-in overrides it.
+nm_rc_manager_override() {
+    local config
+    config=$($SUDO NetworkManager --print-config) || return 1
+    printf '%s\n' "$config" | awk -F= '
+        /^\[/ { main = ($0 == "[main]"); next }
+        main && $1 == "rc-manager" { rc = $2 }
+        END { if (rc != "unmanaged") print "rc-manager=" (rc == "" ? "symlink" : rc) }'
+}
+
+# Writes the NetworkManager drop-in and checks nothing overrides it. Returns 1
+# when it was already in place.
+install_nm_dns_dropin() {
+    local want override changed=1
+    want=$(printf '[main]\ndns=systemd-resolved\nrc-manager=unmanaged')
+    if [ "$(cat "$NM_DNS_DROPIN" 2>/dev/null)" != "$want" ]; then
+        $SUDO mkdir -p "$(dirname "$NM_DNS_DROPIN")"
+        printf '%s\n' "$want" | $SUDO tee "$NM_DNS_DROPIN" > /dev/null || dns_handover_failed "cannot write $NM_DNS_DROPIN"
+        info "  $NM_DNS_DROPIN: NetworkManager leaves $RESOLV_CONF alone"
+        changed=0
+    fi
+    override=$(nm_rc_manager_override) || dns_handover_failed "cannot read NetworkManager's configuration with 'NetworkManager --print-config'"
+    if [ -n "$override" ]; then
+        dns_handover_failed "NetworkManager reads $override from a file after $NM_DNS_DROPIN, so it would replace the $RESOLV_CONF link. Remove that setting ('NetworkManager --print-config' shows the files it reads) and re-run this installer"
+    fi
+    return "$changed"
+}
+
+# Links the stub and checks the link is still there a second later, relinking a
+# bounded number of times in case a write NetworkManager had queued lands first.
+link_resolved_stub() {
+    local i
+    for i in 1 2 3 4 5; do
+        $SUDO ln -sfn "$RESOLVED_STUB" "$RESOLV_CONF"
+        sleep 1
+        [ "$(readlink "$RESOLV_CONF")" = "$RESOLVED_STUB" ] && return 0
+    done
+    dns_handover_failed "$RESOLV_CONF is not a link to $RESOLVED_STUB 5 seconds after linking it, so something keeps rewriting it"
+}
+
 # Checks resolved has upstream servers and still resolves the package
 # repository host, the one name this host is known to have resolved.
 check_resolved_dns() {
@@ -1103,8 +1148,13 @@ hand_dns_to_resolved() {
     [ "${ISO_BUILD:-0}" = "1" ] && return 0
 
     if systemctl is-active --quiet systemd-resolved.service; then
-        # A handover made by an earlier run is checked again on every run.
+        # A handover made by an earlier run is checked again, and repaired, on
+        # every run: installs before the drop-in could lose the link.
         if [ -e "$RESOLV_CONF_SAVED" ] || [ -L "$RESOLV_CONF_SAVED" ]; then
+            if install_nm_dns_dropin && systemctl is-active --quiet NetworkManager.service; then
+                $SUDO systemctl reload NetworkManager.service || dns_handover_failed "NetworkManager did not reload"
+            fi
+            [ "$(readlink "$RESOLV_CONF")" = "$RESOLVED_STUB" ] || link_resolved_stub
             check_resolved_dns
         fi
         return 0
@@ -1131,10 +1181,15 @@ hand_dns_to_resolved() {
         $SUDO cp -a "$RESOLV_CONF" "$RESOLV_CONF_SAVED" || fatal "Cannot save $RESOLV_CONF to $RESOLV_CONF_SAVED"
         info "  saved $RESOLV_CONF to $RESOLV_CONF_SAVED"
     fi
+    install_nm_dns_dropin || true
     $SUDO systemctl enable --now systemd-resolved.service > /dev/null 2>&1 \
         || dns_handover_failed "systemd-resolved did not start"
     $SUDO systemctl restart NetworkManager.service || dns_handover_failed "NetworkManager did not restart"
-    $SUDO ln -sfn "$RESOLVED_STUB" "$RESOLV_CONF"
+    # Best effort: a link with no DHCP server can hold startup past the timeout.
+    if command -v nm-online > /dev/null; then
+        nm-online -s -q --timeout=15 || true
+    fi
+    link_resolved_stub
     check_resolved_dns
     info "  $RESOLV_CONF -> $RESOLVED_STUB"
 }

scripts/setup_host_test.sh

@@ -20,9 +20,10 @@ fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
 echo "-- setup.sh host DNS, chrony, needrestart and nbdkit"
 
 mkdir -p "$TMP/sys"
-for tool in bash awk sed grep readlink basename dirname cp ln mkdir cat head tail wc tee rm chmod install; do
+for tool in bash awk sed grep readlink basename dirname cp ln mkdir cat head tail wc tee rm chmod install mv; do
     ln -s "$(command -v "$tool")" "$TMP/sys/$tool"
 done
+ln -s "$(command -v sleep)" "$TMP/sys/realsleep"
 
 # stub <name> <body> — a stub command; $S is the fixture's state directory.
 stub() {
@@ -30,10 +31,17 @@ stub() {
     chmod +x "$TMP/bin/$1"
 }
 
+# nm_settle — waits for the fixture's NetworkManager to finish writing.
+nm_settle() {
+    n=0
+    while [ -e "$TMP/state/nm-pending" ] && [ $n -lt 500 ]; do "$TMP/sys/realsleep" 0.01; n=$((n + 1)); done
+}
+
 # host — a fresh fixture: resolved installed and inactive, resolvconf the
 # resolvectl link, NetworkManager running with no DNS settings, a dnf host
 # whose first enabled repository is yum.oracle.com.
 host() {
+    nm_settle
     rm -rf "${TMP:?}/state" "${TMP:?}/root" "${TMP:?}/bin"
     mkdir -p "$TMP/state/active" "$TMP/root/etc" "$TMP/root/usr/lib/systemd/system" "$TMP/bin"
     : > "$TMP/root/usr/lib/systemd/system/systemd-resolved.service"
@@ -42,11 +50,28 @@ host() {
     printf '[main]\n# dns=default\nconfigure-and-quit=no\n' > "$TMP/state/nm-config"
     printf 'Global:\nLink 2 (br-wan): 192.168.1.1\n' > "$TMP/state/resolvectl-dns"
     printf '[ol10_UEKR8]\nbaseurl = https://mirror.example/uek/\nenabled = 0\n[ol10_baseos_latest]\nbaseurl = https://yum.oracle.com/repo/OracleLinux/OL10/baseos/latest/x86_64/\nenabled = 1\n' > "$TMP/state/dnf-dump"
+    # A (re)started NetworkManager whose rc-manager is not unmanaged writes
+    # resolv.conf as a plain file on its first DNS update, which can land just
+    # after the installer links the stub.
     stub systemctl '[ "$1" = is-active ] && { [ -e "$S/active/$3" ]; exit; }
 echo "$*" >> "$S/calls"
 [ "$1 $2" = "enable --now" ] && : > "$S/active/$3"
+case "$1 $2" in
+    "restart NetworkManager.service" | "reload NetworkManager.service")
+        NetworkManager --print-config | grep -qx rc-manager=unmanaged && exit 0
+        : > "$S/nm-pending"
+        (
+            n=0
+            while [ ! -L "$S/../root/etc/resolv.conf" ] && [ $n -lt 300 ]; do realsleep 0.01; n=$((n + 1)); done
+            printf "nameserver 192.168.1.1\n" > "$S/nm-resolv.conf"
+            mv -f "$S/nm-resolv.conf" "$S/../root/etc/resolv.conf"
+            rm -f "$S/nm-pending"
+        ) > /dev/null 2>&1 &
+        ;;
+esac
 exit 0'
-    stub NetworkManager 'cat "$S/nm-config"'
+    stub NetworkManager 'cat "$S/nm-config"; cat "$S/../root/nm/90-spinifex-dns.conf" "$S/nm-config-late" 2>/dev/null; :'
+    stub nm-online ':'
     stub resolvectl '[ "$1" = dns ] && cat "$S/resolvectl-dns"'
     stub resolvectl-real ':'
     ln -s resolvectl "$TMP/bin/resolvconf"
@@ -60,10 +85,17 @@ run() {
     RC=0
     OUT=$(PATH="$TMP/bin:$TMP/sys" DISTRO_ROOT="$TMP/root" \
         SPINIFEX_RESOLV_CONF="$TMP/root/etc/resolv.conf" SPINIFEX_RESOLV_CONF_SAVED="$TMP/root/saved/resolv.conf.orig" \
+        SPINIFEX_NM_DNS_DROPIN="$TMP/root/nm/90-spinifex-dns.conf" \
         INSTALL_SPINIFEX_LIB_ONLY=1 bash -c '. "$1"; shift; for f in "$@"; do $f; done' _ "$SETUP" "$@" 2>&1) || RC=$?
 }
 
 calls() { cat "$TMP/state/calls" 2>/dev/null || true; }
+# linked — whether resolv.conf is still the stub link once NetworkManager settles.
+linked() {
+    nm_settle
+    [ "$(readlink "$TMP/root/etc/resolv.conf")" = /run/systemd/resolve/stub-resolv.conf ]
+}
+dropin() { cat "$TMP/root/nm/90-spinifex-dns.conf" 2>/dev/null || echo "<none>"; }
 saved() { cat "$TMP/root/saved/resolv.conf.orig" 2>/dev/null || echo "<none>"; }
 expect_untouched() {
     if [ "$RC" = 0 ] && [ -z "$(calls)" ] && [ ! -L "$TMP/root/etc/resolv.conf" ] && [ "$(saved)" = "<none>" ]; then
@@ -86,7 +118,7 @@ expect_fatal() { # expect_fatal <desc> <text>...
 # --------------------------------------------------------------------------
 host
 run hand_dns_to_resolved
-if [ "$RC" = 0 ] && [ "$(readlink "$TMP/root/etc/resolv.conf")" = /run/systemd/resolve/stub-resolv.conf ] \
+if [ "$RC" = 0 ] && linked \
     && [ "$(saved)" = "$(printf 'nameserver 192.168.1.1\nsearch example.lan')" ] \
     && [ "$(calls)" = "$(printf 'enable --now systemd-resolved.service\nrestart NetworkManager.service')" ] \
     && [ "$(cat "$TMP/state/looked-up")" = yum.oracle.com ]; then
@@ -98,12 +130,44 @@ fi
 : > "$TMP/state/calls"
 rm -f "$TMP/state/looked-up"
 run hand_dns_to_resolved
-if [ "$RC" = 0 ] && [ -z "$(calls)" ] && [ "$(cat "$TMP/state/looked-up")" = yum.oracle.com ]; then
+if [ "$RC" = 0 ] && [ -z "$(calls)" ] && linked && [ "$(cat "$TMP/state/looked-up")" = yum.oracle.com ]; then
     pass "a rerun after a handover checks DNS again and changes nothing"
 else
     fail "rerun after handover: rc=$RC calls='$(calls)': $OUT"
 fi
 
+if [ "$(dropin)" = "$(printf '[main]\ndns=systemd-resolved\nrc-manager=unmanaged')" ]; then
+    pass "the handover tells NetworkManager to send DNS to resolved and leave resolv.conf alone"
+else
+    fail "NetworkManager drop-in: '$(dropin)'"
+fi
+
+# An install made before the drop-in, whose NetworkManager replaced the link.
+host
+mkdir -p "$TMP/root/saved"
+echo "the original" > "$TMP/root/saved/resolv.conf.orig"
+: > "$TMP/state/active/systemd-resolved.service"
+run hand_dns_to_resolved
+if [ "$RC" = 0 ] && linked && [ "$(calls)" = "reload NetworkManager.service" ] \
+    && [ "$(dropin)" = "$(printf '[main]\ndns=systemd-resolved\nrc-manager=unmanaged')" ]; then
+    pass "a rerun after a handover whose link NetworkManager replaced adds the drop-in, reloads NM and relinks the stub"
+else
+    fail "rerun repair: rc=$RC calls='$(calls)' link='$(readlink "$TMP/root/etc/resolv.conf")' dropin='$(dropin)': $OUT"
+fi
+
+host
+printf '[main]\nrc-manager=symlink\n' > "$TMP/state/nm-config-late"
+run hand_dns_to_resolved
+expect_fatal "a NetworkManager file read after the drop-in that sets rc-manager stops the install, naming it" \
+    "rc-manager=symlink" "rm -f $TMP/root/nm/90-spinifex-dns.conf"
+case "$(calls)" in *"restart NetworkManager"*) fail "rc-manager override: restarted NetworkManager first: $(calls)" ;; esac
+
+host
+stub ln ':'
+run hand_dns_to_resolved
+expect_fatal "a resolv.conf that will not stay linked to the stub stops the install" \
+    "is not a link to /run/systemd/resolve/stub-resolv.conf"
+
 host
 mkdir -p "$TMP/root/saved"
 echo "the original" > "$TMP/root/saved/resolv.conf.orig"
@@ -155,7 +219,8 @@ host
 printf 'Global:\nLink 2 (br-wan):\n' > "$TMP/state/resolvectl-dns"
 run hand_dns_to_resolved
 expect_fatal "no upstream servers stops the install, naming the check and printing the restore" \
-    "'resolvectl dns' lists no DNS servers" "systemctl disable --now systemd-resolved" "cp -a --remove-destination $TMP/root/saved/resolv.conf.orig"
+    "'resolvectl dns' lists no DNS servers" "systemctl disable --now systemd-resolved" "cp -a --remove-destination $TMP/root/saved/resolv.conf.orig" \
+    "rm -f $TMP/root/nm/90-spinifex-dns.conf"
 
 host
 : > "$TMP/state/getent-fail"

spinifex/agents/eks/gatewayfetch/fetch.go

@@ -16,6 +16,9 @@
 //
 // The recovery resource returns a single epoch\taction\tsnapshot line the on-VM
 // k3s-recovery agent applies before k3s starts (etcd cluster-reset / wipe-rejoin).
+//
+// The workers resource returns one instance ID per line: every worker the
+// cluster's nodegroups still own, which the on-VM node-gc agent keeps Nodes for.
 package gatewayfetch
 
 import (
@@ -46,7 +49,7 @@ type Config struct {
 	Region      string
 	AccountID   string
 	ClusterName string
-	// Resource is "addons" or "recovery".
+	// Resource is "addons", "recovery" or "workers".
 	Resource   string
 	InstanceID string
 	// Out receives the rendered TSV.
@@ -61,8 +64,8 @@ func Run(ctx context.Context, cfg Config) error {
 		return errors.New("--account-id is required (or set EKS_ACCOUNT_ID)")
 	case cfg.ClusterName == "":
 		return errors.New("--cluster is required (or set EKS_CLUSTER_NAME)")
-	case cfg.Resource != "addons" && cfg.Resource != "recovery":
-		return errors.New("--resource must be addons or recovery")
+	case cfg.Resource != "addons" && cfg.Resource != "recovery" && cfg.Resource != "workers":
+		return errors.New("--resource must be addons, recovery or workers")
 	case cfg.Resource == "recovery" && cfg.InstanceID == "":
 		return errors.New("--instance-id is required for --resource recovery (or set EKS_INSTANCE_ID)")
 	}
@@ -80,6 +83,9 @@ func Run(ctx context.Context, cfg Config) error {
 	case "recovery":
 		path = "/clusters/" + cfg.ClusterName + "/internal-recovery/" + cfg.AccountID + "/" + cfg.InstanceID
 		emit = emitRecoveryTSV
+	case "workers":
+		path = "/clusters/" + cfg.ClusterName + "/internal-workers/" + cfg.AccountID
+		emit = emitWorkers
 	default:
 		path = "/clusters/" + cfg.ClusterName + "/internal-addons/" + cfg.AccountID
 		emit = emitAddonsTSV
@@ -163,3 +169,24 @@ func emitRecoveryTSV(out io.Writer, body []byte) error {
 	}
 	return nil
 }
+
+type internalWorkersResponse struct {
+	InstanceIDs []string `json:"instanceIds"`
+}
+
+// emitWorkers writes one managed-worker instance ID per line. An empty set
+// writes nothing, which the node-gc agent tells apart from a failed fetch by
+// this command's exit status, never by the output being empty.
+func emitWorkers(out io.Writer, body []byte) error {
+	var resp internalWorkersResponse
+	if err := json.Unmarshal(body, &resp); err != nil {
+		return fmt.Errorf("unmarshal internal-workers response: %w", err)
+	}
+	w := bufio.NewWriter(out)
+	for _, id := range resp.InstanceIDs {
+		if _, err := fmt.Fprintln(w, id); err != nil {
+			return err
+		}
+	}
+	return w.Flush()
+}

spinifex/agents/eks/gatewayfetch/fetch_test.go

@@ -49,7 +49,7 @@ func TestRun_RejectsBeforeContactingGateway(t *testing.T) {
 	}{
 		{"account", func(c *Config) { c.AccountID = "" }, "--account-id is required (or set EKS_ACCOUNT_ID)"},
 		{"cluster", func(c *Config) { c.ClusterName = "" }, "--cluster is required (or set EKS_CLUSTER_NAME)"},
-		{"resource", func(c *Config) { c.Resource = "secrets" }, "--resource must be addons or recovery"},
+		{"resource", func(c *Config) { c.Resource = "secrets" }, "--resource must be addons, recovery or workers"},
 		{"recovery instance", func(c *Config) { c.Resource = "recovery" }, "--instance-id is required for --resource recovery (or set EKS_INSTANCE_ID)"},
 		{"gateway client", func(c *Config) { c.GatewayURL = "" }, "build gateway client: eksgw: baseURL is required"},
 	}
@@ -92,6 +92,26 @@ func TestRun_RendersRecoveryDirective(t *testing.T) {
 	}
 }
 
+// The gateway's restjson marshaler names the field InstanceIDs; both that and
+// the json-tag spelling decode, so the agent reads either.
+func TestRun_RendersManagedWorkers(t *testing.T) {
+	for _, body := range []string{
+		`{"InstanceIDs":["i-0aaa","i-0bbb"]}`,
+		`{"instanceIds":["i-0aaa","i-0bbb"]}`,
+	} {
+		var hits atomic.Int32
+		cfg, out := gateway(t, "/clusters/demo/internal-workers/123456789012", http.StatusOK, body, &hits)
+		cfg.Resource = "workers"
+
+		if err := Run(context.Background(), cfg); err != nil {
+			t.Fatalf("Run: %v", err)
+		}
+		if got, want := out.String(), "i-0aaa\ni-0bbb\n"; got != want {
+			t.Fatalf("stdout = %q, want %q", got, want)
+		}
+	}
+}
+
 func TestRun_RenderFailureIsNotRetried(t *testing.T) {
 	var hits atomic.Int32
 	cfg, _ := gateway(t, "/clusters/demo/internal-addons/123456789012", http.StatusOK, `{not-json`, &hits)

spinifex/domains/eks/awsapi/authz.go

@@ -50,10 +50,11 @@ var eksScopes = map[string][]resourceSource{
 	"ListClusters":    {sourceAny},
 
 	// Internal control-plane routes. The cluster's owning account is a path
-	// segment on these two, and the ARN names that account rather than the
+	// segment on these three, and the ARN names that account rather than the
 	// system caller's, because that is the account the handler reads.
 	"ListInternalAddons":   {sourceInternalCluster},
 	"GetRecoveryDirective": {sourceInternalCluster},
+	"ListInternalWorkers":  {sourceInternalCluster},
 	// The owning account arrives in the body on these two, and the ARN names
 	// that account for the same reason: it is the account the handler acts in.
 	"PublishInternal":    {sourceInternalBodyCluster},

spinifex/domains/eks/awsapi/authz_test.go

@@ -179,6 +179,10 @@ func TestResourceARNsInternalRoutesUsePathAccount(t *testing.T) {
 	assert.Equal(t,
 		[]string{"arn:aws:eks:ap-southeast-2:999999999999:cluster/prod"},
 		resolveARNs(t, "GetRecoveryDirective", []string{"prod", "999999999999", "i-abc"}, ""))
+
+	assert.Equal(t,
+		[]string{"arn:aws:eks:ap-southeast-2:999999999999:cluster/prod"},
+		resolveARNs(t, "ListInternalWorkers", []string{"prod", "999999999999"}, ""))
 }
 
 // An identifier the gate cannot read authorizes account-wide, leaving the

spinifex/domains/eks/awsapi/gateway_wire_test.go

@@ -93,6 +93,46 @@ func TestEKSRequest_InternalAddonsWireBody(t *testing.T) {
 		rec.Body.String())
 }
 
+// The node-gc agent reads the same REST-JSON shape: the Go field name, sorted
+// IDs drawn from every nodegroup record of the cluster.
+func TestEKSRequest_InternalWorkersWireBody(t *testing.T) {
+	const cpInstanceID = "i-cp0000000000001"
+	_, nc, js := testutil.StartTestJetStream(t)
+	kv, err := ekscluster.GetOrCreateAccountBucket(t.Context(), js, authzAccountID)
+	require.NoError(t, err)
+	require.NoError(t, ekscluster.PutClusterMeta(t.Context(), kv, &ekscluster.ClusterMeta{
+		Name:              "prod",
+		Status:            ekscluster.ClusterStatusActive,
+		ControlPlaneNodes: []ekscluster.ControlPlaneNode{{InstanceID: cpInstanceID}},
+	}))
+	for _, rec := range []*ekscluster.NodegroupRecord{
+		{ClusterName: "prod", Name: "ng-b", InstanceIDs: []string{"i-0ccc", "i-0aaa"}},
+		{ClusterName: "prod", Name: "ng-a", InstanceIDs: []string{"i-0bbb"}},
+	} {
+		require.NoError(t, ekscluster.PutNodegroupRecord(t.Context(), kv, rec))
+	}
+
+	svc, err := ekscluster.NewEKSServiceImpl(ekscluster.EKSServiceDeps{NATSConn: nc, Region: authzRegion})
+	require.NoError(t, err)
+	sub, err := nc.Subscribe("eks.ListManagedWorkers", func(m *nats.Msg) {
+		acct := natsmsg.AccountIDFromMsg(m)
+		natsmsg.ServeNATSRequestCtx(m, func(ctx context.Context, in *ekscluster.ListManagedWorkersInput) (*ekscluster.ListManagedWorkersOutput, error) {
+			return svc.ListManagedWorkers(ctx, in, acct)
+		})
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+
+	gw := scopedEKS(gwtest.Statement("Allow", "eks:*", "*"))
+	gw.nc = nc
+	rec := httptest.NewRecorder()
+	req := httptest.NewRequest(http.MethodGet, "/clusters/prod/internal-workers/"+authzAccountID, nil)
+	require.NoError(t, gw.as(cpAgentContext(cpInstanceID)).serve(rec, req))
+
+	assert.Equal(t, http.StatusOK, rec.Code)
+	assert.Equal(t, `{"InstanceIDs":["i-0aaa","i-0bbb","i-0ccc"]}`, rec.Body.String())
+}
+
 // CreateAddon and UpdateAddon hand their service-account role to iam:PassRole,
 // same as CreateCluster's roleArn and CreateNodegroup's nodeRole; an absent
 // role passes nothing.

spinifex/domains/eks/awsapi/internal_authz_test.go

@@ -97,6 +97,8 @@ func TestAuthorizeInternal_RejectsNonCPPrincipals(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			assertDenied(t, eksawsapi.AuthorizeInternal(t.Context(), nil, "ListInternalAddons",
 				tc.caller, []string{"alpha", tenantAccount}, nil))
+			assertDenied(t, eksawsapi.AuthorizeInternal(t.Context(), nil, "ListInternalWorkers",
+				tc.caller, []string{"alpha", tenantAccount}, nil))
 			assertDenied(t, eksawsapi.AuthorizeInternal(t.Context(), nil, "GetRecoveryDirective",
 				tc.caller, []string{"alpha", tenantAccount, cpInstanceID}, nil))
 			assertDenied(t, eksawsapi.AuthorizeInternal(t.Context(), nil, "PublishInternal",
@@ -124,6 +126,8 @@ func TestAuthorizeInternal_AllowsClusterMember(t *testing.T) {
 
 	require.NoError(t, eksawsapi.AuthorizeInternal(t.Context(), nc, "ListInternalAddons",
 		cpAgent(cpInstanceID), []string{"alpha", tenantAccount}, nil))
+	require.NoError(t, eksawsapi.AuthorizeInternal(t.Context(), nc, "ListInternalWorkers",
+		cpAgent(cpInstanceID), []string{"alpha", tenantAccount}, nil))
 	require.NoError(t, eksawsapi.AuthorizeInternal(t.Context(), nc, "GetRecoveryDirective",
 		cpAgent(cpInstanceID), []string{"alpha", tenantAccount, cpInstanceID}, nil))
 	require.NoError(t, eksawsapi.AuthorizeInternal(t.Context(), nc, "PublishInternal",

spinifex/domains/eks/awsapi/internal_workers.go

@@ -0,0 +1,35 @@
+package awsapi
+
+import (
+	"context"
+	"errors"
+
+	ekscluster "github.com/mulgadc/spinifex/spinifex/domains/eks/cluster"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/nats-io/nats.go"
+)
+
+// internalWorkersOutput is the body returned to the on-VM node-gc agent: the
+// instance ID of every worker the cluster's nodegroups still own.
+type internalWorkersOutput struct {
+	InstanceIDs []string `json:"instanceIds"`
+}
+
+// ListInternalWorkers — GET /clusters/{name}/internal-workers/{acct}.
+// Internal control-plane VM route (not an AWS-SDK action): same carve-out as
+// ListInternalAddons, gated by AuthorizeInternal, which admits only a CP agent
+// serving that account's cluster.
+func ListInternalWorkers(ctx context.Context, natsConn *nats.Conn, clusterName, accountID string) (*internalWorkersOutput, error) {
+	if natsConn == nil {
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	if clusterName == "" || accountID == "" {
+		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+	out, err := ekscluster.NewNATSEKSService(natsConn).ListManagedWorkers(ctx,
+		&ekscluster.ListManagedWorkersInput{ClusterName: clusterName}, accountID)
+	if err != nil {
+		return nil, err
+	}
+	return &internalWorkersOutput{InstanceIDs: out.InstanceIDs}, nil
+}

spinifex/domains/eks/awsapi/routes.go

@@ -60,6 +60,14 @@ var routes = []route{
 			return GetRecoveryDirective(ctx, nc, p[0], p[1], p[2])
 		}},
 
+	// Internal control-plane route: the on-VM node-gc agent pulls the cluster's
+	// live nodegroup workers to delete the Nodes of terminated ones. Same
+	// system-cred carve-out as internal-addons.
+	{Method: "GET", Pattern: "/clusters/{clusterName}/internal-workers/{accountId}", Action: "ListInternalWorkers",
+		Handler: func(ctx context.Context, nc *nats.Conn, acct, callerARN string, p []string, b []byte) (any, error) {
+			return ListInternalWorkers(ctx, nc, p[0], p[1])
+		}},
+
 	// Nodegroup
 	{Method: "POST", Pattern: "/clusters/{clusterName}/node-groups", Action: "CreateNodegroup",
 		Handler: func(ctx context.Context, nc *nats.Conn, acct, callerARN string, p []string, b []byte) (any, error) {

spinifex/domains/eks/awsapi/routes_test.go

@@ -108,6 +108,7 @@ func TestLookupEKSAction_CoversAllActions(t *testing.T) {
 		"WebhookTokenReview":           false,
 		"ListInternalAddons":           false,
 		"GetRecoveryDirective":         false,
+		"ListInternalWorkers":          false,
 		"DescribeCluster":              false,
 		"ListClusters":                 false,
 		"DeleteCluster":                false,

spinifex/domains/eks/cluster/k3s_server_role.go

@@ -22,9 +22,10 @@ const (
 	// WebhookTokenReview (the eks-token-webhook relays `aws eks get-token` bearer
 	// tokens to the token-review broker for host-side STS verification), and
 	// GetRecoveryDirective (the k3s-recovery agent pulls its per-member etcd
-	// recovery directive at boot). The gateway evaluates these per request against
-	// the role's policies.
-	eksServerInlinePolicy = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["eks:PublishInternal","eks:ListInternalAddons","eks:WebhookTokenReview","eks:GetRecoveryDirective"],"Resource":"*"}]}`
+	// recovery directive at boot), and ListInternalWorkers (the node-gc agent
+	// pulls the live nodegroup workers). The gateway evaluates these per request
+	// against the role's policies.
+	eksServerInlinePolicy = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["eks:PublishInternal","eks:ListInternalAddons","eks:WebhookTokenReview","eks:GetRecoveryDirective","eks:ListInternalWorkers"],"Resource":"*"}]}`
 )
 
 // ensureCPInstanceProfile returns the CP instance-profile ARN, or "" to signal
@@ -35,7 +36,7 @@ func (s *EKSServiceImpl) ensureCPInstanceProfile(accountID string) string {
 	iam := s.iamEnsurer()
 	if iam == nil {
 		slog.Warn("EKS: IAM service unwired; CP VM falls back to baked static gateway creds and "+
-			"will be denied eks:PublishInternal, eks:WebhookTokenReview, eks:ListInternalAddons and eks:GetRecoveryDirective (no instance identity to bind)",
+			"will be denied eks:PublishInternal, eks:WebhookTokenReview, eks:ListInternalAddons, eks:GetRecoveryDirective and eks:ListInternalWorkers (no instance identity to bind)",
 			"accountID", accountID)
 		return ""
 	}
@@ -43,7 +44,7 @@ func (s *EKSServiceImpl) ensureCPInstanceProfile(accountID string) string {
 		CPInstanceRoleName, eksServerInlinePolicyName, eksServerInlinePolicy)
 	if err != nil {
 		slog.Error("EKS: ensure CP instance profile failed; CP VM falls back to baked static gateway creds and "+
-			"will be denied eks:PublishInternal, eks:WebhookTokenReview, eks:ListInternalAddons and eks:GetRecoveryDirective (no instance identity to bind)",
+			"will be denied eks:PublishInternal, eks:WebhookTokenReview, eks:ListInternalAddons, eks:GetRecoveryDirective and eks:ListInternalWorkers (no instance identity to bind)",
 			"accountID", accountID, "role", CPInstanceRoleName, "err", err)
 		return ""
 	}

spinifex/domains/eks/cluster/k3s_server_role_test.go

@@ -38,6 +38,9 @@ func TestEnsureK3sServerInstanceProfile_CreatesRolePolicyProfile(t *testing.T) {
 	// these IMDS creds; without this action the gateway 403s the boot-time fetch
 	// and a wedged control plane can never be auto-reset.
 	assert.Contains(t, policy, "eks:GetRecoveryDirective")
+	// The node-gc agent pulls the live nodegroup workers with these IMDS creds;
+	// without this action a terminated worker's Node is never deleted.
+	assert.Contains(t, policy, "eks:ListInternalWorkers")
 
 	require.NotNil(t, f.Profiles[CPInstanceRoleName])
 	assert.Len(t, f.Profiles[CPInstanceRoleName].Roles, 1)

spinifex/domains/eks/cluster/service.go

@@ -44,6 +44,10 @@ type EKSService interface {
 	// AWS-SDK action): the on-VM addon-sync agent fetches staged manifests for a
 	// cluster via the internal-addons gateway route.
 	ListStagedAddonManifests(ctx context.Context, input *ListStagedAddonManifestsInput, accountID string) (*ListStagedAddonManifestsOutput, error)
+	// ListManagedWorkers is an internal control-plane method (not an AWS-SDK
+	// action): the on-VM node-gc agent fetches the cluster's live nodegroup
+	// workers via the internal-workers gateway route.
+	ListManagedWorkers(ctx context.Context, input *ListManagedWorkersInput, accountID string) (*ListManagedWorkersOutput, error)
 
 	// GetRecoveryDirective / SetRecoveryDirective are internal control-plane
 	// methods (not AWS-SDK actions): the on-VM k3s-recovery agent fetches its

spinifex/domains/eks/cluster/service_nats.go

@@ -129,6 +129,10 @@ func (s *NATSEKSService) ListStagedAddonManifests(ctx context.Context, input *Li
 	return natsmsg.NATSRequest[ListStagedAddonManifestsOutput](ctx, s.natsConn, "eks.ListStagedAddonManifests", input, defaultTimeout, accountID)
 }
 
+func (s *NATSEKSService) ListManagedWorkers(ctx context.Context, input *ListManagedWorkersInput, accountID string) (*ListManagedWorkersOutput, error) {
+	return natsmsg.NATSRequest[ListManagedWorkersOutput](ctx, s.natsConn, "eks.ListManagedWorkers", input, defaultTimeout, accountID)
+}
+
 func (s *NATSEKSService) UpdateAddon(ctx context.Context, input *eks.UpdateAddonInput, accountID string) (*eks.UpdateAddonOutput, error) {
 	return natsmsg.NATSRequest[eks.UpdateAddonOutput](ctx, s.natsConn, "eks.UpdateAddon", input, defaultTimeout, accountID)
 }

spinifex/domains/eks/cluster/worker_membership.go

@@ -0,0 +1,46 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"slices"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+// ListManagedWorkersInput names the cluster whose managed-nodegroup workers to
+// list. It is an internal control-plane request (not an AWS-SDK shape), served
+// over NATS for the guest node-gc agent via the internal-workers route.
+type ListManagedWorkersInput struct {
+	ClusterName string `json:"clusterName"`
+}
+
+// ListManagedWorkersOutput carries the instance ID of every worker a managed
+// nodegroup of the cluster still owns, sorted.
+type ListManagedWorkersOutput struct {
+	InstanceIDs []string `json:"instanceIds"`
+}
+
+// ListManagedWorkers returns every worker instance ID recorded on the cluster's
+// nodegroups. A nodegroup drops an ID only after terminating it (DeleteNodegroup,
+// scale-down, reclaim), so the guest deletes a nodegroup Node whose instance is
+// absent here, as the AWS cloud controller deletes a Node whose instance is gone.
+func (s *EKSServiceImpl) ListManagedWorkers(ctx context.Context, input *ListManagedWorkersInput, accountID string) (*ListManagedWorkersOutput, error) {
+	if input == nil || input.ClusterName == "" {
+		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+	acctKV, err := s.acctKVForCluster(ctx, accountID, input.ClusterName)
+	if err != nil {
+		return nil, err
+	}
+	recs, err := ListNodegroupRecords(ctx, acctKV, input.ClusterName)
+	if err != nil {
+		return nil, err
+	}
+	ids := make([]string, 0)
+	for _, rec := range recs {
+		ids = append(ids, rec.InstanceIDs...)
+	}
+	slices.Sort(ids)
+	return &ListManagedWorkersOutput{InstanceIDs: slices.Compact(ids)}, nil
+}

spinifex/domains/eks/cluster/worker_membership_test.go

@@ -0,0 +1,74 @@
+package cluster
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/eks"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func managedWorkers(t *testing.T, f *eksServiceFixture, cluster string) []string {
+	t.Helper()
+	out, err := f.svc.ListManagedWorkers(context.Background(), &ListManagedWorkersInput{ClusterName: cluster}, testAccountID)
+	require.NoError(t, err)
+	return out.InstanceIDs
+}
+
+// A worker leaves the managed set exactly when its nodegroup terminates it, by
+// scale-down or DeleteNodegroup, so the guest node-gc deletes its Node then and
+// leaves every worker still owned by a nodegroup alone.
+func TestListManagedWorkers_DropsWorkersTheNodegroupTerminated(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	for _, ng := range []string{"ng1", "ng2"} {
+		_, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", ng, 2), testAccountID)
+		require.NoError(t, err)
+		markWorkersReady(t, f, "c1", ng, 2)
+		f.svc.WaitLaunches()
+	}
+	ng1, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
+	require.NoError(t, err)
+	ng2, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng2")
+	require.NoError(t, err)
+	require.Len(t, ng1.InstanceIDs, 2)
+	assert.ElementsMatch(t, append(append([]string{}, ng1.InstanceIDs...), ng2.InstanceIDs...), managedWorkers(t, f, "c1"))
+
+	_, err = f.svc.UpdateNodegroupConfig(context.Background(), &eks.UpdateNodegroupConfigInput{
+		ClusterName:   aws.String("c1"),
+		NodegroupName: aws.String("ng1"),
+		ScalingConfig: &eks.NodegroupScalingConfig{DesiredSize: aws.Int64(1)},
+	}, testAccountID)
+	require.NoError(t, err)
+	require.Len(t, f.worker.terminateCalls, 1)
+	scaledIn := f.worker.terminateCalls[0]
+	got := managedWorkers(t, f, "c1")
+	assert.Len(t, got, 3)
+	for _, id := range scaledIn {
+		assert.NotContains(t, got, id, "a worker terminated by scale-down must leave the managed set")
+	}
+
+	_, err = f.svc.DeleteNodegroup(context.Background(), &eks.DeleteNodegroupInput{
+		ClusterName: aws.String("c1"), NodegroupName: aws.String("ng2"),
+	}, testAccountID)
+	require.NoError(t, err)
+	got = managedWorkers(t, f, "c1")
+	assert.Len(t, got, 1)
+	for _, id := range ng2.InstanceIDs {
+		assert.NotContains(t, got, id, "a deleted nodegroup's workers must leave the managed set")
+	}
+}
+
+func TestListManagedWorkers_UnknownClusterIsNotFound(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+
+	_, err := f.svc.ListManagedWorkers(context.Background(), &ListManagedWorkersInput{ClusterName: "ghost"}, testAccountID)
+	code, ok := awserrors.ResolveErrorCode(err)
+	require.True(t, ok)
+	assert.Equal(t, awserrors.ErrorEKSResourceNotFound, code)
+}

spinifex/ingress/aws/pipeline/operations_test.go

@@ -55,6 +55,7 @@ func TestAWSOperationInventoryMeasuresEKSAgainstTheModel(t *testing.T) {
 	require.ElementsMatch(t, []string{
 		"GetRecoveryDirective",
 		"ListInternalAddons",
+		"ListInternalWorkers",
 		"PublishInternal",
 		"WebhookTokenReview",
 	}, eks.Extra)

spinifex/runtime/daemon/daemon.go

@@ -1139,6 +1139,7 @@ func (d *Daemon) subscribeAll() error {
 			natsSub{"eks.UpdateAddon", handleNATSRequest(d.node, d.eksService.UpdateAddon), "spinifex-workers"},
 			natsSub{"eks.ListStagedAddonManifests", handleNATSRequest(d.node, d.eksService.ListStagedAddonManifests), "spinifex-workers"},
 			natsSub{"eks.GetRecoveryDirective", handleNATSRequest(d.node, d.eksService.GetRecoveryDirective), "spinifex-workers"},
+			natsSub{"eks.ListManagedWorkers", handleNATSRequest(d.node, d.eksService.ListManagedWorkers), "spinifex-workers"},
 			natsSub{"eks.SetRecoveryDirective", handleNATSRequest(d.node, d.eksService.SetRecoveryDirective), "spinifex-workers"},
 			natsSub{"eks.RestoreSnapshot", handleNATSRequest(d.node, d.eksService.RestoreSnapshot), "spinifex-workers"},
 			natsSub{"eks.TagResource", handleNATSRequest(d.node, d.eksService.TagResource), "spinifex-workers"},

tests/e2e/eks/cp_isolation_test.go

@@ -27,9 +27,9 @@ const nodeGroupLabelRows = `jsonpath={range .items[*]}{.metadata.name} {.metadat
 const nodeReadyRows = `jsonpath={range .items[*]}{.metadata.name}{"="}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}`
 
 // runCPIsolation proves control-plane hosts never register as Nodes, as on AWS:
-// only nodegroup workers are listed and one becomes Ready, a DaemonSet tolerating
-// every taint runs on workers only, metrics-server and its APIService become
-// available, and logs work for every kube-system pod on a Ready node.
+// only nodegroup workers are listed, one becomes Ready, deleted nodegroups leave
+// no Node behind, a DaemonSet tolerating every taint runs on workers only,
+// metrics-server and its APIService become available, and kube-system logs work.
 func runCPIsolation(t *testing.T, c *harness.AWSClient, env *harness.Env, artifacts string, fx *clusterFixture) {
 	nodeRoleArn, _ := ensureNodeRole(t, c, fx)
 	createWorkerNodegroup(t, c, fx, cpIsolationNodegroup, nodeRoleArn, nil)
@@ -37,19 +37,39 @@ func runCPIsolation(t *testing.T, c *harness.AWSClient, env *harness.Env, artifa
 	kcPath := writeKubeconfig(t, artifacts, fx.Cluster)
 	kc := harness.NewKubectl(t, kcPath, getTokenEnv(t, env))
 
+	// Earlier subtests' nodegroups are deleted by now, so a Ready Node from one
+	// of them cannot stand in for this nodegroup's worker.
 	t.Run("WorkerNodeReady", func(t *testing.T) {
 		harness.EventuallyErr(t, func() error {
-			out, err := kc.Run(30*time.Second, "get", "nodes", "-o", nodeReadyRows)
+			out, err := kc.Run(30*time.Second, "get", "nodes",
+				"-l", "eks.amazonaws.com/nodegroup="+cpIsolationNodegroup, "-o", nodeReadyRows)
 			if err != nil {
 				return fmt.Errorf("kubectl get nodes: %v\n%s", err, out)
 			}
 			if !strings.Contains(out, "=True") {
-				return fmt.Errorf("no Ready node yet:\n%s", out)
+				return fmt.Errorf("no Ready %s Node yet:\n%s", cpIsolationNodegroup, out)
 			}
 			return nil
 		}, 6*time.Minute, 5*time.Second)
 	})
 
+	// As on AWS, a terminated worker's Node is deleted: once every other
+	// nodegroup is gone, the only Nodes left are this nodegroup's.
+	t.Run("TerminatedWorkersLeaveNoNodes", func(t *testing.T) {
+		harness.EventuallyErr(t, func() error {
+			out, err := kc.Run(30*time.Second, "get", "nodes", "-o", nodeGroupLabelRows)
+			if err != nil {
+				return fmt.Errorf("kubectl get nodes: %v\n%s", err, out)
+			}
+			for _, row := range strings.Split(strings.TrimSpace(out), "\n") {
+				if f := strings.Fields(row); len(f) == 2 && f[1] != cpIsolationNodegroup {
+					return fmt.Errorf("node %s of deleted nodegroup %s is still registered:\n%s", f[0], f[1], out)
+				}
+			}
+			return nil
+		}, 4*time.Minute, 10*time.Second)
+	})
+
 	t.Run("OnlyWorkersAreNodes", func(t *testing.T) {
 		out, err := kc.Run(30*time.Second, "get", "nodes", "-o", nodeGroupLabelRows)
 		require.NoErrorf(t, err, "get nodes:\n%s", out)
@@ -71,19 +91,21 @@ func runCPIsolation(t *testing.T, c *harness.AWSClient, env *harness.Env, artifa
 			_, _ = kc.Run(60*time.Second, "delete", "daemonset", ds, "-n", "default", "--ignore-not-found")
 		})
 
+		// One "<pod>=<Ready status>" row per pod: a pod bound to a Node that never
+		// reported has no Ready condition, and its row must count as not Ready.
 		harness.EventuallyErr(t, func() error {
 			ready, err := kc.Run(30*time.Second, "get", "pods", "-l", "app="+ds,
-				"-o", `jsonpath={range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}`)
+				"-o", `jsonpath={range .items[*]}{.metadata.name}{"="}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}`)
 			if err != nil {
 				return fmt.Errorf("get daemonset pods: %v\n%s", err, ready)
 			}
-			states := strings.Fields(ready)
-			if len(states) == 0 {
+			rows := strings.Fields(ready)
+			if len(rows) == 0 {
 				return fmt.Errorf("daemonset has no pods yet")
 			}
-			for _, s := range states {
-				if s != "True" {
-					return fmt.Errorf("daemonset pods not all Ready: %v", states)
+			for _, row := range rows {
+				if _, status, _ := strings.Cut(row, "="); status != "True" {
+					return fmt.Errorf("daemonset pods not all Ready: %v", rows)
 				}
 			}
 			return nil

tests/e2e/eks/eks_test.go

@@ -536,12 +536,7 @@ func runIRSAPod(t *testing.T, c *harness.AWSClient, env *harness.Env, artifacts
 		},
 	})
 	require.NoError(t, err, "create-nodegroup")
-	t.Cleanup(func() {
-		_, _ = c.EKS.DeleteNodegroup(&eks.DeleteNodegroupInput{
-			ClusterName:   aws.String(fx.ClusterName),
-			NodegroupName: aws.String(nodegroup),
-		})
-	})
+	t.Cleanup(func() { harness.DeleteEKSNodegroupAndWait(t, c, fx.ClusterName, nodegroup) })
 	harness.EventuallyErr(t, func() error {
 		out, derr := c.EKS.DescribeNodegroup(&eks.DescribeNodegroupInput{
 			ClusterName:   aws.String(fx.ClusterName),
@@ -789,12 +784,7 @@ func runEBSCSIVolume(t *testing.T, c *harness.AWSClient, env *harness.Env, artif
 		},
 	})
 	require.NoError(t, err, "create-nodegroup")
-	t.Cleanup(func() {
-		_, _ = c.EKS.DeleteNodegroup(&eks.DeleteNodegroupInput{
-			ClusterName:   aws.String(fx.ClusterName),
-			NodegroupName: aws.String(nodegroup),
-		})
-	})
+	t.Cleanup(func() { harness.DeleteEKSNodegroupAndWait(t, c, fx.ClusterName, nodegroup) })
 	harness.EventuallyErr(t, func() error {
 		out, derr := c.EKS.DescribeNodegroup(&eks.DescribeNodegroupInput{
 			ClusterName:   aws.String(fx.ClusterName),

tests/e2e/eks/worker_imds_test.go

@@ -37,12 +37,7 @@ func createWorkerNodegroup(t *testing.T, c *harness.AWSClient, fx *clusterFixtur
 		},
 	})
 	require.NoError(t, err, "create-nodegroup")
-	t.Cleanup(func() {
-		_, _ = c.EKS.DeleteNodegroup(&eks.DeleteNodegroupInput{
-			ClusterName:   aws.String(fx.ClusterName),
-			NodegroupName: aws.String(nodegroup),
-		})
-	})
+	t.Cleanup(func() { harness.DeleteEKSNodegroupAndWait(t, c, fx.ClusterName, nodegroup) })
 	harness.EventuallyErr(t, func() error {
 		out, derr := c.EKS.DescribeNodegroup(&eks.DescribeNodegroupInput{
 			ClusterName:   aws.String(fx.ClusterName),

tests/e2e/harness/eks.go

@@ -71,3 +71,31 @@ func isClusterNotFound(err error) bool {
 	}
 	return false
 }
+
+// DeleteEKSNodegroupAndWait deletes a nodegroup and polls DescribeNodegroup
+// until it returns ResourceNotFoundException, so a later step never runs
+// beside a worker that is still going away.
+func DeleteEKSNodegroupAndWait(t *testing.T, c *AWSClient, cluster, nodegroup string) {
+	t.Helper()
+	_, err := c.EKS.DeleteNodegroup(&eks.DeleteNodegroupInput{
+		ClusterName:   aws.String(cluster),
+		NodegroupName: aws.String(nodegroup),
+	})
+	if err != nil && !isClusterNotFound(err) {
+		t.Errorf("delete-nodegroup %s: %v", nodegroup, err)
+		return
+	}
+	EventuallyErr(t, func() error {
+		_, err := c.EKS.DescribeNodegroup(&eks.DescribeNodegroupInput{
+			ClusterName:   aws.String(cluster),
+			NodegroupName: aws.String(nodegroup),
+		})
+		if err == nil {
+			return fmt.Errorf("nodegroup %s still exists", nodegroup)
+		}
+		if isClusterNotFound(err) {
+			return nil
+		}
+		return fmt.Errorf("describe-nodegroup %s: %w", nodegroup, err)
+	}, 5*time.Minute, 5*time.Second)
+}