fix(ci): release the ISO build's root-owned cache when the ISO cells end (#1393)

The ISO cells build the rootfs as root inside the workspace and only cleaned
it at the start of their own job, so the next job on the same runner from any
workflow failed its checkout on files it could not remove.

.github/workflows/e2e-nightly.yml

@@ -1203,6 +1203,16 @@ jobs:
           sleep 2
           rm -f "$TMPDIR"/tmp.*.qcow2 "$TMPDIR"/tmp.*.log "$TMPDIR"/tmp.*-node.log
 
+      # The ISO build runs as root inside the workspace, and the next job on
+      # this runner, from any workflow, cannot remove what it leaves there.
+      - name: Release the workspace
+        if: always()
+        run: |
+          sudo umount -R -l "$GITHUB_WORKSPACE"/mulga/scripts/iso-builder/.cache/rootfs 2>/dev/null || true
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/mulga/scripts/iso-builder/.cache \
+              "$GITHUB_WORKSPACE"/mulga/scripts/iso-builder/dist
+          sudo chown -R "$(id -u):$(id -g)" "$GITHUB_WORKSPACE"
+
   iso-multi:
     name: cell-22 (iso/dhcp/three-node)
     # After cell-21 rather than beside it: same concurrency group, and running
@@ -1354,6 +1364,16 @@ jobs:
           rm -rf /tmp/cell22-artifacts-$GITHUB_RUN_ID /tmp/cell22-$GITHUB_RUN_ID.log \
                  "$WORK_DIR"
 
+      # The ISO build runs as root inside the workspace, and the next job on
+      # this runner, from any workflow, cannot remove what it leaves there.
+      - name: Release the workspace
+        if: always()
+        run: |
+          sudo umount -R -l "$GITHUB_WORKSPACE"/mulga/scripts/iso-builder/.cache/rootfs 2>/dev/null || true
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/mulga/scripts/iso-builder/.cache \
+              "$GITHUB_WORKSPACE"/mulga/scripts/iso-builder/dist
+          sudo chown -R "$(id -u):$(id -g)" "$GITHUB_WORKSPACE"
+
   # Predastore throughput ("cell 23"). Predastore has no runners of its own and
   # spinifex depends on it, so the benchmark lives here and checks predastore
   # out as a sibling.

cmd/eks-gateway-fetch/main.go

@@ -6,6 +6,7 @@
 //
 //	eks-gateway-fetch -resource addons   # GET /clusters/{cluster}/internal-addons/{accountId}
 //	eks-gateway-fetch -resource recovery -instance-id i-… # GET /clusters/{cluster}/internal-recovery/{accountId}/{instanceId}
+//	eks-gateway-fetch -resource workers  # GET /clusters/{cluster}/internal-workers/{accountId}
 //
 // Flags default to environment variables seeded by cloud-init:
 // EKS_GATEWAY_URL, EKS_GATEWAY_CA, EKS_ACCESS_KEY, EKS_SECRET_KEY, EKS_REGION,
@@ -33,7 +34,7 @@ func main() {
 	flag.StringVar(&cfg.AccountID, "account-id", os.Getenv("EKS_ACCOUNT_ID"), "Cluster account ID")
 	flag.StringVar(&cfg.ClusterName, "cluster", os.Getenv("EKS_CLUSTER_NAME"), "Cluster name")
 	flag.StringVar(&cfg.InstanceID, "instance-id", os.Getenv("EKS_INSTANCE_ID"), "This member's EC2 instance ID (recovery resource)")
-	flag.StringVar(&cfg.Resource, "resource", "addons", "Resource to fetch: addons | recovery")
+	flag.StringVar(&cfg.Resource, "resource", "addons", "Resource to fetch: addons | recovery | workers")
 	flag.Parse()
 
 	if err := gatewayfetch.Run(context.Background(), cfg); err != nil {

docs/install/install-oracle-linux/README.md

@@ -164,7 +164,7 @@ On Oracle Linux the installer makes these changes, which Debian hosts do not nee
 - **Repositories.** It enables `ol10_UEKR8` and `ol10_kvm_utils`, and adds `qemu-kvm*` and `qemu-img*` to the `excludepkgs` of `ol10_appstream`, keeping any excludes already set, so a later AppStream build can never replace the KVM Utilities QEMU.
 - **NFV SIG.** It writes `/etc/yum.repos.d/spinifex-nfv-sig.repo` with an `includepkgs=` list of the OVS and OVN packages, and imports the SIG key to `/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-NFV` only when its fingerprint is `35154228174901BEFA8E69A621465E289D2A76A7`.
 - **OVS and OVN user.** Both run as root, as on Debian, through `OVS_USER_ID` and `OVN_USER_ID` in `/etc/sysconfig/openvswitch` and `/etc/sysconfig/ovn`.
-- **DNS.** Where systemd-resolved is installed but not running, it saves `/etc/resolv.conf` to `/var/lib/spinifex-reset/resolv.conf.orig`, starts resolved behind NetworkManager and links `/etc/resolv.conf` to the resolved stub. It then checks that names still resolve and stops, printing the commands that put DNS back, if they do not.
+- **DNS.** Where systemd-resolved is installed but not running, it saves `/etc/resolv.conf` to `/var/lib/spinifex-reset/resolv.conf.orig`, starts resolved behind NetworkManager, tells NetworkManager in `/etc/NetworkManager/conf.d/90-spinifex-dns.conf` to leave `/etc/resolv.conf` alone, and links `/etc/resolv.conf` to the resolved stub. It then checks that names still resolve and stops, printing the commands that put DNS back, if they do not.
 - **chrony.** It appends `confdir /etc/chrony.d` to `/etc/chrony.conf` once, and writes its step policy to `/etc/chrony.d/spinifex.conf`.
 - **`spx` under `sudo`.** Oracle Linux's `sudo` path leaves out `/usr/local/bin`, so the installer links `/usr/bin/spx` to `/usr/local/bin/spx`.
 

scripts/README.md

@@ -6,7 +6,7 @@ Node installation, dev-environment lifecycle, guest image builds, and verificati
 
 | Script | What it does |
 |---|---|
-| `setup.sh` | The production binary installer, served from `install.mulgadc.com`. Installs deps, users, sudoers, the host firewall policy, files, systemd units, logrotate and udev rules, and loads `openvswitch`, `geneve`, `vhost_net` and `tun`, listing them in `/etc/modules-load.d/spinifex.conf` for later boots. It masks firewalld wherever it is installed, since the Spinifex nftables policy is the only supported node firewall. It writes a needrestart drop-in so an apt upgrade never restarts the Spinifex, OVS or OVN units; restarting them after an upgrade is left to the operator. An unset `--firewall` keeps the mode recorded in `/etc/spinifex/firewall/mode`, and otherwise arms the policy when firewalld was active. Where systemd-resolved is installed but not running, resolvconf is absent and NetworkManager runs, it hands host DNS to resolved: it saves `/etc/resolv.conf` once to `/var/lib/spinifex-reset/resolv.conf.orig`, links `/etc/resolv.conf` to the resolved stub, and stops unless resolved has upstream servers and the package repository host still resolves. It links `/usr/bin/spx` to `/usr/local/bin/spx` where sudo's PATH leaves out `/usr/local/bin`, and takes the nbdkit plugin directory from `nbdkit --dump-config`, stopping when nbdkit cannot say. `SETUP_STAGES` runs a subset; `ISO_BUILD=1` runs it inside the ISO builder's chroot. |
+| `setup.sh` | The production binary installer, served from `install.mulgadc.com`. Installs deps, users, sudoers, the host firewall policy, files, systemd units, logrotate and udev rules, and loads `openvswitch`, `geneve`, `vhost_net` and `tun`, listing them in `/etc/modules-load.d/spinifex.conf` for later boots. It masks firewalld wherever it is installed, since the Spinifex nftables policy is the only supported node firewall. It writes a needrestart drop-in so an apt upgrade never restarts the Spinifex, OVS or OVN units; restarting them after an upgrade is left to the operator. An unset `--firewall` keeps the mode recorded in `/etc/spinifex/firewall/mode`, and otherwise arms the policy when firewalld was active. Where systemd-resolved is installed but not running, resolvconf is absent and NetworkManager runs, it hands host DNS to resolved: it saves `/etc/resolv.conf` once to `/var/lib/spinifex-reset/resolv.conf.orig`, writes `/etc/NetworkManager/conf.d/90-spinifex-dns.conf` (`dns=systemd-resolved`, `rc-manager=unmanaged`) so NetworkManager never rewrites `/etc/resolv.conf`, links `/etc/resolv.conf` to the resolved stub, and stops unless the link holds, resolved has upstream servers and the package repository host still resolves. A rerun after that handover adds a missing drop-in, reloads NetworkManager and relinks the stub. It links `/usr/bin/spx` to `/usr/local/bin/spx` where sudo's PATH leaves out `/usr/local/bin`, and takes the nbdkit plugin directory from `nbdkit --dump-config`, stopping when nbdkit cannot say. `SETUP_STAGES` runs a subset; `ISO_BUILD=1` runs it inside the ISO builder's chroot. |
 | `setup-ovn.sh` | Bootstraps a compute node for OVN VPC networking: installs OVN/OVS, creates `br-int`, configures the WAN bridge (bridged, DHCP, or routed-NAT uplink), sets the chassis identity, and applies overlay sysctls. It runs OVS and OVN as root, setting `OVS_USER_ID` and `OVN_USER_ID` where `/etc/sysconfig` holds them, and hooks the socket-permission helper onto `ovs-vswitchd`, `ovn-controller`, `openvswitch-ipsec` and, on EL, `ovn-northd`. `--management` also starts the OVN central services, writes their `ovn-ctl` flags to `/etc/default/ovn-central` (Debian) or `/etc/sysconfig/ovn-northd` (EL), and records the role in `/etc/spinifex/ovn-role`. With clustered databases it returns only once this node's NB and SB members are back in their RAFT clusters under a known leader, and fails after `OVN_RAFT_REJOIN_WAIT` seconds (default 60) otherwise, so reruns one node at a time never take two members of a cluster down. A later run without `--management` refuses while that file, or the `OVN_CTL_OPTS` line older nodes carry in `/etc/default/ovn-central`, records the role. It adds `maxsize 100M` to the OVS and OVN logrotate files, plus `rotate 5` where the file sets no count, and rotates as root, matching the daemons. On a host whose IPsec stack is libreswan it enables, stops and masks `openvswitch-ipsec.service` and `ipsec.service` for the daemon to unmask when IPsec is on, unless `openvswitch-ipsec.service` is already running; elsewhere it enables and starts `openvswitch-ipsec.service`, unless the daemon has masked it to keep IPsec off. The links it creates persist through systemd-networkd where that is enabled, and otherwise through `/etc/spinifex/hostnet.json`, which `spinifex-hostnet.service` (`spx admin hostnet apply`) replays at boot and on every OVS start, starting `ovn-controller` again when an OVS stop took it down; the first run records the choice in `/etc/spinifex/hostnet-backend`. Where NetworkManager runs, it leaves the links Spinifex owns unmanaged. |
 | `install-node.sh` | Forms a multi-node cluster across hosts that are already running Spinifex: rebuilds the OVN database in clustered form, runs `spx admin init` on the first host and `spx admin join` on the rest, then verifies. Before touching any host it refuses hosts of different OS families (`distro_os_family`) or `spx` versions, and `--ipsec on` when any host uses libreswan. The scripted form of [install-multi-node](../docs/install/install-multi-node/README.md). Destroys the joining nodes' logical network state and crypto — see its `--help`. |
 | `dev-install.sh` | Full local dev setup through the production installer: builds from source, packs a tarball, runs `setup.sh`, initialises the cluster, and starts services under systemd. |

scripts/images/eks-node/eks-node-role.sh

@@ -12,11 +12,12 @@ set -eu
 # once.
 #
 #   server      → eks-token-webhook, k3s (server), k3s-first-boot (bootstrap
-#                 publish), mulga-eks-state-report
+#                 publish), mulga-eks-state-report, mulga-eks-node-gc
 #   server-join → eks-token-webhook, k3s (server, joins the first server's etcd
-#                 quorum), mulga-eks-state-report. NO k3s-first-boot: the first
-#                 server already publishes the cluster-identical bootstrap
-#                 artifacts; a join re-publish only races the bootstrap bus.
+#                 quorum), mulga-eks-state-report, mulga-eks-node-gc. NO
+#                 k3s-first-boot: the first server already publishes the
+#                 cluster-identical bootstrap artifacts; a join re-publish only
+#                 races the bootstrap bus.
 #   agent       → k3s-agent
 #
 # Paths are overridable via env so the selection logic is unit-testable (bats)
@@ -140,6 +141,7 @@ case "${ROLE}" in
         # writer owns. HA multi-server addon delivery is tracked separately
         # Server-join nodes do not run this setup.
         svc_enable mulga-eks-addon-sync
+        svc_enable mulga-eks-node-gc
         svc_start eks-token-webhook
         svc_start mulga-eks-k3s-recovery
         svc_start k3s
@@ -147,6 +149,7 @@ case "${ROLE}" in
         svc_start k3s-first-boot
         svc_start mulga-eks-state-report
         svc_start mulga-eks-addon-sync
+        svc_start mulga-eks-node-gc
         ;;
     server-join)
         log "configuring server-join role"
@@ -158,11 +161,13 @@ case "${ROLE}" in
         svc_enable k3s
         svc_enable konnectivity-server
         svc_enable mulga-eks-state-report
+        svc_enable mulga-eks-node-gc
         svc_start eks-token-webhook
         svc_start mulga-eks-k3s-recovery
         svc_start k3s
         svc_start konnectivity-server
         svc_start mulga-eks-state-report
+        svc_start mulga-eks-node-gc
         ;;
     agent)
         log "configuring agent role"

scripts/images/eks-node/eks-node-role_test.sh

@@ -141,6 +141,8 @@ for INIT in openrc systemd; do
             END { print (rn > 0 && kn > 0 && rn < kn) ? "yes" : "no" }
         ' "${CALLS}"
     )"
+    check "server(${INIT}): enables node-gc" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-node-gc)")"
+    check "server(${INIT}): starts node-gc" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-node-gc)")"
     check "server(${INIT}): no k3s-agent" "no" "$(present "${CALLS}" "$(enable_cmd "${INIT}" k3s-agent)")"
     check "server(${INIT}): self-disables" "yes" "$(present "${CALLS}" "$(disable_cmd "${INIT}" eks-node-role)")"
 
@@ -153,6 +155,8 @@ for INIT in openrc systemd; do
     check "server-join(${INIT}): enables webhook" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" eks-token-webhook)")"
     check "server-join(${INIT}): enables k3s" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" k3s)")"
     check "server-join(${INIT}): enables state-report" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-state-report)")"
+    check "server-join(${INIT}): enables node-gc" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-node-gc)")"
+    check "server-join(${INIT}): starts node-gc" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-node-gc)")"
     check "server-join(${INIT}): enables konnectivity-server" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" konnectivity-server)")"
     check "server-join(${INIT}): enables k3s-recovery" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-k3s-recovery)")"
     check "server-join(${INIT}): starts k3s-recovery" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-k3s-recovery)")"
@@ -170,7 +174,7 @@ for INIT in openrc systemd; do
     check "agent(${INIT}): exit 0" 0 "${rc}"
     check "agent(${INIT}): role file" "agent" "$(cat "${ROLE_FILE}")"
     check "agent(${INIT}): enables k3s-agent" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" k3s-agent)")"
-    check "agent(${INIT}): no server services" "yes" "$(absent_all "${CALLS}" "eks-token-webhook" "k3s-first-boot" "$(enable_cmd "${INIT}" k3s)")"
+    check "agent(${INIT}): no server services" "yes" "$(absent_all "${CALLS}" "eks-token-webhook" "k3s-first-boot" "mulga-eks-node-gc" "$(enable_cmd "${INIT}" k3s)")"
 
     # --- agent (inferred from agent.env, no explicit role) ---
     run_case "${INIT}" agent_inferred

scripts/images/eks-node/manifest.conf

@@ -42,6 +42,8 @@ scripts/images/eks-node/mulga-eks-state-report.initd:/etc/init.d/mulga-eks-state
 scripts/images/eks-node/mulga-eks-state-report.sh:/usr/local/sbin/mulga-eks-state-report \
 scripts/images/eks-node/mulga-eks-addon-sync.initd:/etc/init.d/mulga-eks-addon-sync \
 scripts/images/eks-node/mulga-eks-addon-sync.sh:/usr/local/sbin/mulga-eks-addon-sync \
+scripts/images/eks-node/mulga-eks-node-gc.initd:/etc/init.d/mulga-eks-node-gc \
+scripts/images/eks-node/mulga-eks-node-gc.sh:/usr/local/sbin/mulga-eks-node-gc \
 scripts/images/eks-node/konnectivity-server.initd:/etc/init.d/konnectivity-server \
 scripts/images/eks-node/konnectivity-server-prestart.sh:/usr/local/sbin/konnectivity-server-prestart \
 scripts/images/eks-node/konnectivity/konnectivity-agent.yaml:/usr/share/spinifex-eks/konnectivity/konnectivity-agent.yaml \

scripts/images/eks-node/mulga-eks-node-gc.initd

@@ -0,0 +1,27 @@
+#!/sbin/openrc-run
+# mulga-eks-node-gc — server-role service that deletes the Node of every
+# nodegroup worker the nodegroup has terminated, every NODE_GC_INTERVAL
+# seconds. Enabled on every server role by eks-node-role.sh.
+
+description="Mulga EKS terminated-worker Node cleanup"
+command="/usr/local/sbin/mulga-eks-node-gc"
+command_background="yes"
+pidfile="/run/mulga-eks-node-gc.pid"
+output_log="/var/log/mulga-eks-node-gc.log"
+error_log="/var/log/mulga-eks-node-gc.log"
+
+# A Node goes NotReady about 40s after its worker stops, so 30s deletes it
+# within a couple of minutes of the terminate.
+export NODE_GC_INTERVAL="${NODE_GC_INTERVAL:-30}"
+
+depend() {
+    need net
+    after k3s
+}
+
+start_pre() {
+    if [ ! -f /etc/spinifex-eks/first-boot.env ]; then
+        eerror "/etc/spinifex-eks/first-boot.env not present — cloud-init did not seed gateway creds"
+        return 1
+    fi
+}

scripts/images/eks-node/mulga-eks-node-gc.sh

@@ -0,0 +1,89 @@
+#!/bin/sh
+set -eu
+
+# mulga-eks-node-gc — deletes the Node of a nodegroup worker whose instance the
+# nodegroup has terminated (DeleteNodegroup, scale-down, reclaim), as the AWS
+# cloud controller deletes a Node whose instance no longer exists. K3s runs no
+# AWS cloud provider: its own reports every instance as existing, so without
+# this a terminated worker stays a NotReady Node forever, its pods stay bound to
+# it, and once most Nodes are NotReady the node lifecycle controller stops
+# evicting at all.
+#
+# Pull (host→VM): GET /clusters/{cluster}/internal-workers/{accountId}, the
+# instance ID of every worker the cluster's nodegroups still own.
+#
+# A Node is deleted only when all of these hold, so a live worker is never
+# touched: it carries the eks.amazonaws.com/nodegroup label, its providerID is
+# aws:///<az>/<instance-id>, it is not Ready, and its instance is not in the
+# fetched set. A failed fetch deletes nothing.
+#
+# Runs on every server; deletes are idempotent. Loops every NODE_GC_INTERVAL
+# seconds in service mode, one-shot otherwise.
+
+ENVFILE=${ENVFILE:-/etc/spinifex-eks/first-boot.env}
+[ -f "${ENVFILE}" ] || { logger -t mulga-eks-node-gc "${ENVFILE} missing — exiting"; exit 0; }
+# set -a so the sourced creds are exported to the eks-gateway-fetch child.
+set -a
+# shellcheck disable=SC1090
+. "${ENVFILE}"
+set +a
+
+: "${EKS_GATEWAY_URL:?}"
+: "${EKS_ACCOUNT_ID:?}"
+: "${EKS_CLUSTER_NAME:?}"
+
+KUBECONFIG=${KUBECONFIG:-/etc/rancher/k3s/k3s.yaml}
+export KUBECONFIG
+
+# One "name|providerID|nodegroup|Ready" row per Node.
+NODE_ROWS='jsonpath={range .items[*]}{.metadata.name}{"|"}{.spec.providerID}{"|"}{.metadata.labels.eks\.amazonaws\.com/nodegroup}{"|"}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}'
+
+log() {
+    logger -t mulga-eks-node-gc "$*"
+    echo "[mulga-eks-node-gc] $*" > "${CONSOLE:-/dev/console}" 2>/dev/null || true
+}
+
+# stale_nodes LIVE_FILE — reads Node rows on stdin and prints the name of each
+# Node to delete: a nodegroup worker, not Ready, whose instance is not listed
+# in LIVE_FILE.
+stale_nodes() {
+    awk -F'|' -v live="$1" '
+        BEGIN { while ((getline id < live) > 0) keep[id] = 1 }
+        $3 == "" || $4 == "True" { next }
+        $2 !~ /^aws:\/\/\/[^\/]+\/i-[0-9a-f]+$/ { next }
+        { n = split($2, p, "/"); if (!(p[n] in keep)) print $1 }
+    '
+}
+
+sync_once() {
+    # Nodes are listed before the workers are fetched. A worker is recorded on
+    # its nodegroup before it boots, so every Node listed here is already in a
+    # set fetched after.
+    if ! rows=$(kubectl get nodes -o "${NODE_ROWS}" 2>/dev/null); then
+        log "kubectl get nodes failed; skipping this pass"
+        return 0
+    fi
+    live=$(mktemp)
+    if ! eks-gateway-fetch -resource workers > "${live}" 2>/dev/null; then
+        log "fetch of the cluster's nodegroup workers failed; deleting nothing"
+        rm -f "${live}"
+        return 0
+    fi
+    printf '%s\n' "${rows}" | stale_nodes "${live}" | while read -r node; do
+        if kubectl delete node "${node}" --ignore-not-found --wait=false >/dev/null 2>&1; then
+            log "deleted Node ${node}: its instance is no longer a nodegroup worker"
+        else
+            log "delete of Node ${node} failed; retrying next pass"
+        fi
+    done
+    rm -f "${live}"
+}
+
+if [ -n "${NODE_GC_INTERVAL:-}" ] && [ "${NODE_GC_INTERVAL}" -gt 0 ] 2>/dev/null; then
+    while true; do
+        sync_once
+        sleep "${NODE_GC_INTERVAL}"
+    done
+else
+    sync_once
+fi

scripts/images/eks-node/mulga-eks-node-gc_test.sh

@@ -0,0 +1,107 @@
+#!/bin/sh
+# Self-contained POSIX test for mulga-eks-node-gc.sh. kubectl, eks-gateway-fetch
+# and logger are stubbed on PATH: kubectl serves canned Node rows and records
+# every delete, and eks-gateway-fetch serves a canned worker set or fails.
+#
+# Run: sh scripts/images/eks-node/mulga-eks-node-gc_test.sh
+set -eu
+
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
+AGENT="${SCRIPT_DIR}/mulga-eks-node-gc.sh"
+WORK=$(mktemp -d)
+trap 'rm -rf "${WORK}"' EXIT
+
+STUBBIN="${WORK}/bin"
+mkdir -p "${STUBBIN}"
+
+cat > "${STUBBIN}/kubectl" <<'EOF'
+#!/bin/sh
+case "$1 $2" in
+    "get nodes")
+        [ "${NODES_RC:-0}" -eq 0 ] || exit "${NODES_RC}"
+        printf '%s\n' "${NODES_BODY:-}" ;;
+    "delete node") echo "$3" >> "${DELETED}" ;;
+esac
+EOF
+
+cat > "${STUBBIN}/eks-gateway-fetch" <<'EOF'
+#!/bin/sh
+echo "$*" >> "${FETCHED}"
+[ "${FETCH_RC:-0}" -eq 0 ] || exit "${FETCH_RC}"
+printf '%s' "${LIVE_BODY:-}"
+EOF
+
+cat > "${STUBBIN}/logger" <<'EOF'
+#!/bin/sh
+:
+EOF
+
+chmod +x "${STUBBIN}"/*
+PATH="${STUBBIN}:${PATH}"
+export PATH
+
+ENVFILE="${WORK}/first-boot.env"
+cat > "${ENVFILE}" <<'EOF'
+EKS_GATEWAY_URL=https://gw.invalid:9999
+EKS_ACCOUNT_ID=000000000001
+EKS_CLUSTER_NAME=demo3
+EOF
+CONSOLE="${WORK}/console"
+DELETED="${WORK}/deleted"
+FETCHED="${WORK}/fetched"
+export ENVFILE CONSOLE DELETED FETCHED
+
+PASS=0
+FAIL=0
+check() {
+    if [ "$2" = "$3" ]; then
+        PASS=$((PASS + 1))
+    else
+        FAIL=$((FAIL + 1))
+        printf 'FAIL: %s\n  want: %s\n  got:  %s\n' "$1" "$2" "$3"
+    fi
+}
+
+run_agent() {
+    : > "${DELETED}"
+    : > "${FETCHED}"
+    sh "${AGENT}"
+}
+
+deleted() { tr '\n' ' ' < "${DELETED}" | sed 's/ $//'; }
+
+# Every way a Node can look: only a NotReady nodegroup worker whose instance the
+# cluster's nodegroups no longer own is deleted.
+NODES_BODY='live|aws:///z1/i-0aaa|ng1|True
+gone|aws:///z1/i-0bbb|ng1|Unknown
+gone-never-reported|aws:///z1/i-0ddd|ng2|
+owned-notready|aws:///z1/i-0ccc|ng1|False
+gone-but-ready|aws:///z1/i-0eee|ng1|True
+self-managed|aws:///z1/i-0fff||Unknown
+k3s-provider|k3s://k3s-node|ng1|Unknown'
+LIVE_BODY='i-0aaa
+i-0ccc
+'
+export NODES_BODY LIVE_BODY
+
+run_agent
+check "deletes only NotReady nodegroup Nodes whose instance is gone" "gone gone-never-reported" "$(deleted)"
+check "fetches the workers resource" "-resource workers" "$(cat "${FETCHED}")"
+
+# A cluster with no nodegroups left: every NotReady nodegroup Node goes.
+LIVE_BODY='' run_agent
+check "empty worker set deletes every NotReady nodegroup Node" "gone gone-never-reported owned-notready" "$(deleted)"
+
+# A failed fetch is not an empty set: nothing is deleted.
+FETCH_RC=1 run_agent
+check "failed fetch deletes nothing" "" "$(deleted)"
+
+# Without a Node listing the workers are not fetched, so a set fetched before a
+# Node registered can never be compared against it.
+NODES_RC=1 run_agent
+check "failed node listing deletes nothing" "" "$(deleted)"
+check "failed node listing does not fetch" "" "$(cat "${FETCHED}")"
+
+echo "---"
+echo "PASS=${PASS} FAIL=${FAIL}"
+[ "${FAIL}" -eq 0 ]

scripts/images/eks-node/setup.sh

@@ -44,9 +44,10 @@ ln -sf /usr/local/bin/k3s /usr/local/bin/ctr
 # role's services are baked; the selector enables the right ones at first boot.
 chmod 0755 /etc/init.d/eks-node-role /etc/init.d/k3s /etc/init.d/k3s-agent \
     /etc/init.d/eks-token-webhook /etc/init.d/k3s-first-boot /etc/init.d/mulga-eks-state-report \
-    /etc/init.d/mulga-eks-addon-sync /etc/init.d/konnectivity-server
+    /etc/init.d/mulga-eks-addon-sync /etc/init.d/mulga-eks-node-gc /etc/init.d/konnectivity-server
 chmod 0755 /usr/local/sbin/eks-node-role /usr/local/sbin/k3s-first-boot \
-    /usr/local/sbin/mulga-eks-state-report /usr/local/sbin/mulga-eks-addon-sync
+    /usr/local/sbin/mulga-eks-state-report /usr/local/sbin/mulga-eks-addon-sync \
+    /usr/local/sbin/mulga-eks-node-gc
 # start_pre helpers shared with the systemd wrappers that will eventually
 # replace these runscripts.
 chmod 0755 /usr/local/sbin/k3s-prestart /usr/local/sbin/k3s-agent-prestart \

scripts/setup.sh

@@ -1014,6 +1014,9 @@ RESOLV_CONF="${SPINIFEX_RESOLV_CONF:-/etc/resolv.conf}"
 RESOLVED_STUB=/run/systemd/resolve/stub-resolv.conf
 # node-reset.sh keeps /var/lib/spinifex-reset, so the copy survives until uninstall.
 RESOLV_CONF_SAVED="${SPINIFEX_RESOLV_CONF_SAVED:-/var/lib/spinifex-reset/resolv.conf.orig}"
+# NetworkManager started while resolv.conf is a plain file rewrites it on its
+# next DNS update, replacing the stub link; this drop-in stops that.
+NM_DNS_DROPIN="${SPINIFEX_NM_DNS_DROPIN:-/etc/NetworkManager/conf.d/90-spinifex-dns.conf}"
 
 # systemd-resolved ships resolvconf as a link to resolvectl, which only works
 # while resolved runs, so that one is not a resolvconf backend.
@@ -1066,10 +1069,52 @@ dns_handover_failed() {
     fatal "Host DNS through systemd-resolved does not work: $1.
     To put host DNS back as it was before this installer:
       sudo systemctl disable --now systemd-resolved
+      sudo rm -f $NM_DNS_DROPIN
       sudo cp -a --remove-destination $RESOLV_CONF_SAVED $RESOLV_CONF
       sudo systemctl restart NetworkManager"
 }
 
+# Prints the rc-manager NetworkManager would use if it is not unmanaged, which
+# means a file read after the Spinifex drop-in overrides it.
+nm_rc_manager_override() {
+    local config
+    config=$($SUDO NetworkManager --print-config) || return 1
+    printf '%s\n' "$config" | awk -F= '
+        /^\[/ { main = ($0 == "[main]"); next }
+        main && $1 == "rc-manager" { rc = $2 }
+        END { if (rc != "unmanaged") print "rc-manager=" (rc == "" ? "symlink" : rc) }'
+}
+
+# Writes the NetworkManager drop-in and checks nothing overrides it. Returns 1
+# when it was already in place.
+install_nm_dns_dropin() {
+    local want override changed=1
+    want=$(printf '[main]\ndns=systemd-resolved\nrc-manager=unmanaged')
+    if [ "$(cat "$NM_DNS_DROPIN" 2>/dev/null)" != "$want" ]; then
+        $SUDO mkdir -p "$(dirname "$NM_DNS_DROPIN")"
+        printf '%s\n' "$want" | $SUDO tee "$NM_DNS_DROPIN" > /dev/null || dns_handover_failed "cannot write $NM_DNS_DROPIN"
+        info "  $NM_DNS_DROPIN: NetworkManager leaves $RESOLV_CONF alone"
+        changed=0
+    fi
+    override=$(nm_rc_manager_override) || dns_handover_failed "cannot read NetworkManager's configuration with 'NetworkManager --print-config'"
+    if [ -n "$override" ]; then
+        dns_handover_failed "NetworkManager reads $override from a file after $NM_DNS_DROPIN, so it would replace the $RESOLV_CONF link. Remove that setting ('NetworkManager --print-config' shows the files it reads) and re-run this installer"
+    fi
+    return "$changed"
+}
+
+# Links the stub and checks the link is still there a second later, relinking a
+# bounded number of times in case a write NetworkManager had queued lands first.
+link_resolved_stub() {
+    local i
+    for i in 1 2 3 4 5; do
+        $SUDO ln -sfn "$RESOLVED_STUB" "$RESOLV_CONF"
+        sleep 1
+        [ "$(readlink "$RESOLV_CONF")" = "$RESOLVED_STUB" ] && return 0
+    done
+    dns_handover_failed "$RESOLV_CONF is not a link to $RESOLVED_STUB 5 seconds after linking it, so something keeps rewriting it"
+}
+
 # Checks resolved has upstream servers and still resolves the package
 # repository host, the one name this host is known to have resolved.
 check_resolved_dns() {
@@ -1103,8 +1148,13 @@ hand_dns_to_resolved() {
     [ "${ISO_BUILD:-0}" = "1" ] && return 0
 
     if systemctl is-active --quiet systemd-resolved.service; then
-        # A handover made by an earlier run is checked again on every run.
+        # A handover made by an earlier run is checked again, and repaired, on
+        # every run: installs before the drop-in could lose the link.
         if [ -e "$RESOLV_CONF_SAVED" ] || [ -L "$RESOLV_CONF_SAVED" ]; then
+            if install_nm_dns_dropin && systemctl is-active --quiet NetworkManager.service; then
+                $SUDO systemctl reload NetworkManager.service || dns_handover_failed "NetworkManager did not reload"
+            fi
+            [ "$(readlink "$RESOLV_CONF")" = "$RESOLVED_STUB" ] || link_resolved_stub
             check_resolved_dns
         fi
         return 0
@@ -1131,10 +1181,15 @@ hand_dns_to_resolved() {
         $SUDO cp -a "$RESOLV_CONF" "$RESOLV_CONF_SAVED" || fatal "Cannot save $RESOLV_CONF to $RESOLV_CONF_SAVED"
         info "  saved $RESOLV_CONF to $RESOLV_CONF_SAVED"
     fi
+    install_nm_dns_dropin || true
     $SUDO systemctl enable --now systemd-resolved.service > /dev/null 2>&1 \
         || dns_handover_failed "systemd-resolved did not start"
     $SUDO systemctl restart NetworkManager.service || dns_handover_failed "NetworkManager did not restart"
-    $SUDO ln -sfn "$RESOLVED_STUB" "$RESOLV_CONF"
+    # Best effort: a link with no DHCP server can hold startup past the timeout.
+    if command -v nm-online > /dev/null; then
+        nm-online -s -q --timeout=15 || true
+    fi
+    link_resolved_stub
     check_resolved_dns
     info "  $RESOLV_CONF -> $RESOLVED_STUB"
 }

scripts/setup_host_test.sh

@@ -20,9 +20,10 @@ fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
 echo "-- setup.sh host DNS, chrony, needrestart and nbdkit"
 
 mkdir -p "$TMP/sys"
-for tool in bash awk sed grep readlink basename dirname cp ln mkdir cat head tail wc tee rm chmod install; do
+for tool in bash awk sed grep readlink basename dirname cp ln mkdir cat head tail wc tee rm chmod install mv; do
     ln -s "$(command -v "$tool")" "$TMP/sys/$tool"
 done
+ln -s "$(command -v sleep)" "$TMP/sys/realsleep"
 
 # stub <name> <body> — a stub command; $S is the fixture's state directory.
 stub() {
@@ -30,10 +31,17 @@ stub() {
     chmod +x "$TMP/bin/$1"
 }
 
+# nm_settle — waits for the fixture's NetworkManager to finish writing.
+nm_settle() {
+    n=0
+    while [ -e "$TMP/state/nm-pending" ] && [ $n -lt 500 ]; do "$TMP/sys/realsleep" 0.01; n=$((n + 1)); done
+}
+
 # host — a fresh fixture: resolved installed and inactive, resolvconf the
 # resolvectl link, NetworkManager running with no DNS settings, a dnf host
 # whose first enabled repository is yum.oracle.com.
 host() {
+    nm_settle
     rm -rf "${TMP:?}/state" "${TMP:?}/root" "${TMP:?}/bin"
     mkdir -p "$TMP/state/active" "$TMP/root/etc" "$TMP/root/usr/lib/systemd/system" "$TMP/bin"
     : > "$TMP/root/usr/lib/systemd/system/systemd-resolved.service"
@@ -42,11 +50,28 @@ host() {
     printf '[main]\n# dns=default\nconfigure-and-quit=no\n' > "$TMP/state/nm-config"
     printf 'Global:\nLink 2 (br-wan): 192.168.1.1\n' > "$TMP/state/resolvectl-dns"
     printf '[ol10_UEKR8]\nbaseurl = https://mirror.example/uek/\nenabled = 0\n[ol10_baseos_latest]\nbaseurl = https://yum.oracle.com/repo/OracleLinux/OL10/baseos/latest/x86_64/\nenabled = 1\n' > "$TMP/state/dnf-dump"
+    # A (re)started NetworkManager whose rc-manager is not unmanaged writes
+    # resolv.conf as a plain file on its first DNS update, which can land just
+    # after the installer links the stub.
     stub systemctl '[ "$1" = is-active ] && { [ -e "$S/active/$3" ]; exit; }
 echo "$*" >> "$S/calls"
 [ "$1 $2" = "enable --now" ] && : > "$S/active/$3"
+case "$1 $2" in
+    "restart NetworkManager.service" | "reload NetworkManager.service")
+        NetworkManager --print-config | grep -qx rc-manager=unmanaged && exit 0
+        : > "$S/nm-pending"
+        (
+            n=0
+            while [ ! -L "$S/../root/etc/resolv.conf" ] && [ $n -lt 300 ]; do realsleep 0.01; n=$((n + 1)); done
+            printf "nameserver 192.168.1.1\n" > "$S/nm-resolv.conf"
+            mv -f "$S/nm-resolv.conf" "$S/../root/etc/resolv.conf"
+            rm -f "$S/nm-pending"
+        ) > /dev/null 2>&1 &
+        ;;
+esac
 exit 0'
-    stub NetworkManager 'cat "$S/nm-config"'
+    stub NetworkManager 'cat "$S/nm-config"; cat "$S/../root/nm/90-spinifex-dns.conf" "$S/nm-config-late" 2>/dev/null; :'
+    stub nm-online ':'
     stub resolvectl '[ "$1" = dns ] && cat "$S/resolvectl-dns"'
     stub resolvectl-real ':'
     ln -s resolvectl "$TMP/bin/resolvconf"
@@ -60,10 +85,17 @@ run() {
     RC=0
     OUT=$(PATH="$TMP/bin:$TMP/sys" DISTRO_ROOT="$TMP/root" \
         SPINIFEX_RESOLV_CONF="$TMP/root/etc/resolv.conf" SPINIFEX_RESOLV_CONF_SAVED="$TMP/root/saved/resolv.conf.orig" \
+        SPINIFEX_NM_DNS_DROPIN="$TMP/root/nm/90-spinifex-dns.conf" \
         INSTALL_SPINIFEX_LIB_ONLY=1 bash -c '. "$1"; shift; for f in "$@"; do $f; done' _ "$SETUP" "$@" 2>&1) || RC=$?
 }
 
 calls() { cat "$TMP/state/calls" 2>/dev/null || true; }
+# linked — whether resolv.conf is still the stub link once NetworkManager settles.
+linked() {
+    nm_settle
+    [ "$(readlink "$TMP/root/etc/resolv.conf")" = /run/systemd/resolve/stub-resolv.conf ]
+}
+dropin() { cat "$TMP/root/nm/90-spinifex-dns.conf" 2>/dev/null || echo "<none>"; }
 saved() { cat "$TMP/root/saved/resolv.conf.orig" 2>/dev/null || echo "<none>"; }
 expect_untouched() {
     if [ "$RC" = 0 ] && [ -z "$(calls)" ] && [ ! -L "$TMP/root/etc/resolv.conf" ] && [ "$(saved)" = "<none>" ]; then
@@ -86,7 +118,7 @@ expect_fatal() { # expect_fatal <desc> <text>...
 # --------------------------------------------------------------------------
 host
 run hand_dns_to_resolved
-if [ "$RC" = 0 ] && [ "$(readlink "$TMP/root/etc/resolv.conf")" = /run/systemd/resolve/stub-resolv.conf ] \
+if [ "$RC" = 0 ] && linked \
     && [ "$(saved)" = "$(printf 'nameserver 192.168.1.1\nsearch example.lan')" ] \
     && [ "$(calls)" = "$(printf 'enable --now systemd-resolved.service\nrestart NetworkManager.service')" ] \
     && [ "$(cat "$TMP/state/looked-up")" = yum.oracle.com ]; then
@@ -98,12 +130,44 @@ fi
 : > "$TMP/state/calls"
 rm -f "$TMP/state/looked-up"
 run hand_dns_to_resolved
-if [ "$RC" = 0 ] && [ -z "$(calls)" ] && [ "$(cat "$TMP/state/looked-up")" = yum.oracle.com ]; then
+if [ "$RC" = 0 ] && [ -z "$(calls)" ] && linked && [ "$(cat "$TMP/state/looked-up")" = yum.oracle.com ]; then
     pass "a rerun after a handover checks DNS again and changes nothing"
 else
     fail "rerun after handover: rc=$RC calls='$(calls)': $OUT"
 fi
 
+if [ "$(dropin)" = "$(printf '[main]\ndns=systemd-resolved\nrc-manager=unmanaged')" ]; then
+    pass "the handover tells NetworkManager to send DNS to resolved and leave resolv.conf alone"
+else
+    fail "NetworkManager drop-in: '$(dropin)'"
+fi
+
+# An install made before the drop-in, whose NetworkManager replaced the link.
+host
+mkdir -p "$TMP/root/saved"
+echo "the original" > "$TMP/root/saved/resolv.conf.orig"
+: > "$TMP/state/active/systemd-resolved.service"
+run hand_dns_to_resolved
+if [ "$RC" = 0 ] && linked && [ "$(calls)" = "reload NetworkManager.service" ] \
+    && [ "$(dropin)" = "$(printf '[main]\ndns=systemd-resolved\nrc-manager=unmanaged')" ]; then
+    pass "a rerun after a handover whose link NetworkManager replaced adds the drop-in, reloads NM and relinks the stub"
+else
+    fail "rerun repair: rc=$RC calls='$(calls)' link='$(readlink "$TMP/root/etc/resolv.conf")' dropin='$(dropin)': $OUT"
+fi
+
+host
+printf '[main]\nrc-manager=symlink\n' > "$TMP/state/nm-config-late"
+run hand_dns_to_resolved
+expect_fatal "a NetworkManager file read after the drop-in that sets rc-manager stops the install, naming it" \
+    "rc-manager=symlink" "rm -f $TMP/root/nm/90-spinifex-dns.conf"
+case "$(calls)" in *"restart NetworkManager"*) fail "rc-manager override: restarted NetworkManager first: $(calls)" ;; esac
+
+host
+stub ln ':'
+run hand_dns_to_resolved
+expect_fatal "a resolv.conf that will not stay linked to the stub stops the install" \
+    "is not a link to /run/systemd/resolve/stub-resolv.conf"
+
 host
 mkdir -p "$TMP/root/saved"
 echo "the original" > "$TMP/root/saved/resolv.conf.orig"
@@ -155,7 +219,8 @@ host
 printf 'Global:\nLink 2 (br-wan):\n' > "$TMP/state/resolvectl-dns"
 run hand_dns_to_resolved
 expect_fatal "no upstream servers stops the install, naming the check and printing the restore" \
-    "'resolvectl dns' lists no DNS servers" "systemctl disable --now systemd-resolved" "cp -a --remove-destination $TMP/root/saved/resolv.conf.orig"
+    "'resolvectl dns' lists no DNS servers" "systemctl disable --now systemd-resolved" "cp -a --remove-destination $TMP/root/saved/resolv.conf.orig" \
+    "rm -f $TMP/root/nm/90-spinifex-dns.conf"
 
 host
 : > "$TMP/state/getent-fail"

spinifex/agents/eks/gatewayfetch/fetch.go

@@ -16,6 +16,9 @@
 //
 // The recovery resource returns a single epoch\taction\tsnapshot line the on-VM
 // k3s-recovery agent applies before k3s starts (etcd cluster-reset / wipe-rejoin).
+//
+// The workers resource returns one instance ID per line: every worker the
+// cluster's nodegroups still own, which the on-VM node-gc agent keeps Nodes for.
 package gatewayfetch
 
 import (
@@ -46,7 +49,7 @@ type Config struct {
 	Region      string
 	AccountID   string
 	ClusterName string
-	// Resource is "addons" or "recovery".
+	// Resource is "addons", "recovery" or "workers".
 	Resource   string
 	InstanceID string
 	// Out receives the rendered TSV.
@@ -61,8 +64,8 @@ func Run(ctx context.Context, cfg Config) error {
 		return errors.New("--account-id is required (or set EKS_ACCOUNT_ID)")
 	case cfg.ClusterName == "":
 		return errors.New("--cluster is required (or set EKS_CLUSTER_NAME)")
-	case cfg.Resource != "addons" && cfg.Resource != "recovery":
-		return errors.New("--resource must be addons or recovery")
+	case cfg.Resource != "addons" && cfg.Resource != "recovery" && cfg.Resource != "workers":
+		return errors.New("--resource must be addons, recovery or workers")
 	case cfg.Resource == "recovery" && cfg.InstanceID == "":
 		return errors.New("--instance-id is required for --resource recovery (or set EKS_INSTANCE_ID)")
 	}
@@ -80,6 +83,9 @@ func Run(ctx context.Context, cfg Config) error {
 	case "recovery":
 		path = "/clusters/" + cfg.ClusterName + "/internal-recovery/" + cfg.AccountID + "/" + cfg.InstanceID
 		emit = emitRecoveryTSV
+	case "workers":
+		path = "/clusters/" + cfg.ClusterName + "/internal-workers/" + cfg.AccountID
+		emit = emitWorkers
 	default:
 		path = "/clusters/" + cfg.ClusterName + "/internal-addons/" + cfg.AccountID
 		emit = emitAddonsTSV
@@ -163,3 +169,24 @@ func emitRecoveryTSV(out io.Writer, body []byte) error {
 	}
 	return nil
 }
+
+type internalWorkersResponse struct {
+	InstanceIDs []string `json:"instanceIds"`
+}
+
+// emitWorkers writes one managed-worker instance ID per line. An empty set
+// writes nothing, which the node-gc agent tells apart from a failed fetch by
+// this command's exit status, never by the output being empty.
+func emitWorkers(out io.Writer, body []byte) error {
+	var resp internalWorkersResponse
+	if err := json.Unmarshal(body, &resp); err != nil {
+		return fmt.Errorf("unmarshal internal-workers response: %w", err)
+	}
+	w := bufio.NewWriter(out)
+	for _, id := range resp.InstanceIDs {
+		if _, err := fmt.Fprintln(w, id); err != nil {
+			return err
+		}
+	}
+	return w.Flush()
+}

spinifex/agents/eks/gatewayfetch/fetch_test.go

@@ -49,7 +49,7 @@ func TestRun_RejectsBeforeContactingGateway(t *testing.T) {
 	}{
 		{"account", func(c *Config) { c.AccountID = "" }, "--account-id is required (or set EKS_ACCOUNT_ID)"},
 		{"cluster", func(c *Config) { c.ClusterName = "" }, "--cluster is required (or set EKS_CLUSTER_NAME)"},
-		{"resource", func(c *Config) { c.Resource = "secrets" }, "--resource must be addons or recovery"},
+		{"resource", func(c *Config) { c.Resource = "secrets" }, "--resource must be addons, recovery or workers"},
 		{"recovery instance", func(c *Config) { c.Resource = "recovery" }, "--instance-id is required for --resource recovery (or set EKS_INSTANCE_ID)"},
 		{"gateway client", func(c *Config) { c.GatewayURL = "" }, "build gateway client: eksgw: baseURL is required"},
 	}
@@ -92,6 +92,26 @@ func TestRun_RendersRecoveryDirective(t *testing.T) {
 	}
 }
 
+// The gateway's restjson marshaler names the field InstanceIDs; both that and
+// the json-tag spelling decode, so the agent reads either.
+func TestRun_RendersManagedWorkers(t *testing.T) {
+	for _, body := range []string{
+		`{"InstanceIDs":["i-0aaa","i-0bbb"]}`,
+		`{"instanceIds":["i-0aaa","i-0bbb"]}`,
+	} {
+		var hits atomic.Int32
+		cfg, out := gateway(t, "/clusters/demo/internal-workers/123456789012", http.StatusOK, body, &hits)
+		cfg.Resource = "workers"
+
+		if err := Run(context.Background(), cfg); err != nil {
+			t.Fatalf("Run: %v", err)
+		}
+		if got, want := out.String(), "i-0aaa\ni-0bbb\n"; got != want {
+			t.Fatalf("stdout = %q, want %q", got, want)
+		}
+	}
+}
+
 func TestRun_RenderFailureIsNotRetried(t *testing.T) {
 	var hits atomic.Int32
 	cfg, _ := gateway(t, "/clusters/demo/internal-addons/123456789012", http.StatusOK, `{not-json`, &hits)

spinifex/domains/eks/awsapi/authz.go

@@ -50,10 +50,11 @@ var eksScopes = map[string][]resourceSource{
 	"ListClusters":    {sourceAny},
 
 	// Internal control-plane routes. The cluster's owning account is a path
-	// segment on these two, and the ARN names that account rather than the
+	// segment on these three, and the ARN names that account rather than the
 	// system caller's, because that is the account the handler reads.
 	"ListInternalAddons":   {sourceInternalCluster},
 	"GetRecoveryDirective": {sourceInternalCluster},
+	"ListInternalWorkers":  {sourceInternalCluster},
 	// The owning account arrives in the body on these two, and the ARN names
 	// that account for the same reason: it is the account the handler acts in.
 	"PublishInternal":    {sourceInternalBodyCluster},

spinifex/domains/eks/awsapi/authz_test.go

@@ -179,6 +179,10 @@ func TestResourceARNsInternalRoutesUsePathAccount(t *testing.T) {
 	assert.Equal(t,
 		[]string{"arn:aws:eks:ap-southeast-2:999999999999:cluster/prod"},
 		resolveARNs(t, "GetRecoveryDirective", []string{"prod", "999999999999", "i-abc"}, ""))
+
+	assert.Equal(t,
+		[]string{"arn:aws:eks:ap-southeast-2:999999999999:cluster/prod"},
+		resolveARNs(t, "ListInternalWorkers", []string{"prod", "999999999999"}, ""))
 }
 
 // An identifier the gate cannot read authorizes account-wide, leaving the

spinifex/domains/eks/awsapi/gateway_wire_test.go

@@ -93,6 +93,46 @@ func TestEKSRequest_InternalAddonsWireBody(t *testing.T) {
 		rec.Body.String())
 }
 
+// The node-gc agent reads the same REST-JSON shape: the Go field name, sorted
+// IDs drawn from every nodegroup record of the cluster.
+func TestEKSRequest_InternalWorkersWireBody(t *testing.T) {
+	const cpInstanceID = "i-cp0000000000001"
+	_, nc, js := testutil.StartTestJetStream(t)
+	kv, err := ekscluster.GetOrCreateAccountBucket(t.Context(), js, authzAccountID)
+	require.NoError(t, err)
+	require.NoError(t, ekscluster.PutClusterMeta(t.Context(), kv, &ekscluster.ClusterMeta{
+		Name:              "prod",
+		Status:            ekscluster.ClusterStatusActive,
+		ControlPlaneNodes: []ekscluster.ControlPlaneNode{{InstanceID: cpInstanceID}},
+	}))
+	for _, rec := range []*ekscluster.NodegroupRecord{
+		{ClusterName: "prod", Name: "ng-b", InstanceIDs: []string{"i-0ccc", "i-0aaa"}},
+		{ClusterName: "prod", Name: "ng-a", InstanceIDs: []string{"i-0bbb"}},
+	} {
+		require.NoError(t, ekscluster.PutNodegroupRecord(t.Context(), kv, rec))
+	}
+
+	svc, err := ekscluster.NewEKSServiceImpl(ekscluster.EKSServiceDeps{NATSConn: nc, Region: authzRegion})
+	require.NoError(t, err)
+	sub, err := nc.Subscribe("eks.ListManagedWorkers", func(m *nats.Msg) {
+		acct := natsmsg.AccountIDFromMsg(m)
+		natsmsg.ServeNATSRequestCtx(m, func(ctx context.Context, in *ekscluster.ListManagedWorkersInput) (*ekscluster.ListManagedWorkersOutput, error) {
+			return svc.ListManagedWorkers(ctx, in, acct)
+		})
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+
+	gw := scopedEKS(gwtest.Statement("Allow", "eks:*", "*"))
+	gw.nc = nc
+	rec := httptest.NewRecorder()
+	req := httptest.NewRequest(http.MethodGet, "/clusters/prod/internal-workers/"+authzAccountID, nil)
+	require.NoError(t, gw.as(cpAgentContext(cpInstanceID)).serve(rec, req))
+
+	assert.Equal(t, http.StatusOK, rec.Code)
+	assert.Equal(t, `{"InstanceIDs":["i-0aaa","i-0bbb","i-0ccc"]}`, rec.Body.String())
+}
+
 // CreateAddon and UpdateAddon hand their service-account role to iam:PassRole,
 // same as CreateCluster's roleArn and CreateNodegroup's nodeRole; an absent
 // role passes nothing.

spinifex/domains/eks/awsapi/internal_authz_test.go

@@ -97,6 +97,8 @@ func TestAuthorizeInternal_RejectsNonCPPrincipals(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			assertDenied(t, eksawsapi.AuthorizeInternal(t.Context(), nil, "ListInternalAddons",
 				tc.caller, []string{"alpha", tenantAccount}, nil))
+			assertDenied(t, eksawsapi.AuthorizeInternal(t.Context(), nil, "ListInternalWorkers",
+				tc.caller, []string{"alpha", tenantAccount}, nil))
 			assertDenied(t, eksawsapi.AuthorizeInternal(t.Context(), nil, "GetRecoveryDirective",
 				tc.caller, []string{"alpha", tenantAccount, cpInstanceID}, nil))
 			assertDenied(t, eksawsapi.AuthorizeInternal(t.Context(), nil, "PublishInternal",
@@ -124,6 +126,8 @@ func TestAuthorizeInternal_AllowsClusterMember(t *testing.T) {
 
 	require.NoError(t, eksawsapi.AuthorizeInternal(t.Context(), nc, "ListInternalAddons",
 		cpAgent(cpInstanceID), []string{"alpha", tenantAccount}, nil))
+	require.NoError(t, eksawsapi.AuthorizeInternal(t.Context(), nc, "ListInternalWorkers",
+		cpAgent(cpInstanceID), []string{"alpha", tenantAccount}, nil))
 	require.NoError(t, eksawsapi.AuthorizeInternal(t.Context(), nc, "GetRecoveryDirective",
 		cpAgent(cpInstanceID), []string{"alpha", tenantAccount, cpInstanceID}, nil))
 	require.NoError(t, eksawsapi.AuthorizeInternal(t.Context(), nc, "PublishInternal",

spinifex/domains/eks/awsapi/internal_workers.go

@@ -0,0 +1,35 @@
+package awsapi
+
+import (
+	"context"
+	"errors"
+
+	ekscluster "github.com/mulgadc/spinifex/spinifex/domains/eks/cluster"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/nats-io/nats.go"
+)
+
+// internalWorkersOutput is the body returned to the on-VM node-gc agent: the
+// instance ID of every worker the cluster's nodegroups still own.
+type internalWorkersOutput struct {
+	InstanceIDs []string `json:"instanceIds"`
+}
+
+// ListInternalWorkers — GET /clusters/{name}/internal-workers/{acct}.
+// Internal control-plane VM route (not an AWS-SDK action): same carve-out as
+// ListInternalAddons, gated by AuthorizeInternal, which admits only a CP agent
+// serving that account's cluster.
+func ListInternalWorkers(ctx context.Context, natsConn *nats.Conn, clusterName, accountID string) (*internalWorkersOutput, error) {
+	if natsConn == nil {
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	if clusterName == "" || accountID == "" {
+		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+	out, err := ekscluster.NewNATSEKSService(natsConn).ListManagedWorkers(ctx,
+		&ekscluster.ListManagedWorkersInput{ClusterName: clusterName}, accountID)
+	if err != nil {
+		return nil, err
+	}
+	return &internalWorkersOutput{InstanceIDs: out.InstanceIDs}, nil
+}

spinifex/domains/eks/awsapi/routes.go

@@ -60,6 +60,14 @@ var routes = []route{
 			return GetRecoveryDirective(ctx, nc, p[0], p[1], p[2])
 		}},
 
+	// Internal control-plane route: the on-VM node-gc agent pulls the cluster's
+	// live nodegroup workers to delete the Nodes of terminated ones. Same
+	// system-cred carve-out as internal-addons.
+	{Method: "GET", Pattern: "/clusters/{clusterName}/internal-workers/{accountId}", Action: "ListInternalWorkers",
+		Handler: func(ctx context.Context, nc *nats.Conn, acct, callerARN string, p []string, b []byte) (any, error) {
+			return ListInternalWorkers(ctx, nc, p[0], p[1])
+		}},
+
 	// Nodegroup
 	{Method: "POST", Pattern: "/clusters/{clusterName}/node-groups", Action: "CreateNodegroup",
 		Handler: func(ctx context.Context, nc *nats.Conn, acct, callerARN string, p []string, b []byte) (any, error) {

spinifex/domains/eks/awsapi/routes_test.go

@@ -108,6 +108,7 @@ func TestLookupEKSAction_CoversAllActions(t *testing.T) {
 		"WebhookTokenReview":           false,
 		"ListInternalAddons":           false,
 		"GetRecoveryDirective":         false,
+		"ListInternalWorkers":          false,
 		"DescribeCluster":              false,
 		"ListClusters":                 false,
 		"DeleteCluster":                false,

spinifex/domains/eks/cluster/k3s_server_role.go

@@ -22,9 +22,10 @@ const (
 	// WebhookTokenReview (the eks-token-webhook relays `aws eks get-token` bearer
 	// tokens to the token-review broker for host-side STS verification), and
 	// GetRecoveryDirective (the k3s-recovery agent pulls its per-member etcd
-	// recovery directive at boot). The gateway evaluates these per request against
-	// the role's policies.
-	eksServerInlinePolicy = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["eks:PublishInternal","eks:ListInternalAddons","eks:WebhookTokenReview","eks:GetRecoveryDirective"],"Resource":"*"}]}`
+	// recovery directive at boot), and ListInternalWorkers (the node-gc agent
+	// pulls the live nodegroup workers). The gateway evaluates these per request
+	// against the role's policies.
+	eksServerInlinePolicy = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["eks:PublishInternal","eks:ListInternalAddons","eks:WebhookTokenReview","eks:GetRecoveryDirective","eks:ListInternalWorkers"],"Resource":"*"}]}`
 )
 
 // ensureCPInstanceProfile returns the CP instance-profile ARN, or "" to signal
@@ -35,7 +36,7 @@ func (s *EKSServiceImpl) ensureCPInstanceProfile(accountID string) string {
 	iam := s.iamEnsurer()
 	if iam == nil {
 		slog.Warn("EKS: IAM service unwired; CP VM falls back to baked static gateway creds and "+
-			"will be denied eks:PublishInternal, eks:WebhookTokenReview, eks:ListInternalAddons and eks:GetRecoveryDirective (no instance identity to bind)",
+			"will be denied eks:PublishInternal, eks:WebhookTokenReview, eks:ListInternalAddons, eks:GetRecoveryDirective and eks:ListInternalWorkers (no instance identity to bind)",
 			"accountID", accountID)
 		return ""
 	}
@@ -43,7 +44,7 @@ func (s *EKSServiceImpl) ensureCPInstanceProfile(accountID string) string {
 		CPInstanceRoleName, eksServerInlinePolicyName, eksServerInlinePolicy)
 	if err != nil {
 		slog.Error("EKS: ensure CP instance profile failed; CP VM falls back to baked static gateway creds and "+
-			"will be denied eks:PublishInternal, eks:WebhookTokenReview, eks:ListInternalAddons and eks:GetRecoveryDirective (no instance identity to bind)",
+			"will be denied eks:PublishInternal, eks:WebhookTokenReview, eks:ListInternalAddons, eks:GetRecoveryDirective and eks:ListInternalWorkers (no instance identity to bind)",
 			"accountID", accountID, "role", CPInstanceRoleName, "err", err)
 		return ""
 	}

spinifex/domains/eks/cluster/k3s_server_role_test.go

@@ -38,6 +38,9 @@ func TestEnsureK3sServerInstanceProfile_CreatesRolePolicyProfile(t *testing.T) {
 	// these IMDS creds; without this action the gateway 403s the boot-time fetch
 	// and a wedged control plane can never be auto-reset.
 	assert.Contains(t, policy, "eks:GetRecoveryDirective")
+	// The node-gc agent pulls the live nodegroup workers with these IMDS creds;
+	// without this action a terminated worker's Node is never deleted.
+	assert.Contains(t, policy, "eks:ListInternalWorkers")
 
 	require.NotNil(t, f.Profiles[CPInstanceRoleName])
 	assert.Len(t, f.Profiles[CPInstanceRoleName].Roles, 1)

spinifex/domains/eks/cluster/service.go

@@ -44,6 +44,10 @@ type EKSService interface {
 	// AWS-SDK action): the on-VM addon-sync agent fetches staged manifests for a
 	// cluster via the internal-addons gateway route.
 	ListStagedAddonManifests(ctx context.Context, input *ListStagedAddonManifestsInput, accountID string) (*ListStagedAddonManifestsOutput, error)
+	// ListManagedWorkers is an internal control-plane method (not an AWS-SDK
+	// action): the on-VM node-gc agent fetches the cluster's live nodegroup
+	// workers via the internal-workers gateway route.
+	ListManagedWorkers(ctx context.Context, input *ListManagedWorkersInput, accountID string) (*ListManagedWorkersOutput, error)
 
 	// GetRecoveryDirective / SetRecoveryDirective are internal control-plane
 	// methods (not AWS-SDK actions): the on-VM k3s-recovery agent fetches its

spinifex/domains/eks/cluster/service_nats.go

@@ -129,6 +129,10 @@ func (s *NATSEKSService) ListStagedAddonManifests(ctx context.Context, input *Li
 	return natsmsg.NATSRequest[ListStagedAddonManifestsOutput](ctx, s.natsConn, "eks.ListStagedAddonManifests", input, defaultTimeout, accountID)
 }
 
+func (s *NATSEKSService) ListManagedWorkers(ctx context.Context, input *ListManagedWorkersInput, accountID string) (*ListManagedWorkersOutput, error) {
+	return natsmsg.NATSRequest[ListManagedWorkersOutput](ctx, s.natsConn, "eks.ListManagedWorkers", input, defaultTimeout, accountID)
+}
+
 func (s *NATSEKSService) UpdateAddon(ctx context.Context, input *eks.UpdateAddonInput, accountID string) (*eks.UpdateAddonOutput, error) {
 	return natsmsg.NATSRequest[eks.UpdateAddonOutput](ctx, s.natsConn, "eks.UpdateAddon", input, defaultTimeout, accountID)
 }

spinifex/domains/eks/cluster/worker_membership.go

@@ -0,0 +1,46 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"slices"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+// ListManagedWorkersInput names the cluster whose managed-nodegroup workers to
+// list. It is an internal control-plane request (not an AWS-SDK shape), served
+// over NATS for the guest node-gc agent via the internal-workers route.
+type ListManagedWorkersInput struct {
+	ClusterName string `json:"clusterName"`
+}
+
+// ListManagedWorkersOutput carries the instance ID of every worker a managed
+// nodegroup of the cluster still owns, sorted.
+type ListManagedWorkersOutput struct {
+	InstanceIDs []string `json:"instanceIds"`
+}
+
+// ListManagedWorkers returns every worker instance ID recorded on the cluster's
+// nodegroups. A nodegroup drops an ID only after terminating it (DeleteNodegroup,
+// scale-down, reclaim), so the guest deletes a nodegroup Node whose instance is
+// absent here, as the AWS cloud controller deletes a Node whose instance is gone.
+func (s *EKSServiceImpl) ListManagedWorkers(ctx context.Context, input *ListManagedWorkersInput, accountID string) (*ListManagedWorkersOutput, error) {
+	if input == nil || input.ClusterName == "" {
+		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+	acctKV, err := s.acctKVForCluster(ctx, accountID, input.ClusterName)
+	if err != nil {
+		return nil, err
+	}
+	recs, err := ListNodegroupRecords(ctx, acctKV, input.ClusterName)
+	if err != nil {
+		return nil, err
+	}
+	ids := make([]string, 0)
+	for _, rec := range recs {
+		ids = append(ids, rec.InstanceIDs...)
+	}
+	slices.Sort(ids)
+	return &ListManagedWorkersOutput{InstanceIDs: slices.Compact(ids)}, nil
+}

spinifex/domains/eks/cluster/worker_membership_test.go

@@ -0,0 +1,74 @@
+package cluster
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/eks"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func managedWorkers(t *testing.T, f *eksServiceFixture, cluster string) []string {
+	t.Helper()
+	out, err := f.svc.ListManagedWorkers(context.Background(), &ListManagedWorkersInput{ClusterName: cluster}, testAccountID)
+	require.NoError(t, err)
+	return out.InstanceIDs
+}
+
+// A worker leaves the managed set exactly when its nodegroup terminates it, by
+// scale-down or DeleteNodegroup, so the guest node-gc deletes its Node then and
+// leaves every worker still owned by a nodegroup alone.
+func TestListManagedWorkers_DropsWorkersTheNodegroupTerminated(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	for _, ng := range []string{"ng1", "ng2"} {
+		_, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", ng, 2), testAccountID)
+		require.NoError(t, err)
+		markWorkersReady(t, f, "c1", ng, 2)
+		f.svc.WaitLaunches()
+	}
+	ng1, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
+	require.NoError(t, err)
+	ng2, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng2")
+	require.NoError(t, err)
+	require.Len(t, ng1.InstanceIDs, 2)
+	assert.ElementsMatch(t, append(append([]string{}, ng1.InstanceIDs...), ng2.InstanceIDs...), managedWorkers(t, f, "c1"))
+
+	_, err = f.svc.UpdateNodegroupConfig(context.Background(), &eks.UpdateNodegroupConfigInput{
+		ClusterName:   aws.String("c1"),
+		NodegroupName: aws.String("ng1"),
+		ScalingConfig: &eks.NodegroupScalingConfig{DesiredSize: aws.Int64(1)},
+	}, testAccountID)
+	require.NoError(t, err)
+	require.Len(t, f.worker.terminateCalls, 1)
+	scaledIn := f.worker.terminateCalls[0]
+	got := managedWorkers(t, f, "c1")
+	assert.Len(t, got, 3)
+	for _, id := range scaledIn {
+		assert.NotContains(t, got, id, "a worker terminated by scale-down must leave the managed set")
+	}
+
+	_, err = f.svc.DeleteNodegroup(context.Background(), &eks.DeleteNodegroupInput{
+		ClusterName: aws.String("c1"), NodegroupName: aws.String("ng2"),
+	}, testAccountID)
+	require.NoError(t, err)
+	got = managedWorkers(t, f, "c1")
+	assert.Len(t, got, 1)
+	for _, id := range ng2.InstanceIDs {
+		assert.NotContains(t, got, id, "a deleted nodegroup's workers must leave the managed set")
+	}
+}
+
+func TestListManagedWorkers_UnknownClusterIsNotFound(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+
+	_, err := f.svc.ListManagedWorkers(context.Background(), &ListManagedWorkersInput{ClusterName: "ghost"}, testAccountID)
+	code, ok := awserrors.ResolveErrorCode(err)
+	require.True(t, ok)
+	assert.Equal(t, awserrors.ErrorEKSResourceNotFound, code)
+}

spinifex/ingress/aws/pipeline/operations_test.go

@@ -55,6 +55,7 @@ func TestAWSOperationInventoryMeasuresEKSAgainstTheModel(t *testing.T) {
 	require.ElementsMatch(t, []string{
 		"GetRecoveryDirective",
 		"ListInternalAddons",
+		"ListInternalWorkers",
 		"PublishInternal",
 		"WebhookTokenReview",
 	}, eks.Extra)

spinifex/runtime/daemon/daemon.go

@@ -1139,6 +1139,7 @@ func (d *Daemon) subscribeAll() error {
 			natsSub{"eks.UpdateAddon", handleNATSRequest(d.node, d.eksService.UpdateAddon), "spinifex-workers"},
 			natsSub{"eks.ListStagedAddonManifests", handleNATSRequest(d.node, d.eksService.ListStagedAddonManifests), "spinifex-workers"},
 			natsSub{"eks.GetRecoveryDirective", handleNATSRequest(d.node, d.eksService.GetRecoveryDirective), "spinifex-workers"},
+			natsSub{"eks.ListManagedWorkers", handleNATSRequest(d.node, d.eksService.ListManagedWorkers), "spinifex-workers"},
 			natsSub{"eks.SetRecoveryDirective", handleNATSRequest(d.node, d.eksService.SetRecoveryDirective), "spinifex-workers"},
 			natsSub{"eks.RestoreSnapshot", handleNATSRequest(d.node, d.eksService.RestoreSnapshot), "spinifex-workers"},
 			natsSub{"eks.TagResource", handleNATSRequest(d.node, d.eksService.TagResource), "spinifex-workers"},

tests/e2e/eks/cp_isolation_test.go

@@ -27,9 +27,9 @@ const nodeGroupLabelRows = `jsonpath={range .items[*]}{.metadata.name} {.metadat
 const nodeReadyRows = `jsonpath={range .items[*]}{.metadata.name}{"="}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}`
 
 // runCPIsolation proves control-plane hosts never register as Nodes, as on AWS:
-// only nodegroup workers are listed and one becomes Ready, a DaemonSet tolerating
-// every taint runs on workers only, metrics-server and its APIService become
-// available, and logs work for every kube-system pod on a Ready node.
+// only nodegroup workers are listed, one becomes Ready, deleted nodegroups leave
+// no Node behind, a DaemonSet tolerating every taint runs on workers only,
+// metrics-server and its APIService become available, and kube-system logs work.
 func runCPIsolation(t *testing.T, c *harness.AWSClient, env *harness.Env, artifacts string, fx *clusterFixture) {
 	nodeRoleArn, _ := ensureNodeRole(t, c, fx)
 	createWorkerNodegroup(t, c, fx, cpIsolationNodegroup, nodeRoleArn, nil)
@@ -37,19 +37,39 @@ func runCPIsolation(t *testing.T, c *harness.AWSClient, env *harness.Env, artifa
 	kcPath := writeKubeconfig(t, artifacts, fx.Cluster)
 	kc := harness.NewKubectl(t, kcPath, getTokenEnv(t, env))
 
+	// Earlier subtests' nodegroups are deleted by now, so a Ready Node from one
+	// of them cannot stand in for this nodegroup's worker.
 	t.Run("WorkerNodeReady", func(t *testing.T) {
 		harness.EventuallyErr(t, func() error {
-			out, err := kc.Run(30*time.Second, "get", "nodes", "-o", nodeReadyRows)
+			out, err := kc.Run(30*time.Second, "get", "nodes",
+				"-l", "eks.amazonaws.com/nodegroup="+cpIsolationNodegroup, "-o", nodeReadyRows)
 			if err != nil {
 				return fmt.Errorf("kubectl get nodes: %v\n%s", err, out)
 			}
 			if !strings.Contains(out, "=True") {
-				return fmt.Errorf("no Ready node yet:\n%s", out)
+				return fmt.Errorf("no Ready %s Node yet:\n%s", cpIsolationNodegroup, out)
 			}
 			return nil
 		}, 6*time.Minute, 5*time.Second)
 	})
 
+	// As on AWS, a terminated worker's Node is deleted: once every other
+	// nodegroup is gone, the only Nodes left are this nodegroup's.
+	t.Run("TerminatedWorkersLeaveNoNodes", func(t *testing.T) {
+		harness.EventuallyErr(t, func() error {
+			out, err := kc.Run(30*time.Second, "get", "nodes", "-o", nodeGroupLabelRows)
+			if err != nil {
+				return fmt.Errorf("kubectl get nodes: %v\n%s", err, out)
+			}
+			for _, row := range strings.Split(strings.TrimSpace(out), "\n") {
+				if f := strings.Fields(row); len(f) == 2 && f[1] != cpIsolationNodegroup {
+					return fmt.Errorf("node %s of deleted nodegroup %s is still registered:\n%s", f[0], f[1], out)
+				}
+			}
+			return nil
+		}, 4*time.Minute, 10*time.Second)
+	})
+
 	t.Run("OnlyWorkersAreNodes", func(t *testing.T) {
 		out, err := kc.Run(30*time.Second, "get", "nodes", "-o", nodeGroupLabelRows)
 		require.NoErrorf(t, err, "get nodes:\n%s", out)
@@ -71,19 +91,21 @@ func runCPIsolation(t *testing.T, c *harness.AWSClient, env *harness.Env, artifa
 			_, _ = kc.Run(60*time.Second, "delete", "daemonset", ds, "-n", "default", "--ignore-not-found")
 		})
 
+		// One "<pod>=<Ready status>" row per pod: a pod bound to a Node that never
+		// reported has no Ready condition, and its row must count as not Ready.
 		harness.EventuallyErr(t, func() error {
 			ready, err := kc.Run(30*time.Second, "get", "pods", "-l", "app="+ds,
-				"-o", `jsonpath={range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}`)
+				"-o", `jsonpath={range .items[*]}{.metadata.name}{"="}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}`)
 			if err != nil {
 				return fmt.Errorf("get daemonset pods: %v\n%s", err, ready)
 			}
-			states := strings.Fields(ready)
-			if len(states) == 0 {
+			rows := strings.Fields(ready)
+			if len(rows) == 0 {
 				return fmt.Errorf("daemonset has no pods yet")
 			}
-			for _, s := range states {
-				if s != "True" {
-					return fmt.Errorf("daemonset pods not all Ready: %v", states)
+			for _, row := range rows {
+				if _, status, _ := strings.Cut(row, "="); status != "True" {
+					return fmt.Errorf("daemonset pods not all Ready: %v", rows)
 				}
 			}
 			return nil

tests/e2e/eks/eks_test.go

@@ -536,12 +536,7 @@ func runIRSAPod(t *testing.T, c *harness.AWSClient, env *harness.Env, artifacts
 		},
 	})
 	require.NoError(t, err, "create-nodegroup")
-	t.Cleanup(func() {
-		_, _ = c.EKS.DeleteNodegroup(&eks.DeleteNodegroupInput{
-			ClusterName:   aws.String(fx.ClusterName),
-			NodegroupName: aws.String(nodegroup),
-		})
-	})
+	t.Cleanup(func() { harness.DeleteEKSNodegroupAndWait(t, c, fx.ClusterName, nodegroup) })
 	harness.EventuallyErr(t, func() error {
 		out, derr := c.EKS.DescribeNodegroup(&eks.DescribeNodegroupInput{
 			ClusterName:   aws.String(fx.ClusterName),
@@ -789,12 +784,7 @@ func runEBSCSIVolume(t *testing.T, c *harness.AWSClient, env *harness.Env, artif
 		},
 	})
 	require.NoError(t, err, "create-nodegroup")
-	t.Cleanup(func() {
-		_, _ = c.EKS.DeleteNodegroup(&eks.DeleteNodegroupInput{
-			ClusterName:   aws.String(fx.ClusterName),
-			NodegroupName: aws.String(nodegroup),
-		})
-	})
+	t.Cleanup(func() { harness.DeleteEKSNodegroupAndWait(t, c, fx.ClusterName, nodegroup) })
 	harness.EventuallyErr(t, func() error {
 		out, derr := c.EKS.DescribeNodegroup(&eks.DescribeNodegroupInput{
 			ClusterName:   aws.String(fx.ClusterName),

tests/e2e/eks/worker_imds_test.go

@@ -37,12 +37,7 @@ func createWorkerNodegroup(t *testing.T, c *harness.AWSClient, fx *clusterFixtur
 		},
 	})
 	require.NoError(t, err, "create-nodegroup")
-	t.Cleanup(func() {
-		_, _ = c.EKS.DeleteNodegroup(&eks.DeleteNodegroupInput{
-			ClusterName:   aws.String(fx.ClusterName),
-			NodegroupName: aws.String(nodegroup),
-		})
-	})
+	t.Cleanup(func() { harness.DeleteEKSNodegroupAndWait(t, c, fx.ClusterName, nodegroup) })
 	harness.EventuallyErr(t, func() error {
 		out, derr := c.EKS.DescribeNodegroup(&eks.DescribeNodegroupInput{
 			ClusterName:   aws.String(fx.ClusterName),

tests/e2e/harness/eks.go

@@ -71,3 +71,31 @@ func isClusterNotFound(err error) bool {
 	}
 	return false
 }
+
+// DeleteEKSNodegroupAndWait deletes a nodegroup and polls DescribeNodegroup
+// until it returns ResourceNotFoundException, so a later step never runs
+// beside a worker that is still going away.
+func DeleteEKSNodegroupAndWait(t *testing.T, c *AWSClient, cluster, nodegroup string) {
+	t.Helper()
+	_, err := c.EKS.DeleteNodegroup(&eks.DeleteNodegroupInput{
+		ClusterName:   aws.String(cluster),
+		NodegroupName: aws.String(nodegroup),
+	})
+	if err != nil && !isClusterNotFound(err) {
+		t.Errorf("delete-nodegroup %s: %v", nodegroup, err)
+		return
+	}
+	EventuallyErr(t, func() error {
+		_, err := c.EKS.DescribeNodegroup(&eks.DescribeNodegroupInput{
+			ClusterName:   aws.String(cluster),
+			NodegroupName: aws.String(nodegroup),
+		})
+		if err == nil {
+			return fmt.Errorf("nodegroup %s still exists", nodegroup)
+		}
+		if isClusterNotFound(err) {
+			return nil
+		}
+		return fmt.Errorf("describe-nodegroup %s: %w", nodegroup, err)
+	}, 5*time.Minute, 5*time.Second)
+}

tests/integration/cross_tenant_test.go

@@ -431,7 +431,7 @@ func eksUncased() map[string]string {
 	// These name the cluster's account in the path or body. AuthorizeInternal
 	// refuses every tenant principal outright and binds a control-plane VM to
 	// the cluster it serves, so no tenant ARN reaches the evaluator.
-	for _, action := range []string{"GetRecoveryDirective", "ListInternalAddons", "PublishInternal", "WebhookTokenReview"} {
+	for _, action := range []string{"GetRecoveryDirective", "ListInternalAddons", "ListInternalWorkers", "PublishInternal", "WebhookTokenReview"} {
 		uncased[action] = "internal control-plane route: refused to tenants, bound to the caller's own cluster"
 	}
 	for _, action := range []string{