fix(eks): make provider-id.sh the single k3s node-label authority for nodegroup + GPU labels (#539)

scripts/images/eks-node-gpu/k3s-agent.service

@@ -20,8 +20,9 @@ TasksMax=infinity
 LimitNOFILE=1048576
 LimitNPROC=infinity
 LimitCORE=infinity
-# Cloud-init drops K3S_URL/K3S_TOKEN/K3S_NODE_NAME/K3S_NODE_LABEL here at
-# RunInstances time per the Nodegroup bootstrap flow.
+# Cloud-init drops K3S_URL/K3S_TOKEN/K3S_NODE_NAME (plus SPINIFEX_NODEGROUP and,
+# for GPU workers, SPINIFEX_GPU_NODE) here at RunInstances time per the Nodegroup
+# bootstrap flow; mulga-eks-provider-id reads the SPINIFEX_* vars for node-labels.
 EnvironmentFile=-/etc/spinifex-eks/agent.env
 # Tee k3s (and its embedded kubelet's klog) stderr to the serial console as well
 # as journald: Ubuntu logs to journald only, which is off-box-invisible on a

scripts/images/eks-node-gpu/mulga-eks-provider-id.service

@@ -1,6 +1,13 @@
 [Unit]
 Description=Set K3s provider-id + topology labels from IMDS for EBS CSI
-After=network-online.target
+# After cloud-config.service so cloud-init has written /etc/spinifex-eks/agent.env
+# (write_files completes by the config stage): this script folds the nodegroup +
+# GPU node-labels (read from agent.env) into the same node-label list, so agent.env
+# must exist before it runs. NOT cloud-final.service: that unit is ordered after
+# multi-user.target and k3s-agent.service (ordered After this unit) is
+# WantedBy=multi-user.target, so chaining onto cloud-final forms an ordering cycle
+# that makes systemd skip both this unit and k3s-agent, so the worker never joins.
+After=network-online.target cloud-config.service
 Wants=network-online.target
 Before=k3s-agent.service
 

scripts/images/eks-node/addons/nvidia-device-plugin/0.17.4/10-daemonset.yaml

@@ -1,6 +1,7 @@
 # nvidia-device-plugin 0.17.4 — CDI-mode DaemonSet exposing nvidia.com/gpu
 # allocatable on GPU-tainted nodes. Gated to GPU nodes via nodeSelector on the
-# K3S_NODE_LABEL nvidia.com/gpu.present=true (nodegroup_userdata.go); inert
+# node-label nvidia.com/gpu.present=true (written by mulga-eks-provider-id from
+# the SPINIFEX_GPU_NODE agent.env signal); inert
 # elsewhere. --device-list-strategy=cdi-cri makes the plugin generate a CDI spec
 # for GPU workloads (containerd 2.x resolves the CRI-passed CDI devices; no
 # enable_cdi drop-in needed on k3s v1.32.5). The plugin's OWN container needs

scripts/images/eks-node/mulga-eks-provider-id.initd

@@ -2,14 +2,16 @@
 # mulga-eks-provider-id — boot oneshot that writes a K3s drop-in from IMDS
 # before K3s starts: the kubelet provider-id plus region/zone topology labels,
 # the node metadata the aws-ebs-csi-driver needs for NodeGetInfo + provisioning.
-# Ordered after local (which installs the IMDS on-link route) and before both
-# K3s roles. Role-agnostic: enabled on every node (server + agent).
+# Also folds the worker's nodegroup + GPU node-labels (from agent.env) into the
+# same node-label list, so it must run after cloud-init has written agent.env
+# (cloud-config/cloud-final stages, same as eks-node-role) and before the role
+# selector starts K3s. Role-agnostic: enabled on every node (server + agent).
 
 description="Set K3s provider-id + topology labels from IMDS for EBS CSI"
 
 depend() {
-    after net local
-    before k3s k3s-agent
+    after net local cloud-config cloud-final
+    before k3s k3s-agent eks-node-role
 }
 
 start() {

scripts/images/eks-node/mulga-eks-provider-id.sh

@@ -17,13 +17,22 @@ set -eu
 # family"). This drop-in supplies all four; region is derived from the AZ
 # (trailing zone letters stripped).
 #
-# K3s merges /etc/rancher/k3s/config.yaml.d/*.yaml over config.yaml; list keys
-# such as kubelet-arg are concatenated, so this adds the flag without touching
-# the cloud-init-written main config. Role-agnostic: runs on server + agent.
+# This drop-in is also the SINGLE writer of the node-label list. K3s replaces
+# (not merges) node-label across config.yaml.d files, so the last-sorted file
+# wins outright and any other node-label drop-in would silently wipe the rest.
+# The worker's nodegroup label (and, on GPU workers, the nvidia.com/gpu.present
+# label + NoSchedule taint) therefore ride here too, read from agent.env, so
+# they share the one list with the topology labels. On the K3s server there is
+# no agent.env, so only the topology labels are written — behavior unchanged.
+#
+# kubelet-arg lists ARE concatenated by k3s across drop-ins, so the provider-id
+# flag coexists with other files' kubelet-arg without touching the main config.
+# Role-agnostic: runs on server + agent.
 
 IMDS="http://169.254.169.254/latest"
 DROPIN_DIR=/etc/rancher/k3s/config.yaml.d
 DROPIN="${DROPIN_DIR}/10-provider-id.yaml"
+AGENT_ENV=/etc/spinifex-eks/agent.env
 LOGTAG="mulga-eks-provider-id"
 
 log() { echo "[${LOGTAG}] $*"; }
@@ -79,13 +88,32 @@ case "${itype}" in
     *) itype="m5.large" ;;
 esac
 
+# Fold the worker's nodegroup + GPU node-labels into the same list. agent.env is
+# written by cloud-init and absent on the server; source it only if present.
+nodegroup=""
+gpu_node=""
+if [ -f "${AGENT_ENV}" ]; then
+    # shellcheck disable=SC1090
+    . "${AGENT_ENV}"
+    nodegroup="${SPINIFEX_NODEGROUP:-}"
+    gpu_node="${SPINIFEX_GPU_NODE:-}"
+fi
+
 mkdir -p "${DROPIN_DIR}"
-cat > "${DROPIN}" <<EOF
-kubelet-arg:
-  - "provider-id=aws:///${az}/${iid}"
-node-label:
-  - "topology.kubernetes.io/region=${region}"
-  - "topology.kubernetes.io/zone=${az}"
-  - "node.kubernetes.io/instance-type=${itype}"
-EOF
-log "wrote provider-id aws:///${az}/${iid}, region=${region} zone=${az} type=${itype}"
+{
+    echo "kubelet-arg:"
+    echo "  - \"provider-id=aws:///${az}/${iid}\""
+    echo "node-label:"
+    echo "  - \"topology.kubernetes.io/region=${region}\""
+    echo "  - \"topology.kubernetes.io/zone=${az}\""
+    echo "  - \"node.kubernetes.io/instance-type=${itype}\""
+    if [ -n "${nodegroup}" ]; then
+        echo "  - \"eks.amazonaws.com/nodegroup=${nodegroup}\""
+    fi
+    if [ "${gpu_node}" = "true" ]; then
+        echo "  - \"nvidia.com/gpu.present=true\""
+        echo "node-taint:"
+        echo "  - \"nvidia.com/gpu=present:NoSchedule\""
+    fi
+} > "${DROPIN}"
+log "wrote provider-id aws:///${az}/${iid}, region=${region} zone=${az} type=${itype} nodegroup=${nodegroup:-<none>} gpu=${gpu_node:-false}"