fix(eks): apply eks.amazonaws.com/nodegroup label via k3s node-label drop-in (#534)

The nodegroup label was only written to K3S_NODE_LABEL in agent.env, but
k3s has no such env var, so non-GPU workers never received the label.
waitWorkersReady buckets Ready nodes by that label (added in #511), so
the affected nodegroups never counted toward their ACTIVE transition and
stayed CREATING until timeout.

Ride the label on a real k3s node-label config drop-in
(15-nodegroup-label.yaml), delivered via cloud-init user-data. Drop the
inert K3S_NODE_LABEL from agent.env and the initd export.

scripts/images/eks-node/k3s-agent.initd

@@ -29,7 +29,7 @@ start_pre() {
     if [ -f /etc/spinifex-eks/agent.env ]; then
         # shellcheck disable=SC1091
         . /etc/spinifex-eks/agent.env
-        export K3S_URL K3S_TOKEN K3S_NODE_NAME K3S_NODE_LABEL
+        export K3S_URL K3S_TOKEN K3S_NODE_NAME
     fi
     if [ -z "${K3S_URL:-}" ] || [ -z "${K3S_TOKEN:-}" ]; then
         eerror "K3S_URL and K3S_TOKEN must be set in /etc/spinifex-eks/agent.env"

spinifex/handlers/eks/nodegroup_userdata.go

@@ -55,19 +55,18 @@ func buildAgentUserData(in agentUserDataInput) string {
 	// K3S_URL is the published NLB endpoint (:443→:6443); the in-VPC CP ENI IP
 	// lives in the unpeered managed CP VPC and is unreachable from the worker VPC.
 	k3sURL := in.ServerURL
-	nodeLabel := "eks.amazonaws.com/nodegroup=" + in.NodegroupName
 
 	envBody := strings.Join([]string{
 		"SPINIFEX_K3S_ROLE=agent",
 	}, "\n")
 
 	// The ECR credential provider reads /etc/spinifex-eks/agent.env, so the EKS_*
-	// settings live alongside the K3s join vars here.
+	// settings live alongside the K3s join vars here. The nodegroup label rides a
+	// node-label config drop-in below, not K3S_NODE_LABEL (k3s has no such env).
 	agentBody := strings.Join([]string{
 		"K3S_URL=" + k3sURL,
 		"K3S_TOKEN=" + in.JoinToken,
 		"K3S_NODE_NAME=" + in.NodeName,
-		"K3S_NODE_LABEL=" + nodeLabel,
 		"EKS_GATEWAY_URL=" + in.GatewayURL,
 		"EKS_GATEWAY_CA=" + k3sGatewayCAPath,
 		"EKS_REGION=" + in.Region,
@@ -125,21 +124,30 @@ func buildAgentUserData(in agentUserDataInput) string {
 	)
 	credProviderConfig := strings.Join(credLines, "\n")
 
+	// The eks.amazonaws.com/nodegroup label must ride a k3s node-label config
+	// drop-in: k3s has no K3S_NODE_LABEL env, so agent.env cannot set it. The
+	// nodegroup readiness gate (waitWorkersReady) buckets Ready nodes by this
+	// label, so a worker without it never counts toward its nodegroup's ACTIVE
+	// transition. k3s concatenates node-label lists across config.yaml.d drop-ins.
+	nodegroupLabelDropin := strings.Join([]string{
+		"node-label:",
+		"  - \"eks.amazonaws.com/nodegroup=" + in.NodegroupName + "\"",
+	}, "\n")
+
 	files := []userDataFile{
 		{Path: k3sFirstBootEnvPath, Perms: "0600", Body: envBody},
 		{Path: agentEnvPath, Perms: "0600", Body: agentBody},
 		{Path: k3sGatewayCAPath, Perms: "0644", Body: strings.TrimRight(in.GatewayCACert, "\n")},
 		{Path: "/etc/rancher/k3s/registries.yaml", Perms: "0644", Body: registriesYAML},
 		{Path: "/etc/rancher/k3s/config.yaml.d/20-ecr-credential-provider.yaml", Perms: "0644", Body: credProviderDropin},
+		{Path: "/etc/rancher/k3s/config.yaml.d/15-nodegroup-label.yaml", Perms: "0644", Body: nodegroupLabelDropin},
 		{Path: "/etc/spinifex-eks/credential-provider-config.yaml", Perms: "0644", Body: credProviderConfig},
 	}
 
 	// GPU nodes advertise nvidia.com/gpu.present=true (so the device-plugin
 	// DaemonSet nodeSelector matches) and default to NoSchedule so ordinary
-	// workloads don't consume scarce GPU capacity. Both must come from k3s
-	// config keys — k3s has no K3S_NODE_LABEL env, so the label rides a
-	// node-label drop-in, not agent.env. Non-GPU nodegroups get no drop-in,
-	// keeping their user-data byte-identical.
+	// workloads don't consume scarce GPU capacity. Both ride a separate
+	// node-label/node-taint drop-in; k3s merges it over the nodegroup-label one.
 	if in.GPUEnabled {
 		gpuDropin := strings.Join([]string{
 			"node-label:",

spinifex/handlers/eks/nodegroup_userdata_test.go

@@ -131,7 +131,16 @@ func TestBuildAgentUserData_NonGPUHasNoLabelOrTaint(t *testing.T) {
 	if strings.Contains(got, "20-gpu.yaml") {
 		t.Errorf("non-GPU nodegroup user-data must not carry the GPU drop-in, got:\n%s", got)
 	}
-	if !strings.Contains(got, "K3S_NODE_LABEL=eks.amazonaws.com/nodegroup=ng-1") {
-		t.Errorf("expected plain nodegroup label, got:\n%s", got)
+	// The nodegroup label must ride a node-label config drop-in, not the no-op
+	// K3S_NODE_LABEL env — waitWorkersReady buckets Ready nodes by this label,
+	// so a worker missing it never counts toward its nodegroup's ACTIVE gate.
+	if !strings.Contains(got, "/etc/rancher/k3s/config.yaml.d/15-nodegroup-label.yaml") {
+		t.Errorf("expected nodegroup-label drop-in path, got:\n%s", got)
+	}
+	if !strings.Contains(got, `"eks.amazonaws.com/nodegroup=ng-1"`) {
+		t.Errorf("expected nodegroup label in the node-label drop-in, got:\n%s", got)
+	}
+	if strings.Contains(got, "K3S_NODE_LABEL") {
+		t.Errorf("nodegroup label must not ride the no-op K3S_NODE_LABEL env, got:\n%s", got)
 	}
 }