From 4d8a34b545d3d2b15aefc04cbd3d61038a45dfca Mon Sep 17 00:00:00 2001 From: Josh Siviour Date: Tue, 14 Jul 2026 12:43:42 +1000 Subject: [PATCH] fix(eks): apply eks.amazonaws.com/nodegroup label via k3s node-label drop-in (#534) The nodegroup label was only written to K3S_NODE_LABEL in agent.env, but k3s has no such env var, so non-GPU workers never received the label. waitWorkersReady buckets Ready nodes by that label (added in #511), so the affected nodegroups never counted toward their ACTIVE transition and stayed CREATING until timeout. Ride the label on a real k3s node-label config drop-in (15-nodegroup-label.yaml), delivered via cloud-init user-data. Drop the inert K3S_NODE_LABEL from agent.env and the initd export. --- scripts/images/eks-node/k3s-agent.initd | 2 +- spinifex/handlers/eks/nodegroup_userdata.go | 22 +++++++++++++------ .../handlers/eks/nodegroup_userdata_test.go | 13 +++++++++-- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/scripts/images/eks-node/k3s-agent.initd b/scripts/images/eks-node/k3s-agent.initd index 45eeeda26..5de4cc62e 100755 --- a/scripts/images/eks-node/k3s-agent.initd +++ b/scripts/images/eks-node/k3s-agent.initd @@ -29,7 +29,7 @@ start_pre() { if [ -f /etc/spinifex-eks/agent.env ]; then # shellcheck disable=SC1091 . /etc/spinifex-eks/agent.env - export K3S_URL K3S_TOKEN K3S_NODE_NAME K3S_NODE_LABEL + export K3S_URL K3S_TOKEN K3S_NODE_NAME fi if [ -z "${K3S_URL:-}" ] || [ -z "${K3S_TOKEN:-}" ]; then eerror "K3S_URL and K3S_TOKEN must be set in /etc/spinifex-eks/agent.env" diff --git a/spinifex/handlers/eks/nodegroup_userdata.go b/spinifex/handlers/eks/nodegroup_userdata.go index a68d66e22..86492642d 100644 --- a/spinifex/handlers/eks/nodegroup_userdata.go +++ b/spinifex/handlers/eks/nodegroup_userdata.go @@ -55,19 +55,18 @@ func buildAgentUserData(in agentUserDataInput) string { // K3S_URL is the published NLB endpoint (:443→:6443); the in-VPC CP ENI IP // lives in the unpeered managed CP VPC and is unreachable from the worker VPC. k3sURL := in.ServerURL - nodeLabel := "eks.amazonaws.com/nodegroup=" + in.NodegroupName envBody := strings.Join([]string{ "SPINIFEX_K3S_ROLE=agent", }, "\n") // The ECR credential provider reads /etc/spinifex-eks/agent.env, so the EKS_* - // settings live alongside the K3s join vars here. + // settings live alongside the K3s join vars here. The nodegroup label rides a + // node-label config drop-in below, not K3S_NODE_LABEL (k3s has no such env). agentBody := strings.Join([]string{ "K3S_URL=" + k3sURL, "K3S_TOKEN=" + in.JoinToken, "K3S_NODE_NAME=" + in.NodeName, - "K3S_NODE_LABEL=" + nodeLabel, "EKS_GATEWAY_URL=" + in.GatewayURL, "EKS_GATEWAY_CA=" + k3sGatewayCAPath, "EKS_REGION=" + in.Region, @@ -125,21 +124,30 @@ func buildAgentUserData(in agentUserDataInput) string { ) credProviderConfig := strings.Join(credLines, "\n") + // The eks.amazonaws.com/nodegroup label must ride a k3s node-label config + // drop-in: k3s has no K3S_NODE_LABEL env, so agent.env cannot set it. The + // nodegroup readiness gate (waitWorkersReady) buckets Ready nodes by this + // label, so a worker without it never counts toward its nodegroup's ACTIVE + // transition. k3s concatenates node-label lists across config.yaml.d drop-ins. + nodegroupLabelDropin := strings.Join([]string{ + "node-label:", + " - \"eks.amazonaws.com/nodegroup=" + in.NodegroupName + "\"", + }, "\n") + files := []userDataFile{ {Path: k3sFirstBootEnvPath, Perms: "0600", Body: envBody}, {Path: agentEnvPath, Perms: "0600", Body: agentBody}, {Path: k3sGatewayCAPath, Perms: "0644", Body: strings.TrimRight(in.GatewayCACert, "\n")}, {Path: "/etc/rancher/k3s/registries.yaml", Perms: "0644", Body: registriesYAML}, {Path: "/etc/rancher/k3s/config.yaml.d/20-ecr-credential-provider.yaml", Perms: "0644", Body: credProviderDropin}, + {Path: "/etc/rancher/k3s/config.yaml.d/15-nodegroup-label.yaml", Perms: "0644", Body: nodegroupLabelDropin}, {Path: "/etc/spinifex-eks/credential-provider-config.yaml", Perms: "0644", Body: credProviderConfig}, } // GPU nodes advertise nvidia.com/gpu.present=true (so the device-plugin // DaemonSet nodeSelector matches) and default to NoSchedule so ordinary - // workloads don't consume scarce GPU capacity. Both must come from k3s - // config keys — k3s has no K3S_NODE_LABEL env, so the label rides a - // node-label drop-in, not agent.env. Non-GPU nodegroups get no drop-in, - // keeping their user-data byte-identical. + // workloads don't consume scarce GPU capacity. Both ride a separate + // node-label/node-taint drop-in; k3s merges it over the nodegroup-label one. if in.GPUEnabled { gpuDropin := strings.Join([]string{ "node-label:", diff --git a/spinifex/handlers/eks/nodegroup_userdata_test.go b/spinifex/handlers/eks/nodegroup_userdata_test.go index dc51943f2..2ac098305 100644 --- a/spinifex/handlers/eks/nodegroup_userdata_test.go +++ b/spinifex/handlers/eks/nodegroup_userdata_test.go @@ -131,7 +131,16 @@ func TestBuildAgentUserData_NonGPUHasNoLabelOrTaint(t *testing.T) { if strings.Contains(got, "20-gpu.yaml") { t.Errorf("non-GPU nodegroup user-data must not carry the GPU drop-in, got:\n%s", got) } - if !strings.Contains(got, "K3S_NODE_LABEL=eks.amazonaws.com/nodegroup=ng-1") { - t.Errorf("expected plain nodegroup label, got:\n%s", got) + // The nodegroup label must ride a node-label config drop-in, not the no-op + // K3S_NODE_LABEL env — waitWorkersReady buckets Ready nodes by this label, + // so a worker missing it never counts toward its nodegroup's ACTIVE gate. + if !strings.Contains(got, "/etc/rancher/k3s/config.yaml.d/15-nodegroup-label.yaml") { + t.Errorf("expected nodegroup-label drop-in path, got:\n%s", got) + } + if !strings.Contains(got, `"eks.amazonaws.com/nodegroup=ng-1"`) { + t.Errorf("expected nodegroup label in the node-label drop-in, got:\n%s", got) + } + if strings.Contains(got, "K3S_NODE_LABEL") { + t.Errorf("nodegroup label must not ride the no-op K3S_NODE_LABEL env, got:\n%s", got) } }