diff --git a/scripts/images/eks-node/k3s-agent.initd b/scripts/images/eks-node/k3s-agent.initd index 45eeeda26..5de4cc62e 100755 --- a/scripts/images/eks-node/k3s-agent.initd +++ b/scripts/images/eks-node/k3s-agent.initd @@ -29,7 +29,7 @@ start_pre() { if [ -f /etc/spinifex-eks/agent.env ]; then # shellcheck disable=SC1091 . /etc/spinifex-eks/agent.env - export K3S_URL K3S_TOKEN K3S_NODE_NAME K3S_NODE_LABEL + export K3S_URL K3S_TOKEN K3S_NODE_NAME fi if [ -z "${K3S_URL:-}" ] || [ -z "${K3S_TOKEN:-}" ]; then eerror "K3S_URL and K3S_TOKEN must be set in /etc/spinifex-eks/agent.env" diff --git a/spinifex/handlers/eks/nodegroup_userdata.go b/spinifex/handlers/eks/nodegroup_userdata.go index a68d66e22..86492642d 100644 --- a/spinifex/handlers/eks/nodegroup_userdata.go +++ b/spinifex/handlers/eks/nodegroup_userdata.go @@ -55,19 +55,18 @@ func buildAgentUserData(in agentUserDataInput) string { // K3S_URL is the published NLB endpoint (:443→:6443); the in-VPC CP ENI IP // lives in the unpeered managed CP VPC and is unreachable from the worker VPC. k3sURL := in.ServerURL - nodeLabel := "eks.amazonaws.com/nodegroup=" + in.NodegroupName envBody := strings.Join([]string{ "SPINIFEX_K3S_ROLE=agent", }, "\n") // The ECR credential provider reads /etc/spinifex-eks/agent.env, so the EKS_* - // settings live alongside the K3s join vars here. + // settings live alongside the K3s join vars here. The nodegroup label rides a + // node-label config drop-in below, not K3S_NODE_LABEL (k3s has no such env). agentBody := strings.Join([]string{ "K3S_URL=" + k3sURL, "K3S_TOKEN=" + in.JoinToken, "K3S_NODE_NAME=" + in.NodeName, - "K3S_NODE_LABEL=" + nodeLabel, "EKS_GATEWAY_URL=" + in.GatewayURL, "EKS_GATEWAY_CA=" + k3sGatewayCAPath, "EKS_REGION=" + in.Region, @@ -125,21 +124,30 @@ func buildAgentUserData(in agentUserDataInput) string { ) credProviderConfig := strings.Join(credLines, "\n") + // The eks.amazonaws.com/nodegroup label must ride a k3s node-label config + // drop-in: k3s has no K3S_NODE_LABEL env, so agent.env cannot set it. The + // nodegroup readiness gate (waitWorkersReady) buckets Ready nodes by this + // label, so a worker without it never counts toward its nodegroup's ACTIVE + // transition. k3s concatenates node-label lists across config.yaml.d drop-ins. + nodegroupLabelDropin := strings.Join([]string{ + "node-label:", + " - \"eks.amazonaws.com/nodegroup=" + in.NodegroupName + "\"", + }, "\n") + files := []userDataFile{ {Path: k3sFirstBootEnvPath, Perms: "0600", Body: envBody}, {Path: agentEnvPath, Perms: "0600", Body: agentBody}, {Path: k3sGatewayCAPath, Perms: "0644", Body: strings.TrimRight(in.GatewayCACert, "\n")}, {Path: "/etc/rancher/k3s/registries.yaml", Perms: "0644", Body: registriesYAML}, {Path: "/etc/rancher/k3s/config.yaml.d/20-ecr-credential-provider.yaml", Perms: "0644", Body: credProviderDropin}, + {Path: "/etc/rancher/k3s/config.yaml.d/15-nodegroup-label.yaml", Perms: "0644", Body: nodegroupLabelDropin}, {Path: "/etc/spinifex-eks/credential-provider-config.yaml", Perms: "0644", Body: credProviderConfig}, } // GPU nodes advertise nvidia.com/gpu.present=true (so the device-plugin // DaemonSet nodeSelector matches) and default to NoSchedule so ordinary - // workloads don't consume scarce GPU capacity. Both must come from k3s - // config keys — k3s has no K3S_NODE_LABEL env, so the label rides a - // node-label drop-in, not agent.env. Non-GPU nodegroups get no drop-in, - // keeping their user-data byte-identical. + // workloads don't consume scarce GPU capacity. Both ride a separate + // node-label/node-taint drop-in; k3s merges it over the nodegroup-label one. if in.GPUEnabled { gpuDropin := strings.Join([]string{ "node-label:", diff --git a/spinifex/handlers/eks/nodegroup_userdata_test.go b/spinifex/handlers/eks/nodegroup_userdata_test.go index dc51943f2..2ac098305 100644 --- a/spinifex/handlers/eks/nodegroup_userdata_test.go +++ b/spinifex/handlers/eks/nodegroup_userdata_test.go @@ -131,7 +131,16 @@ func TestBuildAgentUserData_NonGPUHasNoLabelOrTaint(t *testing.T) { if strings.Contains(got, "20-gpu.yaml") { t.Errorf("non-GPU nodegroup user-data must not carry the GPU drop-in, got:\n%s", got) } - if !strings.Contains(got, "K3S_NODE_LABEL=eks.amazonaws.com/nodegroup=ng-1") { - t.Errorf("expected plain nodegroup label, got:\n%s", got) + // The nodegroup label must ride a node-label config drop-in, not the no-op + // K3S_NODE_LABEL env — waitWorkersReady buckets Ready nodes by this label, + // so a worker missing it never counts toward its nodegroup's ACTIVE gate. + if !strings.Contains(got, "/etc/rancher/k3s/config.yaml.d/15-nodegroup-label.yaml") { + t.Errorf("expected nodegroup-label drop-in path, got:\n%s", got) + } + if !strings.Contains(got, `"eks.amazonaws.com/nodegroup=ng-1"`) { + t.Errorf("expected nodegroup label in the node-label drop-in, got:\n%s", got) + } + if strings.Contains(got, "K3S_NODE_LABEL") { + t.Errorf("nodegroup label must not ride the no-op K3S_NODE_LABEL env, got:\n%s", got) } }