diff --git a/scripts/images/eks-node/addons/aws-ebs-csi-driver/1.40.1/20-node.yaml b/scripts/images/eks-node/addons/aws-ebs-csi-driver/1.40.1/20-node.yaml index d2b936fb7..b9fd2394d 100644 --- a/scripts/images/eks-node/addons/aws-ebs-csi-driver/1.40.1/20-node.yaml +++ b/scripts/images/eks-node/addons/aws-ebs-csi-driver/1.40.1/20-node.yaml @@ -46,6 +46,13 @@ spec: - a1.4xlarge nodeSelector: kubernetes.io/os: linux + # The node plugin must run on every worker, including GPU-only workers + # tainted nvidia.com/gpu=present:NoSchedule, or GPU pods there cannot + # mount EBS volumes (no node plugin = no CSI NodeStage/NodePublish). + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule serviceAccountName: ebs-csi-node-sa terminationGracePeriodSeconds: 30 priorityClassName: system-node-critical diff --git a/scripts/images/eks-node/coredns/coredns-mulga.yaml b/scripts/images/eks-node/coredns/coredns-mulga.yaml index ae38658cc..5663ebaf9 100644 --- a/scripts/images/eks-node/coredns/coredns-mulga.yaml +++ b/scripts/images/eks-node/coredns/coredns-mulga.yaml @@ -135,6 +135,13 @@ spec: operator: DoesNotExist nodeSelector: kubernetes.io/os: linux + # With internalTrafficPolicy=Local each worker resolves through its OWN + # CoreDNS pod, so a GPU-only worker (tainted nvidia.com/gpu=present: + # NoSchedule) with no local pod would have no DNS at all for its pods. + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule containers: - name: coredns image: rancher/mirrored-coredns-coredns:1.12.1 diff --git a/scripts/images/eks-node/konnectivity/konnectivity-agent.yaml b/scripts/images/eks-node/konnectivity/konnectivity-agent.yaml index 7f99da12e..f3207d15c 100644 --- a/scripts/images/eks-node/konnectivity/konnectivity-agent.yaml +++ b/scripts/images/eks-node/konnectivity/konnectivity-agent.yaml @@ -9,7 +9,9 @@ # # Runs on workers only: it carries no toleration for the control-plane # CriticalAddonsOnly=true:NoExecute taint, so it is never scheduled onto the CP -# nodes (which run the server, not an agent). +# nodes (which run the server, not an agent). It DOES tolerate the GPU +# nvidia.com/gpu=present:NoSchedule taint — a GPU-only worker still needs +# kubectl logs/exec against pods there, which routes through this agent. apiVersion: v1 kind: ServiceAccount metadata: @@ -39,6 +41,13 @@ spec: spec: serviceAccountName: konnectivity-agent priorityClassName: system-node-critical + # Without this, a GPU-only worker (tainted nvidia.com/gpu=present:NoSchedule) + # shows DESIRED=0 for this DaemonSet and kubectl logs/exec against pods + # scheduled there fails "No agent available". + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule # hostNetwork so the agent reaches the NLB over the node network and # identifies itself to the server by the node's own IP (the address the # apiserver dials back through for kubelet/pod egress). diff --git a/scripts/images/eks-node/mulga-eks-state-report.sh b/scripts/images/eks-node/mulga-eks-state-report.sh index 0c45b6a14..4abc31dec 100755 --- a/scripts/images/eks-node/mulga-eks-state-report.sh +++ b/scripts/images/eks-node/mulga-eks-state-report.sh @@ -70,11 +70,21 @@ publish_report() { # Probe /healthz through the node's admin kubeconfig (apiserver runs # anonymous-auth=false, CIS): an unauthenticated curl would return 401. reason= + nodegroup_ready='{}' if kubectl get --raw='/healthz' 2>/dev/null | grep -q '^ok$'; then health=ok # Count Ready nodes only — k3s retains terminated workers as NotReady # until manually pruned, so a raw node count never drops on scale-down. node_count=$(kubectl get nodes --no-headers 2>/dev/null | awk '$2=="Ready"' | wc -l | tr -d ' ') + # Per-nodegroup Ready breakdown: -L appends the node's + # eks.amazonaws.com/nodegroup label as the last column ("" when + # absent, e.g. the control-plane node itself). Grouping by that label — + # rather than only totalling — lets each nodegroup's own Ready-gate be + # scoped to its own workers instead of the cluster-wide count, so one + # nodegroup's Ready nodes can never mask another's stuck launch. + nodegroup_ready=$(kubectl get nodes --no-headers -L eks.amazonaws.com/nodegroup 2>/dev/null \ + | awk '$2=="Ready" && $NF!=""{c[$NF]++} END{sep=""; printf "{"; for (k in c) {printf "%s\"%s\":%d", sep, k, c[k]; sep=","}; printf "}"}') + [ -n "${nodegroup_ready}" ] || nodegroup_ready='{}' else health=fail node_count=0 @@ -91,11 +101,11 @@ publish_report() { } > "${CONSOLE:-/dev/console}" 2>&1 || true fi if [ -n "${reason}" ]; then - payload=$(printf '{"healthz":"%s","node_count":%s,"reason":"%s","ts":%s}' \ - "${health}" "${node_count}" "${reason}" "$(date +%s)") + payload=$(printf '{"healthz":"%s","node_count":%s,"nodegroup_ready":%s,"reason":"%s","ts":%s}' \ + "${health}" "${node_count}" "${nodegroup_ready}" "${reason}" "$(date +%s)") else - payload=$(printf '{"healthz":"%s","node_count":%s,"ts":%s}' \ - "${health}" "${node_count}" "$(date +%s)") + payload=$(printf '{"healthz":"%s","node_count":%s,"nodegroup_ready":%s,"ts":%s}' \ + "${health}" "${node_count}" "${nodegroup_ready}" "$(date +%s)") fi printf '%s' "${payload}" | eks-gateway-publish -channel state 2>&1 \ | logger -t mulga-eks-state-report diff --git a/scripts/images/eks-node/mulga-eks-state-report_test.sh b/scripts/images/eks-node/mulga-eks-state-report_test.sh index 0c942d3f6..15a9411d3 100644 --- a/scripts/images/eks-node/mulga-eks-state-report_test.sh +++ b/scripts/images/eks-node/mulga-eks-state-report_test.sh @@ -24,6 +24,7 @@ case "$*" in *"--raw=/healthz/etcd"*) printf '%s\n' "${ETCD_BODY:-ok}" ;; *"--raw=/readyz?verbose"*) printf '%s\n' "${READYZ_BODY:-}" ;; *"--raw=/healthz"*) printf '%s\n' "${HEALTHZ_BODY:-ok}" ;; + *"get nodes"*" -L "*) printf '%s\n' "${NODES_LABELED_BODY:-}" ;; *"get nodes"*) printf '%s\n' "${NODES_BODY:-}" ;; *) : ;; esac @@ -75,15 +76,23 @@ run_agent() { } # --- Case 1: healthy apiserver, 3 Ready nodes -> no reason, quiet console --- +# NODES_LABELED_BODY carries the -L eks.amazonaws.com/nodegroup breakdown: a +# CP node ("", excluded), 2 Ready ng-a workers, 1 Ready ng-b worker, and +# a NotReady ng-b worker that must not count — proving the per-nodegroup tally +# scopes to THAT nodegroup's own Ready nodes, not the cluster-wide total. HEALTHZ_BODY=ok NODES_BODY=$(printf 'ip-a Ready s\nip-b Ready s\nip-c Ready s\n') +NODES_LABELED_BODY=$(printf 'ip-cp Ready s \nip-a Ready s ng-a\nip-b Ready s ng-a\nip-c Ready s ng-b\nip-d NotReady s ng-b\n') ETCD_DB_DIR="${WORK}" -export HEALTHZ_BODY NODES_BODY +export HEALTHZ_BODY NODES_BODY NODES_LABELED_BODY run_agent P=$(cat "${WORK}/payload.json") case "${P}" in *'"healthz":"ok"'*) pass "healthy: healthz ok" ;; *) fail "healthy: healthz not ok: ${P}" ;; esac case "${P}" in *'"node_count":3'*) pass "healthy: node_count 3" ;; *) fail "healthy: node_count wrong: ${P}" ;; esac case "${P}" in *'"reason"'*) fail "healthy: reason must be absent: ${P}" ;; *) pass "healthy: no reason field" ;; esac +case "${P}" in *'"nodegroup_ready":{'*) pass "healthy: nodegroup_ready object present" ;; *) fail "healthy: nodegroup_ready missing: ${P}" ;; esac +case "${P}" in *'"ng-a":2'*) pass "healthy: ng-a Ready count scoped to its own 2 workers" ;; *) fail "healthy: ng-a count wrong: ${P}" ;; esac +case "${P}" in *'"ng-b":1'*) pass "healthy: ng-b Ready count excludes its NotReady worker" ;; *) fail "healthy: ng-b count wrong: ${P}" ;; esac [ -s "${WORK}/console.out" ] && fail "healthy: console must be quiet" || pass "healthy: console quiet" # --- Case 2: unhealthy, etcd subcheck failing, etcd unreachable, disk ok --- @@ -95,6 +104,7 @@ export HEALTHZ_BODY READYZ_BODY ETCD_BODY DF_AVAIL_KB run_agent P=$(cat "${WORK}/payload.json") case "${P}" in *'"healthz":"fail"'*) pass "unhealthy: healthz fail" ;; *) fail "unhealthy: healthz wrong: ${P}" ;; esac +case "${P}" in *'"nodegroup_ready":{}'*) pass "unhealthy: nodegroup_ready empty (no kubectl node query on a failing apiserver)" ;; *) fail "unhealthy: nodegroup_ready wrong: ${P}" ;; esac case "${P}" in *'"reason":"readyz:[etcd poststarthook/start-service-ip-repair-controllers]; etcd:unreachable; disk:ok"'*) pass "unhealthy: reason names failing subchecks + etcd + disk" ;; *) fail "unhealthy: reason wrong: ${P}" ;; esac case "${P}" in *'"reason":"'*'"'*'"'*) : ;; esac C=$(cat "${WORK}/console.out") diff --git a/spinifex/handlers/eks/claim_race_test.go b/spinifex/handlers/eks/claim_race_test.go index dea046192..790ed14a4 100644 --- a/spinifex/handlers/eks/claim_race_test.go +++ b/spinifex/handlers/eks/claim_race_test.go @@ -109,7 +109,7 @@ func TestCreateNodegroup_ConcurrentSameNameSingleOwner(t *testing.T) { } wg.Wait() // The single winning owner launches one worker; let its Ready-gate resolve. - markWorkersReady(t, f, "c1", 1) + markWorkersReady(t, f, "c1", "ng1", 1) f.svc.WaitLaunches() ok, inUse := classifyCreateErrs(t, errs) diff --git a/spinifex/handlers/eks/cluster_reconciler.go b/spinifex/handlers/eks/cluster_reconciler.go index cd3f786bc..2423ffe0d 100644 --- a/spinifex/handlers/eks/cluster_reconciler.go +++ b/spinifex/handlers/eks/cluster_reconciler.go @@ -499,7 +499,7 @@ func (r *ClusterReconciler) reconcileOnce(ctx context.Context) error { "cluster", r.clusterName, "reason", reason) return r.failIfCreateTimedOut(meta, "bootstrap not ready: "+reason) } - issue, nodeCount := r.observe(ctx) + issue, nodeCount, nodegroupReady := r.observe(ctx) if issue != "" { slog.Info("ClusterReconciler: health not ready", "cluster", r.clusterName, "issue", issue) @@ -508,7 +508,7 @@ func (r *ClusterReconciler) reconcileOnce(ctx context.Context) error { if err := SetClusterStatus(r.acctKV, r.clusterName, ClusterStatusActive); err != nil { return err } - if err := SetClusterHealthState(r.acctKV, r.clusterName, "", nodeCount); err != nil { + if err := SetClusterHealthState(r.acctKV, r.clusterName, "", nodeCount, nodegroupReady); err != nil { if errors.Is(err, ErrClusterNotFound) { return ErrClusterNotFound } @@ -517,12 +517,12 @@ func (r *ClusterReconciler) reconcileOnce(ctx context.Context) error { slog.Info("ClusterReconciler: cluster transitioned to ACTIVE", "cluster", r.clusterName, "nodes", nodeCount) case ClusterStatusActive: - issue, nodeCount := r.observe(ctx) + issue, nodeCount, nodegroupReady := r.observe(ctx) if issue != "" { slog.Warn("ClusterReconciler: health failing for ACTIVE cluster", "cluster", r.clusterName, "issue", issue) } - if err := SetClusterHealthState(r.acctKV, r.clusterName, issue, nodeCount); err != nil { + if err := SetClusterHealthState(r.acctKV, r.clusterName, issue, nodeCount, nodegroupReady); err != nil { if errors.Is(err, ErrClusterNotFound) { return ErrClusterNotFound } @@ -958,30 +958,32 @@ func liveHosts(live []ControlPlaneNode) []string { return hosts } -// observe returns the health issue ("" = healthy) and node count from the CP's -// latest NATS self-report, or from the HTTP /healthz probe when no NATS source is set. -func (r *ClusterReconciler) observe(ctx context.Context) (issue string, nodeCount int) { +// observe returns the health issue ("" = healthy), node count, and per-nodegroup +// Ready breakdown from the CP's latest NATS self-report, or from the HTTP +// /healthz probe when no NATS source is set (nodegroupReady is nil in that case +// — the probe carries no per-node label data). +func (r *ClusterReconciler) observe(ctx context.Context) (issue string, nodeCount int, nodegroupReady map[string]int) { if r.stateSub != nil && r.stateSubject != "" { report := r.latest.Load() if report == nil { - return "no control-plane state report received yet", 0 + return "no control-plane state report received yet", 0, nil } age := time.Since(time.Unix(report.TS, 0)) if age > r.stateStaleAfter { - return fmt.Sprintf("control-plane state report stale (%s old)", age.Round(time.Second)), report.NodeCount + return fmt.Sprintf("control-plane state report stale (%s old)", age.Round(time.Second)), report.NodeCount, report.NodegroupReady } if !report.Healthy() { if report.Reason != "" { - return fmt.Sprintf("apiserver healthz=%q: %s", report.Healthz, report.Reason), report.NodeCount + return fmt.Sprintf("apiserver healthz=%q: %s", report.Healthz, report.Reason), report.NodeCount, report.NodegroupReady } - return fmt.Sprintf("apiserver healthz=%q", report.Healthz), report.NodeCount + return fmt.Sprintf("apiserver healthz=%q", report.Healthz), report.NodeCount, report.NodegroupReady } - return "", report.NodeCount + return "", report.NodeCount, report.NodegroupReady } if err := r.probeHealthz(ctx); err != nil { - return err.Error(), 0 + return err.Error(), 0, nil } - return "", 0 + return "", 0, nil } // failIfCreateTimedOut marks the cluster FAILED once createTimeout has elapsed diff --git a/spinifex/handlers/eks/cluster_reconciler_state_test.go b/spinifex/handlers/eks/cluster_reconciler_state_test.go index d959f0427..fe85480d9 100644 --- a/spinifex/handlers/eks/cluster_reconciler_state_test.go +++ b/spinifex/handlers/eks/cluster_reconciler_state_test.go @@ -79,7 +79,7 @@ func TestClusterReconciler_CreatingStaysOnUnhealthyReport(t *testing.T) { func TestClusterReconciler_ActiveRecordsNodeCountAndClearsIssue(t *testing.T) { r, _, acctKV := newStateReconcilerHarness(t) require.NoError(t, SetClusterStatus(acctKV, "alpha", ClusterStatusActive)) - require.NoError(t, SetClusterHealthState(acctKV, "alpha", "stale", 0)) + require.NoError(t, SetClusterHealthState(acctKV, "alpha", "stale", 0, nil)) r.latest.Store(freshReport("ok", 4)) require.NoError(t, r.reconcileOnce(context.Background())) diff --git a/spinifex/handlers/eks/cluster_state.go b/spinifex/handlers/eks/cluster_state.go index cca936047..f59afdfdc 100644 --- a/spinifex/handlers/eks/cluster_state.go +++ b/spinifex/handlers/eks/cluster_state.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "maps" "strings" "time" @@ -99,6 +100,12 @@ type ClusterMeta struct { LastHealthProbe time.Time `json:"lastHealthProbe"` // NodeCount is the node total from the CP's last NATS state report. NodeCount int `json:"nodeCount,omitempty"` + // NodegroupNodeCounts is the per-nodegroup Ready count from the CP's last NATS + // state report, keyed by the eks.amazonaws.com/nodegroup node label value. + // waitWorkersReady gates a nodegroup's ACTIVE transition on ITS OWN entry here, + // never the cluster-wide NodeCount total — otherwise one nodegroup's Ready + // workers could mask another nodegroup whose own workers never registered. + NodegroupNodeCounts map[string]int `json:"nodegroupNodeCounts,omitempty"` // Tags are the create-time resource tags, stored verbatim so DescribeCluster // echoes them back. Without the round-trip a stock terraform-aws provider // reconciling default_tags sees perpetual drift and issues TagResource on @@ -283,18 +290,28 @@ func SetClusterHealth(kv nats.KeyValue, name, issue string) error { }) } -// SetClusterHealthState records health + node count. LastHealthProbe is stamped -// on any change; no KV write when both are unchanged. -func SetClusterHealthState(kv nats.KeyValue, name, issue string, nodeCount int) error { +// SetClusterHealthState records health + node count (cluster-wide and, when the +// CP state report carries it, per-nodegroup). LastHealthProbe is stamped on any +// change; no KV write when nothing changed. nodegroupReady may be nil (older +// AMI / no report yet), in which case the persisted per-nodegroup counts are +// left untouched. +func SetClusterHealthState(kv nats.KeyValue, name, issue string, nodeCount int, nodegroupReady map[string]int) error { if name == "" { return errors.New("eks: SetClusterHealthState empty name") } return casUpdateMeta(kv, name, func(m *ClusterMeta) bool { - if m.HealthIssue == issue && m.NodeCount == nodeCount { + changed := m.HealthIssue != issue || m.NodeCount != nodeCount + if nodegroupReady != nil && !maps.Equal(m.NodegroupNodeCounts, nodegroupReady) { + changed = true + } + if !changed { return false } m.HealthIssue = issue m.NodeCount = nodeCount + if nodegroupReady != nil { + m.NodegroupNodeCounts = nodegroupReady + } m.LastHealthProbe = time.Now().UTC() return true }) diff --git a/spinifex/handlers/eks/create_cluster_test.go b/spinifex/handlers/eks/create_cluster_test.go index 8cd358942..dc8bc1614 100644 --- a/spinifex/handlers/eks/create_cluster_test.go +++ b/spinifex/handlers/eks/create_cluster_test.go @@ -189,6 +189,12 @@ func TestCreateCluster_FailedClusterVisibleInDescribeAndList(t *testing.T) { desc, err := f.svc.DescribeCluster(context.Background(), &eks.DescribeClusterInput{Name: aws.String("alpha")}, testAccountID) require.NoError(t, err) assert.Equal(t, eks.ClusterStatusFailed, aws.StringValue(desc.Cluster.Status)) + // The failure cause must be visible, not just the bare FAILED status — otherwise + // describe-cluster gives no way to diagnose why the launch failed. + require.NotNil(t, desc.Cluster.Health) + require.Len(t, desc.Cluster.Health.Issues, 1) + assert.Equal(t, eks.ClusterIssueCodeInternalFailure, aws.StringValue(desc.Cluster.Health.Issues[0].Code)) + assert.Contains(t, aws.StringValue(desc.Cluster.Health.Issues[0].Message), "bootstrap failed: boom") list, err := f.svc.ListClusters(context.Background(), &eks.ListClustersInput{}, testAccountID) require.NoError(t, err) diff --git a/spinifex/handlers/eks/delete_cluster_test.go b/spinifex/handlers/eks/delete_cluster_test.go index b65f7022a..4f175e322 100644 --- a/spinifex/handlers/eks/delete_cluster_test.go +++ b/spinifex/handlers/eks/delete_cluster_test.go @@ -105,6 +105,10 @@ func newEKSServiceFixture(t *testing.T) *eksServiceFixture { // production 10m timeout; tests drive meta.NodeCount to simulate the reconciler. svc.nodegroupReadyTimeout = 1 * time.Second svc.nodegroupReadyPoll = 2 * time.Millisecond + // Tight worker-launch-retry knobs so a launch-failure retry test resolves + // quickly instead of against the production 5m/10s budget. + svc.workerLaunchRetryTimeout = 100 * time.Millisecond + svc.workerLaunchRetryBackoff = 20 * time.Millisecond t.Cleanup(svc.Shutdown) return &eksServiceFixture{svc: svc, kv: kv, nlb: nlb, inst: inst, vpc: vpc, ami: ami, eip: eip, sg: sg, worker: worker, vpcMgr: vpcMgr, igw: igw, ngw: ngw, rt: rt} diff --git a/spinifex/handlers/eks/nlb.go b/spinifex/handlers/eks/nlb.go index 57f2fd31f..c014ded2a 100644 --- a/spinifex/handlers/eks/nlb.go +++ b/spinifex/handlers/eks/nlb.go @@ -2,6 +2,8 @@ package handlers_eks import ( "context" + "crypto/sha256" + "encoding/hex" "errors" "fmt" "log/slog" @@ -73,21 +75,45 @@ type ClusterNLB struct { FrontendIP string } -// ClusterNLBName returns the deterministic NLB name for a cluster. +// ClusterNLBName returns the deterministic NLB name for a cluster, truncated +// (with a stable hash suffix) so it never exceeds the ELBv2 32-char limit. func ClusterNLBName(clusterName string) string { - return "eks-" + clusterName + return safeELBv2Name("eks-", clusterName, "") } // ClusterTargetGroupName returns the deterministic target-group name for a -// cluster's control-plane TG. +// cluster's control-plane TG, truncated (with a stable hash suffix) so it +// never exceeds the ELBv2 32-char limit. func ClusterTargetGroupName(clusterName string) string { - return "eks-" + clusterName + "-cp" + return safeELBv2Name("eks-", clusterName, "-cp") } // ClusterKonnTargetGroupName returns the deterministic target-group name for a -// cluster's konnectivity TG (:8132 → 3 CP servers). +// cluster's konnectivity TG (:8132 → 3 CP servers), truncated (with a stable +// hash suffix) so it never exceeds the ELBv2 32-char limit. func ClusterKonnTargetGroupName(clusterName string) string { - return "eks-" + clusterName + "-konn" + return safeELBv2Name("eks-", clusterName, "-konn") +} + +// safeELBv2Name derives prefix+clusterName+suffix, and — only when that would +// exceed the ELBv2 32-char name limit — deterministically truncates +// clusterName and appends a short stable hash of the FULL cluster name so long +// names stay unique and the derived name is identical across calls (create, +// lookup, delete all agree without persisting anything extra). +func safeELBv2Name(prefix, clusterName, suffix string) string { + full := prefix + clusterName + suffix + if len(full) <= maxELBv2NameLen { + return full + } + sum := sha256.Sum256([]byte(clusterName)) + hash := hex.EncodeToString(sum[:])[:8] + // "-" + hash separates the truncated name from the disambiguating hash. + budget := max(maxELBv2NameLen-len(prefix)-len(suffix)-len(hash)-1, 0) + truncated := clusterName + if len(truncated) > budget { + truncated = truncated[:budget] + } + return prefix + truncated + "-" + hash + suffix } // EnsureClusterNLB provisions (or returns) the cluster's NLB + target group + diff --git a/spinifex/handlers/eks/nlb_test.go b/spinifex/handlers/eks/nlb_test.go index e531c2458..3aaa3b216 100644 --- a/spinifex/handlers/eks/nlb_test.go +++ b/spinifex/handlers/eks/nlb_test.go @@ -304,14 +304,47 @@ func TestEnsureClusterNLB_EmptyInputsRejected(t *testing.T) { assert.Empty(t, nlbp.createLBCalls) } -func TestEnsureClusterNLB_NameTooLongRejected(t *testing.T) { +func TestEnsureClusterNLB_LongNameTruncatedDeterministically(t *testing.T) { nlbp := newFakeNLBProvisioner() - longName := strings.Repeat("x", maxELBv2NameLen) // "eks-" + 32x = 36 chars - _, err := EnsureClusterNLB(context.Background(), nlbp, "111122223333", longName, []string{"subnet-aaa"}, false, nil, nil) - require.Error(t, err) - assert.Contains(t, err.Error(), "exceeds") - assert.Empty(t, nlbp.createLBCalls) + longName := strings.Repeat("x", maxELBv2NameLen) // "eks-" + 32x = 36 chars, over the ELBv2 limit + out, err := EnsureClusterNLB(context.Background(), nlbp, "111122223333", longName, []string{"subnet-aaa"}, false, nil, nil) + require.NoError(t, err) + require.NotNil(t, out) + + require.Len(t, nlbp.createLBCalls, 1) + lbName := aws.StringValue(nlbp.createLBCalls[0].Name) + assert.LessOrEqual(t, len(lbName), maxELBv2NameLen) + + require.Len(t, nlbp.createTGCalls, 2) + tgName := aws.StringValue(nlbp.createTGCalls[0].Name) + konnTGName := aws.StringValue(nlbp.createTGCalls[1].Name) + assert.LessOrEqual(t, len(tgName), maxELBv2NameLen) + assert.LessOrEqual(t, len(konnTGName), maxELBv2NameLen) + // The three derived names must stay distinct (different suffixes) despite truncation. + assert.NotEqual(t, lbName, tgName) + assert.NotEqual(t, tgName, konnTGName) + + // Deterministic + stable: recomputing from the same cluster name yields the + // same names, so create/lookup/delete all agree without persisting anything extra. + assert.Equal(t, lbName, ClusterNLBName(longName)) + assert.Equal(t, tgName, ClusterTargetGroupName(longName)) + assert.Equal(t, konnTGName, ClusterKonnTargetGroupName(longName)) +} + +func TestSafeELBv2Name(t *testing.T) { + short := ClusterNLBName("alpha") + assert.Equal(t, "eks-alpha", short) + + long := strings.Repeat("y", 64) + name1 := ClusterNLBName(long) + name2 := ClusterNLBName(long) + assert.LessOrEqual(t, len(name1), maxELBv2NameLen) + assert.Equal(t, name1, name2, "must be stable across calls for the same cluster name") + + // Two different long names must not collide onto the same truncated name. + other := strings.Repeat("z", 64) + assert.NotEqual(t, name1, ClusterNLBName(other)) } func TestEnsureClusterNLB_FreshCreatesAllThree(t *testing.T) { diff --git a/spinifex/handlers/eks/nodegroup.go b/spinifex/handlers/eks/nodegroup.go index 05411898e..f3734b5c8 100644 --- a/spinifex/handlers/eks/nodegroup.go +++ b/spinifex/handlers/eks/nodegroup.go @@ -41,6 +41,17 @@ const ( defaultNodegroupReadyPoll = 15 * time.Second ) +// defaultWorkerLaunchRetryTimeout / defaultWorkerLaunchRetryBackoff bound how +// long launchNodegroupInfra retries a worker-launch shortfall (a RunInstances +// call that failed — e.g. a transient QMP timeout or host capacity pressure) +// before giving up. Distinct from nodegroupReadyTimeout: this bounds getting the +// desired instance count RUNNING, not the subsequent wait for k3s-agent to +// register Ready. +const ( + defaultWorkerLaunchRetryTimeout = 5 * time.Minute + defaultWorkerLaunchRetryBackoff = 10 * time.Second +) + // ngCASMaxRetries bounds the compare-and-swap retry loop that serializes // concurrent nodegroup-record mutations (overlapping UpdateNodegroupConfig). const ngCASMaxRetries = 16 @@ -375,7 +386,7 @@ func (s *EKSServiceImpl) launchNodegroupInfra(ctx context.Context, lc nodegroupL s.stageGPUDeviceAddon(ctx, accountID, cluster) } - if _, err := s.launchWorkers(ctx, acctKV, accountID, rec, meta, ngSGID, amiID, token, lc.desired); err != nil { + if _, err := s.launchWorkersUntilDesired(ctx, acctKV, accountID, rec, meta, ngSGID, amiID, token, lc.desired); err != nil { // launchWorkers persisted each worker it launched (incrementally), so the // reclaim path can already tear them down; just record the terminal failure. rec.Status = eks.NodegroupStatusCreateFailed @@ -388,10 +399,13 @@ func (s *EKSServiceImpl) launchNodegroupInfra(ctx context.Context, lc nodegroupL } // Gate ACTIVE on the workers registering Ready (observed via the CP state - // report's Ready-node count), not merely on RunInstances success — a worker - // that boots but never joins must surface CREATE_FAILED, not falsely ACTIVE. - // Baseline is the create-time node count; the workers add lc.desired Ready nodes. - if err := s.waitWorkersReady(acctKV, cluster, meta.NodeCount, lc.desired); err != nil { + // report's per-nodegroup Ready count, eks.amazonaws.com/nodegroup=ng), not + // merely on RunInstances success — a worker that boots but never joins must + // surface CREATE_FAILED, not falsely ACTIVE. Scoped to THIS nodegroup so + // another nodegroup's Ready workers can never mask this one's shortfall. + // Baseline is the create-time count for this nodegroup; the workers add + // lc.desired Ready nodes. + if err := s.waitWorkersReady(acctKV, cluster, ng, meta.NodegroupNodeCounts[ng], lc.desired); err != nil { rec.Status = eks.NodegroupStatusCreateFailed rec.StatusReason = "workers did not become Ready: " + err.Error() rec.ModifiedAt = time.Now().UTC() @@ -408,13 +422,16 @@ func (s *EKSServiceImpl) launchNodegroupInfra(ctx context.Context, lc nodegroupL } } -// waitWorkersReady blocks until the cluster's Ready-node count rises by want over -// baseline — every nodegroup worker registered Ready — or the timeout / bgCtx -// fires. Ready count is meta.NodeCount, which the ClusterReconciler refreshes -// from the CP's NATS state report (Ready nodes only). Mirrors the CP reconciler's -// healthy-observe gating: a nodegroup is ACTIVE only once its workers are observed -// Ready, not merely launched. -func (s *EKSServiceImpl) waitWorkersReady(acctKV nats.KeyValue, cluster string, baseline, want int) error { +// waitWorkersReady blocks until nodegroup ng's own Ready-node count rises by +// want over baseline — every worker THIS nodegroup launched registered Ready — +// or the timeout / bgCtx fires. Ready count is +// meta.NodegroupNodeCounts[ng], scoped to nodes carrying the +// eks.amazonaws.com/nodegroup=ng label, which the ClusterReconciler refreshes +// from the CP's NATS state report. Scoping per-nodegroup (rather than the +// cluster-wide NodeCount total) is deliberate: in a multi-nodegroup cluster, +// another nodegroup's Ready workers must never let this one falsely reach +// ACTIVE while its own workers never joined. +func (s *EKSServiceImpl) waitWorkersReady(acctKV nats.KeyValue, cluster, ng string, baseline, want int) error { target := baseline + want deadline := time.Now().Add(s.nodegroupReadyTimeout) for { @@ -422,12 +439,13 @@ func (s *EKSServiceImpl) waitWorkersReady(acctKV nats.KeyValue, cluster string, if err != nil { return err } - if meta.NodeCount >= target { + ready := meta.NodegroupNodeCounts[ng] + if ready >= target { return nil } if time.Now().After(deadline) { - return fmt.Errorf("timed out after %s: cluster reports %d Ready nodes, want >= %d (baseline %d + %d workers)", - s.nodegroupReadyTimeout, meta.NodeCount, target, baseline, want) + return fmt.Errorf("timed out after %s: nodegroup %s reports %d Ready nodes, want >= %d (baseline %d + %d workers)", + s.nodegroupReadyTimeout, ng, ready, target, baseline, want) } select { case <-s.bgCtx.Done(): @@ -466,6 +484,39 @@ func (s *EKSServiceImpl) launchWorkers(ctx context.Context, acctKV nats.KeyValue return ids, nil } +// launchWorkersUntilDesired launches count workers, retrying the shortfall when +// launchWorkers fails partway (a transient RunInstances failure — QMP timeout, +// host capacity pressure) instead of abandoning the nodegroup at its first +// failed launch. Each retry relaunches only what's still missing — +// already-launched workers are never reissued — until the desired count is +// reached, workerLaunchRetryTimeout elapses, or bgCtx is cancelled. Every retry +// is logged so a stuck launch is diagnosable without live-tailing the daemon. +func (s *EKSServiceImpl) launchWorkersUntilDesired(ctx context.Context, acctKV nats.KeyValue, accountID string, rec *NodegroupRecord, meta *ClusterMeta, ngSGID, amiID, token string, count int) ([]string, error) { + deadline := time.Now().Add(s.workerLaunchRetryTimeout) + all := make([]string, 0, count) + remaining := count + var lastErr error + for attempt := 1; ; attempt++ { + ids, err := s.launchWorkers(ctx, acctKV, accountID, rec, meta, ngSGID, amiID, token, remaining) + all = append(all, ids...) + remaining -= len(ids) + if remaining <= 0 { + return all, nil + } + lastErr = err + if time.Now().After(deadline) { + return all, fmt.Errorf("worker launch shortfall after %d attempt(s), %d still missing: %w", attempt, remaining, lastErr) + } + slog.WarnContext(ctx, "launchNodegroupInfra: worker launch failed short of desired capacity, relaunching shortfall", + "cluster", rec.ClusterName, "nodegroup", rec.Name, "attempt", attempt, "remaining", remaining, "err", err) + select { + case <-s.bgCtx.Done(): + return all, fmt.Errorf("shutdown during worker relaunch retry (%d still missing): %w", remaining, s.bgCtx.Err()) + case <-time.After(s.workerLaunchRetryBackoff): + } + } +} + // launchOneWorker provisions a single tagged worker VM and returns its instance // ID. It performs no durable record write; the caller owns persistence (plain // for the single-owner create path, CAS-append for concurrent scale-up). diff --git a/spinifex/handlers/eks/nodegroup_test.go b/spinifex/handlers/eks/nodegroup_test.go index 344317f3e..0ce983cba 100644 --- a/spinifex/handlers/eks/nodegroup_test.go +++ b/spinifex/handlers/eks/nodegroup_test.go @@ -77,6 +77,10 @@ type fakeWorkerLauncher struct { nextID int runErr error termErr error + // failNextN makes the next N RunWorkerInstanceOnNode calls fail with runErr + // before subsequent calls succeed — simulates a transient launch failure + // (QMP timeout, host pressure) that clears on retry. + failNextN int } var _ WorkerLauncher = (*fakeWorkerLauncher)(nil) @@ -94,6 +98,17 @@ func (f *fakeWorkerLauncher) RunWorkerInstanceOnNode(_ context.Context, nodeID s defer f.mu.Unlock() f.runCalls = append(f.runCalls, input) f.runNodes = append(f.runNodes, nodeID) + if f.failNextN > 0 { + f.failNextN-- + err := f.runErr + if f.failNextN == 0 { + // Transient window exhausted: clear runErr so later calls succeed + // unless the test also wants a permanent failure (it wouldn't set + // failNextN in that case). + f.runErr = nil + } + return nil, err + } if f.runErr != nil { return nil, f.runErr } @@ -151,13 +166,14 @@ func seedActiveClusterWithToken(t *testing.T, f *eksServiceFixture, cluster stri require.NoError(t, err) } -// markWorkersReady simulates the cluster reconciler observing n Ready nodes from -// the CP state report, so launchNodegroupInfra's Ready-gate (waitWorkersReady) -// can resolve. Seed clusters start at NodeCount 0, so n is the create baseline -// (0) plus the worker count the test expects Ready. -func markWorkersReady(t *testing.T, f *eksServiceFixture, cluster string, n int) { +// markWorkersReady simulates the cluster reconciler observing n Ready nodes, +// scoped to nodegroup ng's eks.amazonaws.com/nodegroup label, from the CP state +// report, so launchNodegroupInfra's Ready-gate (waitWorkersReady) can resolve. +// Seed clusters start at NodeCount/NodegroupNodeCounts 0, so n is the create +// baseline (0) plus the worker count the test expects Ready. +func markWorkersReady(t *testing.T, f *eksServiceFixture, cluster, ng string, n int) { t.Helper() - require.NoError(t, SetClusterHealthState(f.kv, cluster, "", n)) + require.NoError(t, SetClusterHealthState(f.kv, cluster, "", n, map[string]int{ng: n})) } func createNGInput(cluster, ng string, desired int64) *eks.CreateNodegroupInput { @@ -324,7 +340,7 @@ func TestCreateNodegroup_HappyPath(t *testing.T) { assert.Contains(t, aws.StringValue(out.Nodegroup.NodegroupArn), ":nodegroup/c1/ng1/") // Both workers register Ready → the Ready-gate lets the nodegroup go ACTIVE. - markWorkersReady(t, f, "c1", 2) + markWorkersReady(t, f, "c1", "ng1", 2) f.svc.WaitLaunches() // The async launch transitions the record to ACTIVE once workers run. @@ -375,7 +391,7 @@ func TestCreateNodegroup_GPUInstanceTypeSetsGPUFields(t *testing.T) { assert.True(t, rec.GPUEnabled) assert.Equal(t, "nvidia", rec.GPUVendor) - markWorkersReady(t, f, "c1", 1) + markWorkersReady(t, f, "c1", "ng-gpu", 1) f.svc.WaitLaunches() } @@ -391,7 +407,7 @@ func TestCreateNodegroup_GPUNodegroupStagesDevicePluginAddon(t *testing.T) { _, err := f.svc.CreateNodegroup(context.Background(), in, testAccountID) require.NoError(t, err) - markWorkersReady(t, f, "c1", 1) + markWorkersReady(t, f, "c1", "ng-gpu", 1) f.svc.WaitLaunches() rec, err := GetAddonRecord(f.kv, "c1", nvidiaDevicePluginAddonName) @@ -408,7 +424,7 @@ func TestCreateNodegroup_NonGPUNodegroupDoesNotStageDevicePluginAddon(t *testing _, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 1), testAccountID) require.NoError(t, err) - markWorkersReady(t, f, "c1", 1) + markWorkersReady(t, f, "c1", "ng1", 1) f.svc.WaitLaunches() _, err = GetAddonRecord(f.kv, "c1", nvidiaDevicePluginAddonName) @@ -429,7 +445,7 @@ func TestCreateNodegroup_NonGPUInstanceTypeClearsGPUFields(t *testing.T) { assert.False(t, rec.GPUEnabled) assert.Empty(t, rec.GPUVendor) - markWorkersReady(t, f, "c1", 1) + markWorkersReady(t, f, "c1", "ng1", 1) f.svc.WaitLaunches() } @@ -454,7 +470,7 @@ func TestCreateNodegroup_GPUWorkerLaunchUsesGPUAMI(t *testing.T) { _, err := f.svc.CreateNodegroup(context.Background(), in, testAccountID) require.NoError(t, err) - markWorkersReady(t, f, "c1", 1) + markWorkersReady(t, f, "c1", "ng-gpu", 1) f.svc.WaitLaunches() rec, err := GetNodegroupRecord(f.kv, "c1", "ng-gpu") @@ -511,6 +527,84 @@ func TestCreateNodegroup_WorkersNeverReady_CreateFailed(t *testing.T) { assert.Len(t, rec.InstanceIDs, 2, "launched workers retained for the reclaim path") } +// The Ready-gate must be scoped to each nodegroup's OWN workers +// (eks.amazonaws.com/nodegroup=), never a cluster-wide tally: one +// nodegroup's Ready nodes must never mask another nodegroup whose own workers +// never registered. Regresses a bug where waitWorkersReady compared against the +// global cluster Ready-node count, so ng-a's Ready worker could have falsely +// satisfied ng-b's gate too. +func TestCreateNodegroup_ReadyGateScopedPerNodegroupNotGlobal(t *testing.T) { + f := newEKSServiceFixture(t) + seedActiveClusterWithToken(t, f, "c1") + + _, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng-a", 1), testAccountID) + require.NoError(t, err) + _, err = f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng-b", 1), testAccountID) + require.NoError(t, err) + + // Only ng-a's worker registers Ready; ng-b's worker never does. If the + // Ready-gate were still scoped globally, the cluster-wide count (1) would + // wrongly satisfy ng-b's gate too. + markWorkersReady(t, f, "c1", "ng-a", 1) + f.svc.WaitLaunches() + + ngA, err := GetNodegroupRecord(f.kv, "c1", "ng-a") + require.NoError(t, err) + assert.Equal(t, eks.NodegroupStatusActive, ngA.Status, "ng-a's own Ready worker must activate it") + + ngB, err := GetNodegroupRecord(f.kv, "c1", "ng-b") + require.NoError(t, err) + assert.Equal(t, eks.NodegroupStatusCreateFailed, ngB.Status, + "ng-b must not be marked ACTIVE by ng-a's Ready nodes — its own workers never registered") + assert.Contains(t, ngB.StatusReason, "did not become Ready") +} + +// A worker launch that fails transiently (QMP timeout / host pressure) must be +// retried, not abandon the nodegroup at its first failed RunInstances call — +// the relaunch converges to desired capacity within the retry budget. +func TestCreateNodegroup_RelaunchesOnWorkerLaunchFailure(t *testing.T) { + f := newEKSServiceFixture(t) + seedActiveClusterWithToken(t, f, "c1") + + // The first RunInstances call fails; the retry must relaunch the shortfall + // and succeed on its next attempt instead of leaving the nodegroup stuck. + f.worker.runErr = errors.New("qmp: timeout waiting for launch") + f.worker.failNextN = 1 + + _, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 1), testAccountID) + require.NoError(t, err) + + markWorkersReady(t, f, "c1", "ng1", 1) + f.svc.WaitLaunches() + + rec, err := GetNodegroupRecord(f.kv, "c1", "ng1") + require.NoError(t, err) + assert.Equal(t, eks.NodegroupStatusActive, rec.Status, + "the retried launch must converge to ACTIVE, not CREATE_FAILED") + assert.Len(t, rec.InstanceIDs, 1) + assert.GreaterOrEqual(t, len(f.worker.runCalls), 2, "must have retried after the first failed launch") +} + +// A worker launch that keeps failing must still terminate (CREATE_FAILED) once +// the retry budget is exhausted — retrying must never hang the nodegroup +// forever — while having genuinely retried at least once before giving up. +func TestCreateNodegroup_WorkerLaunchFailureExhaustsRetryBudget(t *testing.T) { + f := newEKSServiceFixture(t) + seedActiveClusterWithToken(t, f, "c1") + + f.worker.runErr = errors.New("qmp: timeout waiting for launch") + + _, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 1), testAccountID) + require.NoError(t, err) + f.svc.WaitLaunches() + + rec, err := GetNodegroupRecord(f.kv, "c1", "ng1") + require.NoError(t, err) + assert.Equal(t, eks.NodegroupStatusCreateFailed, rec.Status) + assert.Contains(t, rec.StatusReason, "worker launch shortfall after") + assert.Greater(t, len(f.worker.runCalls), 1, "must have retried at least once before giving up") +} + func TestCreateNodegroup_DiskSizePropagatesToBlockDeviceMapping(t *testing.T) { f := newEKSServiceFixture(t) seedActiveClusterWithToken(t, f, "c1") @@ -520,7 +614,7 @@ func TestCreateNodegroup_DiskSizePropagatesToBlockDeviceMapping(t *testing.T) { _, err := f.svc.CreateNodegroup(context.Background(), in, testAccountID) require.NoError(t, err) - markWorkersReady(t, f, "c1", 1) + markWorkersReady(t, f, "c1", "ng1", 1) f.svc.WaitLaunches() require.Len(t, f.worker.runCalls, 1) @@ -537,7 +631,7 @@ func TestCreateNodegroup_NoDiskSizeOmitsBlockDeviceMapping(t *testing.T) { _, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 1), testAccountID) require.NoError(t, err) - markWorkersReady(t, f, "c1", 1) + markWorkersReady(t, f, "c1", "ng1", 1) f.svc.WaitLaunches() require.Len(t, f.worker.runCalls, 1) @@ -569,7 +663,7 @@ func TestDescribeAndListNodegroups(t *testing.T) { seedActiveClusterWithToken(t, f, "c1") _, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 1), testAccountID) require.NoError(t, err) - markWorkersReady(t, f, "c1", 1) + markWorkersReady(t, f, "c1", "ng1", 1) f.svc.WaitLaunches() desc, err := f.svc.DescribeNodegroup(context.Background(), &eks.DescribeNodegroupInput{ @@ -594,7 +688,7 @@ func TestUpdateNodegroupConfig_ScaleUpAndDown(t *testing.T) { seedActiveClusterWithToken(t, f, "c1") _, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 1), testAccountID) require.NoError(t, err) - markWorkersReady(t, f, "c1", 1) + markWorkersReady(t, f, "c1", "ng1", 1) f.svc.WaitLaunches() require.Len(t, f.worker.runCalls, 1) @@ -634,7 +728,7 @@ func TestUpdateNodegroupConfig_ConcurrentScaleUpConverges(t *testing.T) { seedActiveClusterWithToken(t, f, "c1") _, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 1), testAccountID) require.NoError(t, err) - markWorkersReady(t, f, "c1", 1) + markWorkersReady(t, f, "c1", "ng1", 1) f.svc.WaitLaunches() require.Len(t, f.worker.runCalls, 1) @@ -680,7 +774,7 @@ func TestUpdateNodegroupConfig_CapacityErrorSurfacesCode(t *testing.T) { seedActiveClusterWithToken(t, f, "c1") _, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 1), testAccountID) require.NoError(t, err) - markWorkersReady(t, f, "c1", 1) + markWorkersReady(t, f, "c1", "ng1", 1) f.svc.WaitLaunches() f.worker.runErr = runErr @@ -711,7 +805,7 @@ func TestDeleteNodegroup_TerminatesAndIsIdempotent(t *testing.T) { seedActiveClusterWithToken(t, f, "c1") _, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 2), testAccountID) require.NoError(t, err) - markWorkersReady(t, f, "c1", 2) + markWorkersReady(t, f, "c1", "ng1", 2) f.svc.WaitLaunches() _, err = f.svc.DeleteNodegroup(context.Background(), &eks.DeleteNodegroupInput{ diff --git a/spinifex/handlers/eks/service_impl.go b/spinifex/handlers/eks/service_impl.go index d5665706b..42e8cf260 100644 --- a/spinifex/handlers/eks/service_impl.go +++ b/spinifex/handlers/eks/service_impl.go @@ -242,6 +242,13 @@ type EKSServiceImpl struct { // CREATE_FAILED. Tests inject small values. nodegroupReadyTimeout time.Duration nodegroupReadyPoll time.Duration + + // workerLaunchRetryTimeout / workerLaunchRetryBackoff bound how long + // launchNodegroupInfra retries a shortfall (a worker RunInstances call that + // failed, e.g. a transient QMP timeout or host capacity pressure) before + // giving up and marking the nodegroup CREATE_FAILED. Tests inject small values. + workerLaunchRetryTimeout time.Duration + workerLaunchRetryBackoff time.Duration } var _ EKSService = (*EKSServiceImpl)(nil) @@ -266,13 +273,15 @@ func NewEKSServiceImpl(deps EKSServiceDeps) (*EKSServiceImpl, error) { } ctx, cancel := context.WithCancel(context.Background()) return &EKSServiceImpl{ - deps: deps, - leaderKV: leaderKV, - registry: NewReconcilerRegistry(), - bgCtx: ctx, - bgCancel: cancel, - nodegroupReadyTimeout: defaultNodegroupReadyTimeout, - nodegroupReadyPoll: defaultNodegroupReadyPoll, + deps: deps, + leaderKV: leaderKV, + registry: NewReconcilerRegistry(), + bgCtx: ctx, + bgCancel: cancel, + nodegroupReadyTimeout: defaultNodegroupReadyTimeout, + nodegroupReadyPoll: defaultNodegroupReadyPoll, + workerLaunchRetryTimeout: defaultWorkerLaunchRetryTimeout, + workerLaunchRetryBackoff: defaultWorkerLaunchRetryBackoff, }, nil } @@ -2034,13 +2043,24 @@ func clusterMetaToAWS(meta *ClusterMeta) *eks.Cluster { PublicAccessCidrs: aws.StringSlice(meta.ResourcesVpcConfig.PublicAccessCidrs), } } + var issues []*eks.ClusterIssue if meta.HealthIssue != "" { - out.Health = &eks.ClusterHealth{ - Issues: []*eks.ClusterIssue{{ - Code: aws.String(eks.ClusterIssueCodeClusterUnreachable), - Message: aws.String(meta.HealthIssue), - }}, - } + issues = append(issues, &eks.ClusterIssue{ + Code: aws.String(eks.ClusterIssueCodeClusterUnreachable), + Message: aws.String(meta.HealthIssue), + }) + } + // StatusReason is the launch-failure cause MarkClusterFailed recorded; surface + // it here too, since the real EKS DescribeCluster response has no dedicated + // "why did CREATE_FAILED happen" field and Health.Issues is the analogue. + if meta.Status == ClusterStatusFailed && meta.StatusReason != "" { + issues = append(issues, &eks.ClusterIssue{ + Code: aws.String(eks.ClusterIssueCodeInternalFailure), + Message: aws.String(meta.StatusReason), + }) + } + if len(issues) > 0 { + out.Health = &eks.ClusterHealth{Issues: issues} } if len(meta.Tags) > 0 { out.Tags = aws.StringMap(meta.Tags) diff --git a/spinifex/handlers/eks/state_report.go b/spinifex/handlers/eks/state_report.go index 9ce4d1340..7abc820e6 100644 --- a/spinifex/handlers/eks/state_report.go +++ b/spinifex/handlers/eks/state_report.go @@ -27,6 +27,11 @@ type ServerStateReport struct { // unhealthy (failing /readyz subchecks, etcd reachability, etcd-disk free). // Empty from a healthy CP or an older AMI that predates the field. Reason string `json:"reason,omitempty"` + // NodegroupReady is the Ready-node count grouped by the + // eks.amazonaws.com/nodegroup label value, so each nodegroup's readiness can + // be gated on its OWN workers rather than the cluster-wide total. Nil from an + // older AMI that predates per-nodegroup reporting. + NodegroupReady map[string]int `json:"nodegroup_ready,omitempty"` } // Healthy reports whether the apiserver was serving at publish time.