mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-21 11:46:37 +00:00
Josh Siviour
f7bf921ce7
* fix(eks): keep derived ELBv2 names under 32 chars and surface CREATE_FAILED reason Long cluster names made the derived "eks-<name>[-cp|-konn]" NLB/target-group names exceed the ELBv2 32-char limit, failing cluster creation asynchronously with no visible cause. safeELBv2Name now deterministically truncates the name and appends a stable hash of the full cluster name when needed, so the same name is derived on every call (create, lookup, delete). DescribeCluster also now surfaces ClusterMeta.StatusReason as a Health.Issues entry when a cluster is FAILED, instead of leaving callers to guess why. * fix(eks): plumb per-nodegroup Ready-node counts through state reports The on-VM state-report script now emits nodegroup_ready, a per-nodegroup Ready count keyed by the eks.amazonaws.com/nodegroup label value, alongside the existing cluster-wide node_count. ServerStateReport, the ClusterReconciler's observe step, and SetClusterHealthState now carry this map through to ClusterMeta.NodegroupNodeCounts, so per-nodegroup consumers (wired in the next commit) can gate on a nodegroup's own workers instead of the cluster total. * fix(eks): scope nodegroup Ready-gate per nodegroup and retry failed worker launches waitWorkersReady compared the cluster-wide Ready-node count against a nodegroup's baseline+desired, so in a multi-nodegroup cluster one nodegroup's Ready workers could mask another whose own workers never registered. It now reads meta.NodegroupNodeCounts[ng], scoped to nodes carrying the eks.amazonaws.com/nodegroup=<name> label. Separately, launchNodegroupInfra never retried a failed worker launch (a transient QMP timeout or host capacity pressure), leaving the nodegroup stuck CREATING with zero workers. The new launchWorkersUntilDesired wrapper relaunches only the shortfall on failure, logging each retry, until the desired count is reached or workerLaunchRetryTimeout elapses. Both fixes are covered by regression tests: a two-nodegroup scenario where only one has Ready workers, a transient-failure-then-success relaunch, and an exhausted-retry-budget CREATE_FAILED path. Not covered here: a worker that dies after the nodegroup is already ACTIVE — that needs an ongoing reconciler, which this change does not add. * fix(eks): tolerate GPU NoSchedule taint in konnectivity-agent DaemonSet A GPU-only worker is tainted nvidia.com/gpu=present:NoSchedule, so the konnectivity-agent DaemonSet (no tolerations) showed DESIRED=0 there and kubectl logs/exec against pods on that node failed "No agent available". Adds an Exists toleration for nvidia.com/gpu:NoSchedule. Live-cluster validation (DESIRED count, kubectl exec) is deferred — no cluster available to test against right now. * fix: tolerate nvidia.com/gpu taint on ebs-csi-node and coredns DaemonSets so GPU workers get storage and DNS
99 lines
3.7 KiB
YAML
99 lines
3.7 KiB
YAML
# Konnectivity agent DaemonSet — the worker half of the apiserver egress proxy.
|
|
# Each agent dials OUT to the konnectivity-server (fronted by the cluster NLB at
|
|
# __KONN_HOST__:8132) and, via --sync-forever, opens a reverse tunnel to EVERY
|
|
# apiserver replica (it learns the replica count from the server handshake).
|
|
# apiserver→pod/kubelet/webhook egress rides back down that tunnel, so an HA
|
|
# control plane reaches worker pod IPs with no L4-VIP fan-out gap. Staged into
|
|
# the K3s auto-deploy dir by k3s.initd, which fills the __KONN_HOST__ /
|
|
# __KONN_AGENT_IMAGE__ tokens at boot.
|
|
#
|
|
# Runs on workers only: it carries no toleration for the control-plane
|
|
# CriticalAddonsOnly=true:NoExecute taint, so it is never scheduled onto the CP
|
|
# nodes (which run the server, not an agent). It DOES tolerate the GPU
|
|
# nvidia.com/gpu=present:NoSchedule taint — a GPU-only worker still needs
|
|
# kubectl logs/exec against pods there, which routes through this agent.
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: konnectivity-agent
|
|
namespace: kube-system
|
|
labels:
|
|
app.kubernetes.io/managed-by: spinifex-eks
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: DaemonSet
|
|
metadata:
|
|
name: konnectivity-agent
|
|
namespace: kube-system
|
|
labels:
|
|
k8s-app: konnectivity-agent
|
|
app.kubernetes.io/managed-by: spinifex-eks
|
|
spec:
|
|
selector:
|
|
matchLabels:
|
|
k8s-app: konnectivity-agent
|
|
updateStrategy:
|
|
type: RollingUpdate
|
|
template:
|
|
metadata:
|
|
labels:
|
|
k8s-app: konnectivity-agent
|
|
spec:
|
|
serviceAccountName: konnectivity-agent
|
|
priorityClassName: system-node-critical
|
|
# Without this, a GPU-only worker (tainted nvidia.com/gpu=present:NoSchedule)
|
|
# shows DESIRED=0 for this DaemonSet and kubectl logs/exec against pods
|
|
# scheduled there fails "No agent available".
|
|
tolerations:
|
|
- key: nvidia.com/gpu
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
# hostNetwork so the agent reaches the NLB over the node network and
|
|
# identifies itself to the server by the node's own IP (the address the
|
|
# apiserver dials back through for kubelet/pod egress).
|
|
hostNetwork: true
|
|
dnsPolicy: ClusterFirstWithHostNet
|
|
containers:
|
|
- name: konnectivity-agent
|
|
image: __KONN_AGENT_IMAGE__
|
|
command:
|
|
- /proxy-agent
|
|
args:
|
|
- --logtostderr=true
|
|
- --ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
|
- --proxy-server-host=__KONN_HOST__
|
|
- --proxy-server-port=8132
|
|
# proxy-agent v0.30.3 has no server-count flag — it learns the
|
|
# replica count from the server handshake and keeps re-syncing
|
|
# (sync-forever) until it holds a tunnel to every apiserver, which
|
|
# is what makes the HA control plane reachable.
|
|
- --sync-forever=true
|
|
- --service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token
|
|
- --agent-identifiers=ipv4=$(HOST_IP)
|
|
- --admin-server-port=8133
|
|
- --health-server-port=8134
|
|
env:
|
|
- name: HOST_IP
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: status.hostIP
|
|
livenessProbe:
|
|
httpGet:
|
|
scheme: HTTP
|
|
host: 127.0.0.1
|
|
port: 8134
|
|
path: /healthz
|
|
initialDelaySeconds: 15
|
|
timeoutSeconds: 15
|
|
volumeMounts:
|
|
- name: konnectivity-agent-token
|
|
mountPath: /var/run/secrets/tokens
|
|
readOnly: true
|
|
volumes:
|
|
- name: konnectivity-agent-token
|
|
projected:
|
|
sources:
|
|
- serviceAccountToken:
|
|
path: konnectivity-agent-token
|
|
audience: system:konnectivity-server
|