mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-19 19:01:42 +00:00
Josh Siviour
6873875248
On a zero-worker or GPU-only cluster no node tolerates the DaemonSet, so it has zero desired pods. The apiserver then has no tunnel to kube-system pods (including metrics-server), v1beta1.metrics.k8s.io never goes Available, and namespace GC hangs on the kubernetes finalizer for every namespace, not just ones using metrics. Add the same CriticalAddonsOnly + nvidia.com/gpu tolerations already used by every other control-plane-critical addon in this repo (aws-load-balancer-controller, ebs-csi-driver, argocd) so the agent also schedules onto the control plane.
106 lines
4.2 KiB
YAML
106 lines
4.2 KiB
YAML
# Konnectivity agent DaemonSet — the worker half of the apiserver egress proxy.
|
|
# Each agent dials OUT to the konnectivity-server (fronted by the cluster NLB at
|
|
# __KONN_HOST__:8132) and, via --sync-forever, opens a reverse tunnel to EVERY
|
|
# apiserver replica (it learns the replica count from the server handshake).
|
|
# apiserver→pod/kubelet/webhook egress rides back down that tunnel, so an HA
|
|
# control plane reaches worker pod IPs with no L4-VIP fan-out gap. Staged into
|
|
# the K3s auto-deploy dir by k3s.initd, which fills the __KONN_HOST__ /
|
|
# __KONN_AGENT_IMAGE__ tokens at boot.
|
|
#
|
|
# Also tolerates the control-plane CriticalAddonsOnly=true:NoExecute taint and
|
|
# the GPU nvidia.com/gpu=present:NoSchedule taint, matching every other
|
|
# CP-critical addon in this repo (aws-load-balancer-controller, ebs-csi-driver,
|
|
# argocd): a zero-worker or GPU-only cluster has no untainted node, and other
|
|
# addons already schedule onto the CP, so it needs its own tunnel too — without
|
|
# an agent there, kubectl logs/exec against CP-hosted pods and apiserver
|
|
# aggregated-API egress (metrics-server discovery) both fail. Real EKS never
|
|
# needs this: its control plane isn't a schedulable node.
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: konnectivity-agent
|
|
namespace: kube-system
|
|
labels:
|
|
app.kubernetes.io/managed-by: spinifex-eks
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: DaemonSet
|
|
metadata:
|
|
name: konnectivity-agent
|
|
namespace: kube-system
|
|
labels:
|
|
k8s-app: konnectivity-agent
|
|
app.kubernetes.io/managed-by: spinifex-eks
|
|
spec:
|
|
selector:
|
|
matchLabels:
|
|
k8s-app: konnectivity-agent
|
|
updateStrategy:
|
|
type: RollingUpdate
|
|
template:
|
|
metadata:
|
|
labels:
|
|
k8s-app: konnectivity-agent
|
|
spec:
|
|
serviceAccountName: konnectivity-agent
|
|
priorityClassName: system-node-critical
|
|
# Without these, a zero-worker or GPU-only cluster (CP tainted
|
|
# CriticalAddonsOnly, workers tainted nvidia.com/gpu) shows DESIRED=0 for
|
|
# this DaemonSet: no tunnel exists, so kubectl logs/exec against any pod
|
|
# fails "No agent available" and apiserver aggregated-API egress
|
|
# (e.g. metrics-server discovery) fails, which stalls namespace GC.
|
|
tolerations:
|
|
- key: CriticalAddonsOnly
|
|
operator: Exists
|
|
- key: nvidia.com/gpu
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
# hostNetwork so the agent reaches the NLB over the node network and
|
|
# identifies itself to the server by the node's own IP (the address the
|
|
# apiserver dials back through for kubelet/pod egress).
|
|
hostNetwork: true
|
|
dnsPolicy: ClusterFirstWithHostNet
|
|
containers:
|
|
- name: konnectivity-agent
|
|
image: __KONN_AGENT_IMAGE__
|
|
command:
|
|
- /proxy-agent
|
|
args:
|
|
- --logtostderr=true
|
|
- --ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
|
- --proxy-server-host=__KONN_HOST__
|
|
- --proxy-server-port=8132
|
|
# proxy-agent v0.30.3 has no server-count flag — it learns the
|
|
# replica count from the server handshake and keeps re-syncing
|
|
# (sync-forever) until it holds a tunnel to every apiserver, which
|
|
# is what makes the HA control plane reachable.
|
|
- --sync-forever=true
|
|
- --service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token
|
|
- --agent-identifiers=ipv4=$(HOST_IP)
|
|
- --admin-server-port=8133
|
|
- --health-server-port=8134
|
|
env:
|
|
- name: HOST_IP
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: status.hostIP
|
|
livenessProbe:
|
|
httpGet:
|
|
scheme: HTTP
|
|
host: 127.0.0.1
|
|
port: 8134
|
|
path: /healthz
|
|
initialDelaySeconds: 15
|
|
timeoutSeconds: 15
|
|
volumeMounts:
|
|
- name: konnectivity-agent-token
|
|
mountPath: /var/run/secrets/tokens
|
|
readOnly: true
|
|
volumes:
|
|
- name: konnectivity-agent-token
|
|
projected:
|
|
sources:
|
|
- serviceAccountToken:
|
|
path: konnectivity-agent-token
|
|
audience: system:konnectivity-server
|