mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-20 03:11:46 +00:00
Josh Siviour
a3fb4cccec
* fix: bound EKS DELETING re-drive with backoff and a terminal give-up reapCluster re-drove purgeClusterInfra on every GC tick with no attempt tracking, so a permanently-failing teardown (e.g. an unretriable DependencyViolation) was retried forever. Track attempt count and last attempt time on ClusterMeta, back off exponentially between re-drives, and give up after maxDeleteReapAttempts with a single error-level log. Status stays DELETING on give-up so the cluster's infra remains tracked and billable pending operator intervention, preserving the ADR-0006 §6 billable-before-sweep invariant. * fix: create the EKS e2e node role independently in each subtest runEBSCSIVolume string-built the worker node role ARN without ever creating it, only passing because runIRSAPod's t.Cleanup discarded DeleteRole's error while the role was still attached to its instance profile, leaving it behind for EBSCSIVolume to reuse by naming convention. Extract ensureNodeRole and call it independently from both subtests, detaching the role from its instance profile before deleting it and asserting that error instead of discarding it. * fix: reclaim CSI-provisioned EBS volumes on EKS cluster delete purgeClusterInfra tore down the control plane, nodegroups, and networking but never looked at EBS volumes, so any volume the in-cluster CSI driver had provisioned for a PVC was left behind untracked and unbilled for. Add a report-only reclaim step that scans for volumes tagged with the cluster's CSI owner tag and logs them for operator follow-up. Volumes are never auto-deleted: a PV's reclaim policy (Delete vs Retain) is a Kubernetes-side property the CSI driver does not tag onto the EC2 volume, so there is no safe way to tell from tags alone whether deleting a given volume is intended. This mirrors the volume-leak reaper's existing mark-and-alarm policy for the same reason. The cluster-owner tag requires the EBS CSI controller's --k8s-tag-cluster-id flag, added here to the controller manifest; without it a CSI volume carries no cluster-scoped tag and reclaim would have to guess across every cluster sharing the account. * fix: persist the last purge error on a wedged DELETING cluster DeleteReapExhausted told an operator a cluster's teardown had given up, but not why: nothing recorded the failing purge's error. Persist it on every failed re-drive, not only once the reaper exhausts its attempts, so it is visible on a still-retrying cluster too. Status stays DELETING throughout, preserving the billing invariant that untorn-down infra remains tracked. * fix: make the EKS e2e suite confirm its CSI volume is actually deleted The EBSCSIVolume subtest deleted its PVC on cleanup but never confirmed the underlying volume was reclaimed, so a CSI driver that silently failed to delete it would leak a volume with nobody noticing. The suite provisions and owns this volume, so unlike production teardown it can safely wait on its StorageClass's Delete reclaim policy: capture the CSI volumeHandle while the PVC is bound, then have cleanup poll EC2 for that exact volume to disappear before the addon/nodegroup that run the CSI controller are torn down.
290 lines
8.8 KiB
YAML
290 lines
8.8 KiB
YAML
# aws-ebs-csi-driver 1.40.1 — controller Deployment + gateway-CA Secret.
|
|
# AWS credentials filled by mulga-eks-addon-sync via the IRSA env/volume/mount
|
|
# markers below: web-identity (projected SA token aud sts.amazonaws.com + role
|
|
# ARN + STS endpoint + CA) when a service account role ARN is staged, otherwise
|
|
# node instance-profile creds over IMDS. AWS_EC2_ENDPOINT / AWS_ENDPOINT_URL_EC2
|
|
# point EBS calls at the gateway in both modes. Static aws-secret key auth removed.
|
|
---
|
|
apiVersion: v1
|
|
kind: Secret
|
|
metadata:
|
|
name: spinifex-gateway-ca
|
|
namespace: kube-system
|
|
labels:
|
|
app.kubernetes.io/name: aws-ebs-csi-driver
|
|
type: Opaque
|
|
data:
|
|
ca.pem: '{{GATEWAY_CA_PEM_B64}}'
|
|
---
|
|
kind: Deployment
|
|
apiVersion: apps/v1
|
|
metadata:
|
|
name: ebs-csi-controller
|
|
labels:
|
|
app.kubernetes.io/name: aws-ebs-csi-driver
|
|
namespace: kube-system
|
|
spec:
|
|
replicas: 2
|
|
revisionHistoryLimit: 10
|
|
strategy:
|
|
rollingUpdate:
|
|
maxUnavailable: 1
|
|
type: RollingUpdate
|
|
selector:
|
|
matchLabels:
|
|
app: ebs-csi-controller
|
|
app.kubernetes.io/name: aws-ebs-csi-driver
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: ebs-csi-controller
|
|
app.kubernetes.io/name: aws-ebs-csi-driver
|
|
spec:
|
|
nodeSelector:
|
|
kubernetes.io/os: linux
|
|
serviceAccountName: ebs-csi-controller-sa
|
|
priorityClassName: system-cluster-critical
|
|
# A cluster with a GPU-only nodegroup has no untainted node: the CP carries
|
|
# CriticalAddonsOnly (EKS parity keeps user workloads off it) and every
|
|
# worker carries nvidia.com/gpu (nodegroup_userdata.go). Without these
|
|
# tolerations the controller has nowhere to schedule and stays Pending
|
|
# forever. Real EKS never needs this: its control plane isn't a schedulable
|
|
# node, so the stock upstream manifest carries no such toleration.
|
|
tolerations:
|
|
- key: CriticalAddonsOnly
|
|
operator: Exists
|
|
- key: nvidia.com/gpu
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
affinity:
|
|
nodeAffinity:
|
|
preferredDuringSchedulingIgnoredDuringExecution:
|
|
- preference:
|
|
matchExpressions:
|
|
- key: eks.amazonaws.com/compute-type
|
|
operator: NotIn
|
|
values:
|
|
- fargate
|
|
- auto
|
|
- hybrid
|
|
weight: 1
|
|
podAntiAffinity:
|
|
preferredDuringSchedulingIgnoredDuringExecution:
|
|
- podAffinityTerm:
|
|
labelSelector:
|
|
matchExpressions:
|
|
- key: app
|
|
operator: In
|
|
values:
|
|
- ebs-csi-controller
|
|
topologyKey: kubernetes.io/hostname
|
|
weight: 100
|
|
securityContext:
|
|
fsGroup: 1000
|
|
runAsGroup: 1000
|
|
runAsNonRoot: true
|
|
runAsUser: 1000
|
|
containers:
|
|
- name: ebs-plugin
|
|
image: public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver:v1.40.1
|
|
imagePullPolicy: IfNotPresent
|
|
args:
|
|
- controller
|
|
- --endpoint=$(CSI_ENDPOINT)
|
|
- --batching=true
|
|
- --logging-format=text
|
|
- --user-agent-extra=kustomize
|
|
- --k8s-tag-cluster-id={{CLUSTER_NAME}}
|
|
- --v=2
|
|
env:
|
|
- name: CSI_ENDPOINT
|
|
value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
|
|
- name: CSI_NODE_NAME
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: spec.nodeName
|
|
- name: AWS_EC2_ENDPOINT
|
|
value: '{{GATEWAY_ENDPOINT}}'
|
|
- name: AWS_ENDPOINT_URL_EC2
|
|
value: '{{GATEWAY_ENDPOINT}}'
|
|
{{IRSA_ENV}}
|
|
volumeMounts:
|
|
- name: socket-dir
|
|
mountPath: /var/lib/csi/sockets/pluginproxy/
|
|
{{IRSA_VOLUME_MOUNT}}
|
|
ports:
|
|
- name: healthz
|
|
containerPort: 9808
|
|
protocol: TCP
|
|
livenessProbe:
|
|
httpGet:
|
|
path: /healthz
|
|
port: healthz
|
|
initialDelaySeconds: 10
|
|
timeoutSeconds: 3
|
|
periodSeconds: 10
|
|
failureThreshold: 5
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /healthz
|
|
port: healthz
|
|
initialDelaySeconds: 10
|
|
timeoutSeconds: 3
|
|
periodSeconds: 10
|
|
failureThreshold: 5
|
|
resources:
|
|
limits:
|
|
memory: 256Mi
|
|
requests:
|
|
cpu: 10m
|
|
memory: 40Mi
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
readOnlyRootFilesystem: true
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
- name: csi-provisioner
|
|
image: public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner:v5.2.0-eks-1-32-7
|
|
imagePullPolicy: IfNotPresent
|
|
args:
|
|
- --timeout=60s
|
|
- --csi-address=$(ADDRESS)
|
|
- --v=2
|
|
- --feature-gates=Topology=true
|
|
- --extra-create-metadata
|
|
- --leader-election=true
|
|
- --default-fstype=ext4
|
|
- --kube-api-qps=20
|
|
- --kube-api-burst=100
|
|
- --worker-threads=100
|
|
- --retry-interval-max=30m
|
|
- --feature-gates=VolumeAttributesClass=true
|
|
env:
|
|
- name: ADDRESS
|
|
value: /var/lib/csi/sockets/pluginproxy/csi.sock
|
|
volumeMounts:
|
|
- name: socket-dir
|
|
mountPath: /var/lib/csi/sockets/pluginproxy/
|
|
resources:
|
|
limits:
|
|
memory: 256Mi
|
|
requests:
|
|
cpu: 10m
|
|
memory: 40Mi
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
readOnlyRootFilesystem: true
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
- name: csi-attacher
|
|
image: public.ecr.aws/eks-distro/kubernetes-csi/external-attacher:v4.8.1-eks-1-32-7
|
|
imagePullPolicy: IfNotPresent
|
|
args:
|
|
- --timeout=60s
|
|
- --csi-address=$(ADDRESS)
|
|
- --v=2
|
|
- --leader-election=true
|
|
- --kube-api-qps=20
|
|
- --kube-api-burst=100
|
|
- --worker-threads=100
|
|
- --retry-interval-max=5m
|
|
env:
|
|
- name: ADDRESS
|
|
value: /var/lib/csi/sockets/pluginproxy/csi.sock
|
|
volumeMounts:
|
|
- name: socket-dir
|
|
mountPath: /var/lib/csi/sockets/pluginproxy/
|
|
resources:
|
|
limits:
|
|
memory: 256Mi
|
|
requests:
|
|
cpu: 10m
|
|
memory: 40Mi
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
readOnlyRootFilesystem: true
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
- name: csi-snapshotter
|
|
image: public.ecr.aws/eks-distro/kubernetes-csi/external-snapshotter/csi-snapshotter:v8.2.1-eks-1-32-7
|
|
imagePullPolicy: IfNotPresent
|
|
args:
|
|
- --csi-address=$(ADDRESS)
|
|
- --leader-election=true
|
|
- --v=2
|
|
- --extra-create-metadata
|
|
- --kube-api-qps=20
|
|
- --kube-api-burst=100
|
|
- --worker-threads=100
|
|
- --retry-interval-max=30m
|
|
env:
|
|
- name: ADDRESS
|
|
value: /var/lib/csi/sockets/pluginproxy/csi.sock
|
|
volumeMounts:
|
|
- name: socket-dir
|
|
mountPath: /var/lib/csi/sockets/pluginproxy/
|
|
resources:
|
|
limits:
|
|
memory: 256Mi
|
|
requests:
|
|
cpu: 10m
|
|
memory: 40Mi
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
readOnlyRootFilesystem: true
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
- name: csi-resizer
|
|
image: public.ecr.aws/eks-distro/kubernetes-csi/external-resizer:v1.13.2-eks-1-32-7
|
|
imagePullPolicy: IfNotPresent
|
|
args:
|
|
- --timeout=60s
|
|
- --extra-modify-metadata
|
|
- --csi-address=$(ADDRESS)
|
|
- --v=2
|
|
- --handle-volume-inuse-error=false
|
|
- --leader-election=true
|
|
- --kube-api-qps=20
|
|
- --kube-api-burst=100
|
|
- --workers=100
|
|
- --retry-interval-max=30m
|
|
- --feature-gates=VolumeAttributesClass=true
|
|
env:
|
|
- name: ADDRESS
|
|
value: /var/lib/csi/sockets/pluginproxy/csi.sock
|
|
volumeMounts:
|
|
- name: socket-dir
|
|
mountPath: /var/lib/csi/sockets/pluginproxy/
|
|
resources:
|
|
limits:
|
|
memory: 256Mi
|
|
requests:
|
|
cpu: 10m
|
|
memory: 40Mi
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
readOnlyRootFilesystem: true
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
- name: liveness-probe
|
|
image: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe:v2.14.0-eks-1-32-7
|
|
imagePullPolicy: IfNotPresent
|
|
args:
|
|
- --csi-address=/csi/csi.sock
|
|
volumeMounts:
|
|
- name: socket-dir
|
|
mountPath: /csi
|
|
resources:
|
|
limits:
|
|
memory: 256Mi
|
|
requests:
|
|
cpu: 10m
|
|
memory: 40Mi
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
readOnlyRootFilesystem: true
|
|
volumes:
|
|
- name: socket-dir
|
|
emptyDir: {}
|
|
{{IRSA_VOLUME}}
|