✨(infra) add celery backend to helm & improvements

* Add a simple celery-backend deployment to helm chart, which
  uses the same docker image as the backend, and runs a single
  worker.
* Update dev keycloack values accordingly
* Update Tiltfile to properly set service dependencies
* Update minio deployment to increase the default max body size limit
  (relevant to uploading files)

bin/Tiltfile

@@ -99,6 +99,9 @@ k8s_yaml(local('cd ../src/helm && helmfile -n meet -e ${DEV_ENV:-dev} template .
 
 k8s_resource('minio-bucket', resource_deps=['minio'])
 k8s_resource('meet-backend', resource_deps=['postgresql', 'minio', 'redis', 'livekit-livekit-server'])
+k8s_resource('meet-celery-backend', resource_deps=['redis'])
+k8s_resource('meet-celery-summarize', resource_deps=['redis'])
+k8s_resource('meet-celery-transcribe', resource_deps=['redis'])
 k8s_resource('meet-backend-migrate', resource_deps=['meet-backend'])
 k8s_resource('livekit-livekit-server-test-connection', resource_deps=['livekit-livekit-server'])
 k8s_resource('keycloak', resource_deps=['kc-postgresql'])

src/helm/env.d/dev-keycloak/values.meet.yaml.gotmpl

@@ -83,6 +83,8 @@ backend:
     APPLICATION_BASE_URL: https://meet.127.0.0.1.nip.io
     FILE_UPLOAD_ENABLED: True
     AWS_S3_DOMAIN_REPLACE: https://minio.127.0.0.1.nip.io
+    CELERY_ENABLED: True
+    CELERY_BROKER_URL: redis://default:pass@redis-master:6379/1
 
 
   migrate:

src/helm/extra/templates/minio.yaml

@@ -3,6 +3,8 @@ apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   name: minio
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-body-size: 10m
 spec:
   rules:
   - host: "minio.127.0.0.1.nip.io"

src/helm/meet/templates/_helpers.tpl

@@ -175,6 +175,15 @@ Requires top level scope
 {{ include "meet.fullname" . }}-summary
 {{- end }}
 
+{{/*
+Full name for the Celery Backend
+
+Requires top level scope
+*/}}
+{{- define "meet.celeryBackend.fullname" -}}
+{{ include "meet.fullname" . }}-celery-backend
+{{- end }}
+
 {{/*
 Full name for the Celery Transcribe
 

src/helm/meet/templates/celery_backend_deployment.yaml

@@ -0,0 +1,153 @@
+{{- $envVars := include "meet.env.transformDict" (mergeOverwrite (default dict .Values.backend.envVars) (default dict .Values.celeryBackend.envVars)) -}}
+{{- $fullName := include "meet.celeryBackend.fullname" . -}}
+{{- $component := "celery-backend" -}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ $fullName }}
+  annotations:
+   {{- with .Values.celeryBackend.dpAnnotations }}
+   {{- toYaml . | nindent 4 }}
+   {{- end }}
+  namespace: {{ .Release.Namespace | quote }}
+  labels:
+    {{- include "meet.common.labels" (list . $component) | nindent 4 }}
+spec:
+  replicas: {{ .Values.celeryBackend.replicas }}
+  selector:
+    matchLabels:
+      {{- include "meet.common.selectorLabels" (list . $component) | nindent 6 }}
+  template:
+    metadata:
+      annotations:
+        {{- with .Values.celeryBackend.podAnnotations }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      labels:
+        {{- include "meet.common.selectorLabels" (list . $component) | nindent 8 }}
+    spec:
+      {{- if $.Values.image.credentials }}
+      imagePullSecrets:
+        - name: {{ include "meet.secret.dockerconfigjson.name" (dict "fullname" (include "meet.fullname" .) "imageCredentials" $.Values.image.credentials) }}
+      {{- end }}
+      shareProcessNamespace: {{ .Values.celeryBackend.shareProcessNamespace }}
+      containers:
+        {{- with .Values.celeryBackend.sidecars }}
+          {{- toYaml . | nindent 8 }}
+        {{- end }}
+        - name: {{ .Chart.Name }}
+          image: "{{ (.Values.backend.image | default dict).repository | default .Values.image.repository }}:{{ (.Values.backend.image | default dict).tag | default .Values.image.tag }}"
+          imagePullPolicy: {{ (.Values.backend.image | default dict).pullPolicy | default .Values.image.pullPolicy }}
+          {{- with .Values.celeryBackend.command }}
+          command:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.celeryBackend.args }}
+          args:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          env:
+            {{- if $envVars }}
+            {{- $envVars | indent 12 }}
+            {{- end }}
+          {{- with .Values.celeryBackend.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          ports:
+            - name: http
+              containerPort: {{ .Values.celeryBackend.service.targetPort }}
+              protocol: TCP
+          {{- if .Values.celeryBackend.probes.liveness }}
+          livenessProbe:
+            {{- include "meet.probes.abstract" (merge .Values.celeryBackend.probes.liveness (dict "targetPort" .Values.celeryBackend.service.targetPort )) | nindent 12 }}
+          {{- end }}
+          {{- if .Values.celeryBackend.probes.readiness }}
+          readinessProbe:
+            {{- include "meet.probes.abstract" (merge .Values.celeryBackend.probes.readiness (dict "targetPort" .Values.celeryBackend.service.targetPort )) | nindent 12 }}
+          {{- end }}
+          {{- if .Values.celeryBackend.probes.startup }}
+          startupProbe:
+            {{- include "meet.probes.abstract" (merge .Values.celeryBackend.probes.startup (dict "targetPort" .Values.celeryBackend.service.targetPort )) | nindent 12 }}
+          {{- end }}
+          {{- with .Values.celeryBackend.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          volumeMounts:
+            {{- range $index, $value := .Values.mountFiles }}
+            - name: "files-{{ $index }}"
+              mountPath: {{ $value.path }}
+              subPath: content
+            {{- end }}
+            {{- range $name, $volume := .Values.celeryBackend.persistence }}
+            - name: "{{ $name }}"
+              mountPath: "{{ $volume.mountPath }}"
+            {{- end }}
+            {{- range .Values.celeryBackend.extraVolumeMounts }}
+            - name: {{ .name }}
+              mountPath: {{ .mountPath }}
+              subPath: {{ .subPath | default "" }}
+              readOnly: {{ .readOnly }}
+            {{- end }}
+      {{- with .Values.celeryBackend.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.celeryBackend.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.celeryBackend.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      volumes:
+        {{- range $index, $value := .Values.mountFiles }}
+        - name: "files-{{ $index }}"
+          configMap:
+            name: "{{ include "meet.fullname" $ }}-files-{{ $index }}"
+        {{- end }}
+        {{- range $name, $volume := .Values.celeryBackend.persistence }}
+        - name: "{{ $name }}"
+          {{- if eq $volume.type "emptyDir" }}
+          emptyDir: {}
+          {{- else }}
+          persistentVolumeClaim:
+            claimName: "{{ $fullName }}-{{ $name }}"
+          {{- end }}
+        {{- end }}
+        {{- range .Values.celeryBackend.extraVolumes }}
+        - name: {{ .name }}
+          {{- if .existingClaim }}
+          persistentVolumeClaim:
+            claimName: {{ .existingClaim }}
+          {{- else if .hostPath }}
+          hostPath:
+            {{ toYaml .hostPath | nindent 12 }}
+          {{- else if .csi }}
+          csi:
+            {{- toYaml .csi | nindent 12 }}
+          {{- else if .configMap }}
+          configMap:
+            {{- toYaml .configMap | nindent 12 }}
+          {{- else if .emptyDir }}
+          emptyDir:
+            {{- toYaml .emptyDir | nindent 12 }}
+          {{- else }}
+          emptyDir: {}
+          {{- end }}
+        {{- end }}
+---
+{{ if .Values.celeryBackend.pdb.enabled }}
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: {{ $fullName }}
+  namespace: {{ .Release.Namespace | quote }}
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      {{- include "meet.common.selectorLabels" (list . $component) | nindent 6 }}
+{{ end }}

src/helm/meet/values.yaml

@@ -601,6 +601,108 @@ summary:
   pdb:
     enabled: true
 
+## @section celeryBackend
+
+celeryBackend:
+
+  ## @param celeryBackend.dpAnnotations Annotations to add to the celeryBackend Deployment
+  dpAnnotations: {}
+
+  ## @param celeryBackend.command Override the celeryBackend container command
+  command:
+    - "celery"
+    - "-A"
+    - "meet.celery_app"
+    - "worker"
+    - "--pool=solo"
+    - "--loglevel=info"
+    - "-Q"
+    - "meet-backend"
+
+  ## @param celeryBackend.args Override the celeryBackend container args
+  args: []
+
+  ## @param celeryBackend.replicas Amount of celeryBackend replicas
+  replicas: 1
+
+  ## @param celeryBackend.shareProcessNamespace Enable share process namespace between containers
+  shareProcessNamespace: false
+
+  ## @param celeryBackend.sidecars Add sidecars containers to celeryBackend deployment
+  sidecars: []
+
+  ## @param celeryBackend.migrateJobAnnotations Annotations for the migrate job
+  migrateJobAnnotations: {}
+
+  ## @param celeryBackend.securityContext Configure celeryBackend Pod security context
+  securityContext: null
+
+  ## @param celeryBackend.envVars Configure celeryBackend container environment variables
+  ## @extra celeryBackend.envVars.BY_VALUE Example environment variable by setting value directly
+  ## @extra celeryBackend.envVars.FROM_CONFIGMAP.configMapKeyRef.name Name of a ConfigMap when configuring env vars from a ConfigMap
+  ## @extra celeryBackend.envVars.FROM_CONFIGMAP.configMapKeyRef.key Key within a ConfigMap when configuring env vars from a ConfigMap
+  ## @extra celeryBackend.envVars.FROM_SECRET.secretKeyRef.name Name of a Secret when configuring env vars from a Secret
+  ## @extra celeryBackend.envVars.FROM_SECRET.secretKeyRef.key Key within a Secret when configuring env vars from a Secret
+  ## @skip celeryBackend.envVars
+  envVars:
+    <<: *commonEnvVars
+
+  ## @param celeryBackend.podAnnotations Annotations to add to the celeryBackend Pod
+  podAnnotations: {}
+
+  ## @param celeryBackend.service.type celeryBackend Service type
+  ## @param celeryBackend.service.port celeryBackend Service listening port
+  ## @param celeryBackend.service.targetPort celeryBackend container listening port
+  ## @param celeryBackend.service.annotations Annotations to add to the celeryBackend Service
+  service:
+    type: ClusterIP
+    port: 80
+    targetPort: 8000
+    annotations: {}
+
+  ## @param celeryBackend.probes Configure celeryBackend probes
+  ## @param celeryBackend.probes.liveness.path [nullable] Configure path for celeryBackend HTTP liveness probe
+  ## @param celeryBackend.probes.liveness.targetPort [nullable] Configure port for celeryBackend HTTP liveness probe
+  ## @param celeryBackend.probes.liveness.initialDelaySeconds [nullable] Configure initial delay for celeryBackend liveness probe
+  ## @param celeryBackend.probes.liveness.initialDelaySeconds [nullable] Configure timeout for celeryBackend liveness probe
+  ## @param celeryBackend.probes.startup.path [nullable] Configure path for celeryBackend HTTP startup probe
+  ## @param celeryBackend.probes.startup.targetPort [nullable] Configure port for celeryBackend HTTP startup probe
+  ## @param celeryBackend.probes.startup.initialDelaySeconds [nullable] Configure initial delay for celeryBackend startup probe
+  ## @param celeryBackend.probes.startup.initialDelaySeconds [nullable] Configure timeout for celeryBackend startup probe
+  ## @param celeryBackend.probes.readiness.path [nullable] Configure path for celeryBackend HTTP readiness probe
+  ## @param celeryBackend.probes.readiness.targetPort [nullable] Configure port for celeryBackend HTTP readiness probe
+  ## @param celeryBackend.probes.readiness.initialDelaySeconds [nullable] Configure initial delay for celeryBackend readiness probe
+  ## @param celeryBackend.probes.readiness.initialDelaySeconds [nullable] Configure timeout for celeryBackend readiness probe
+  probes: {}
+
+  ## @param celeryBackend.resources Resource requirements for the celeryBackend container
+  resources: {}
+
+  ## @param celeryBackend.nodeSelector Node selector for the celeryBackend Pod
+  nodeSelector: {}
+
+  ## @param celeryBackend.tolerations Tolerations for the celeryBackend Pod
+  tolerations: []
+
+  ## @param celeryBackend.affinity Affinity for the celeryBackend Pod
+  affinity: {}
+
+  ## @param celeryBackend.persistence Additional volumes to create and mount on the celeryBackend. Used for debugging purposes
+  ## @extra celeryBackend.persistence.volume-name.size Size of the additional volume
+  ## @extra celeryBackend.persistence.volume-name.type Type of the additional volume, persistentVolumeClaim or emptyDir
+  ## @extra celeryBackend.persistence.volume-name.mountPath Path where the volume should be mounted to
+  persistence: {}
+
+  ## @param celeryBackend.extraVolumeMounts Additional volumes to mount on the celeryBackend.
+  extraVolumeMounts: []
+
+  ## @param celeryBackend.extraVolumes Additional volumes to mount on the celeryBackend.
+  extraVolumes: []
+
+  ## @param celeryBackend.pdb.enabled Enable pdb on celeryBackend
+  pdb:
+    enabled: false
+
 ## @section celeryTranscribe
 
 celeryTranscribe: