♻️(tilt) use hem dev-backend chart

Use the helm dev-backend chart to share the backend across all projects

bin/Tiltfile

@@ -85,12 +85,13 @@ clean_old_images('localhost:5001/meet-livekit')
 
 k8s_yaml(local('cd ../src/helm && helmfile -n meet -e ${DEV_ENV:-dev} template .'))
 
-k8s_resource('minio-bucket', resource_deps=['minio'])
+k8s_resource('meet-backend', resource_deps=['dev-backend-redis', 'dev-backend-keycloak', 'dev-backend-postgres', 'livekit-livekit-server', 'dev-backend-minio:statefulset'])
-k8s_resource('meet-backend', resource_deps=['postgresql', 'minio', 'redis', 'livekit-livekit-server'])
+k8s_resource('meet-backend-migrate', resource_deps=['dev-backend-postgres'])
-k8s_resource('meet-backend-migrate', resource_deps=['meet-backend'])
+k8s_resource('livekit-livekit-server', resource_deps=['dev-backend-redis'])
 k8s_resource('livekit-livekit-server-test-connection', resource_deps=['livekit-livekit-server'])
-k8s_resource('keycloak', resource_deps=['kc-postgresql'])
 k8s_resource('meet-backend-createsuperuser', resource_deps=['meet-backend-migrate'])
+k8s_resource('dev-backend-keycloak', resource_deps=['dev-backend-keycloak-pg'])
+k8s_resource('livekit-egress', resource_deps=['livekit-livekit-server'])
 
 migration = '''
 set -eu

src/helm/env.d/dev-dinum/values.egress.yaml.gotmpl

@@ -13,8 +13,9 @@ egress:
   {{- end }}
   {{- end }}
   redis:
-    address: redis-master:6379
+    address: dev-backend-redis:6379
     password: pass
+    username: user
   s3:
     access_key: meet
     secret: password

src/helm/env.d/dev-dinum/values.livekit.yaml.gotmpl

@@ -14,8 +14,9 @@ livekit:
     port_range_end: 60000
     tcp_port: 7881
   redis:
-    address: redis-master:6379
+    address: dev-backend-redis:6379
     password: pass
+    username: user
   keys:
   turn:
     enabled: true

src/helm/env.d/dev-dinum/values.meet.yaml.gotmpl

@@ -35,12 +35,21 @@ backend:
     LOGIN_REDIRECT_URL: https://meet.127.0.0.1.nip.io
     LOGIN_REDIRECT_URL_FAILURE: https://meet.127.0.0.1.nip.io
     LOGOUT_REDIRECT_URL: https://meet.127.0.0.1.nip.io
-    DB_HOST: postgres
+    DB_HOST: dev-backend-postgres
-    DB_NAME: meet
+    DB_NAME:
-    DB_USER: dinum
+      secretKeyRef:
-    DB_PASSWORD: pass
+        name: dev-backend-postgres
+        key: database
+    DB_USER:
+      secretKeyRef:
+        name: dev-backend-postgres
+        key: username
+    DB_PASSWORD:
+      secretKeyRef:
+        name: dev-backend-postgres
+        key: password
     DB_PORT: 5432
-    REDIS_URL: redis://default:pass@redis-master:6379/1
+    REDIS_URL: redis://user:pass@dev-backend-redis:6379/1
     STORAGES_STATICFILES_BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage
     {{- with .Values.livekit.keys }}
     {{- range $key, $value := . }}
@@ -58,8 +67,8 @@ backend:
     FRONTEND_USE_FRENCH_GOV_FOOTER: True
     FRONTEND_USE_PROCONNECT_BUTTON: True
     FRONTEND_MANIFEST_LINK: "https://docs.numerique.gouv.fr/docs/1ef86abf-f7e0-46ce-b6c7-8be8b8af4c3d/"
-    AWS_S3_ENDPOINT_URL: http://minio.meet.svc.cluster.local:9000
+    AWS_S3_ENDPOINT_URL: http://dev-backend-minio.meet.svc.cluster.local:9000
-    AWS_S3_ACCESS_KEY_ID: meet
+    AWS_S3_ACCESS_KEY_ID: dinum
     AWS_S3_SECRET_ACCESS_KEY: password
     AWS_STORAGE_BUCKET_NAME: meet-media-storage
     AWS_S3_REGION_NAME: local

src/helm/env.d/dev-keycloak/values.egress.yaml.gotmpl

@@ -13,8 +13,9 @@ egress:
   {{- end }}
   {{- end }}
   redis:
-    address: redis-master:6379
+    address: dev-backend-redis:6379
     password: pass
+    username: user
   s3:
     access_key: meet
     secret: password

src/helm/env.d/dev-keycloak/values.livekit.yaml.gotmpl

@@ -14,8 +14,9 @@ livekit:
     port_range_end: 60000
     tcp_port: 7881
   redis:
-    address: redis-master:6379
+    address: dev-backend-redis:6379
     password: pass
+    username: user
   keys:
   turn:
     enabled: true

src/helm/env.d/dev-keycloak/values.meet.yaml.gotmpl

@@ -35,12 +35,21 @@ backend:
     LOGIN_REDIRECT_URL: https://meet.127.0.0.1.nip.io
     LOGIN_REDIRECT_URL_FAILURE: https://meet.127.0.0.1.nip.io
     LOGOUT_REDIRECT_URL: https://meet.127.0.0.1.nip.io
-    DB_HOST: postgres
+    DB_HOST: dev-backend-postgres
-    DB_NAME: meet
+    DB_NAME:
-    DB_USER: dinum
+      secretKeyRef:
-    DB_PASSWORD: pass
+        name: dev-backend-postgres
+        key: database
+    DB_USER:
+      secretKeyRef:
+        name: dev-backend-postgres
+        key: username
+    DB_PASSWORD:
+      secretKeyRef:
+        name: dev-backend-postgres
+        key: password
     DB_PORT: 5432
-    REDIS_URL: redis://default:pass@redis-master:6379/1
+    REDIS_URL: redis://user:pass@dev-backend-redis:6379/1
     STORAGES_STATICFILES_BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage
     {{- with .Values.livekit.keys }}
     {{- range $key, $value := . }}
@@ -57,8 +66,8 @@ backend:
     FRONTEND_TRANSCRIPT: "{'form_beta_users': 'https://grist.numerique.gouv.fr/o/docs/forms/3fFfvJoTBEQ6ZiMi8zsQwX/17'}"
     FRONTEND_FEEDBACK: "{'url': 'https://grist.numerique.gouv.fr/o/docs/cbMv4G7pLY3Z/USER-RESEARCH-or-LA-SUITE/f/26'}"
     FRONTEND_MANIFEST_LINK: "https://docs.numerique.gouv.fr/docs/1ef86abf-f7e0-46ce-b6c7-8be8b8af4c3d/"
-    AWS_S3_ENDPOINT_URL: http://minio.meet.svc.cluster.local:9000
+    AWS_S3_ENDPOINT_URL: http://dev-backend-minio.meet.svc.cluster.local:9000
-    AWS_S3_ACCESS_KEY_ID: meet
+    AWS_S3_ACCESS_KEY_ID: dinum
     AWS_S3_SECRET_ACCESS_KEY: password
     AWS_STORAGE_BUCKET_NAME: meet-media-storage
     AWS_S3_REGION_NAME: local

src/helm/env.d/dev/values.egress.yaml.gotmpl

@@ -13,8 +13,9 @@ egress:
   {{- end }}
   {{- end }}
   redis:
-    address: redis-master:6379
+    address: dev-backend-redis:6379
     password: pass
+    username: user
   s3:
     access_key: meet
     secret: password

src/helm/env.d/dev/values.livekit.yaml.gotmpl

@@ -14,8 +14,9 @@ livekit:
     port_range_end: 60000
     tcp_port: 7881
   redis:
-    address: redis-master:6379
+    address: dev-backend-redis:6379
     password: pass
+    username: user
   keys:
   turn:
     enabled: true

src/helm/env.d/dev/values.meet.yaml.gotmpl

@@ -57,12 +57,21 @@ backend:
     LOGIN_REDIRECT_URL: https://meet.127.0.0.1.nip.io
     LOGIN_REDIRECT_URL_FAILURE: https://meet.127.0.0.1.nip.io
     LOGOUT_REDIRECT_URL: https://meet.127.0.0.1.nip.io
-    DB_HOST: postgres
+    DB_HOST: dev-backend-postgres
-    DB_NAME: meet
+    DB_NAME:
-    DB_USER: dinum
+      secretKeyRef:
-    DB_PASSWORD: pass
+        name: dev-backend-postgres
+        key: database
+    DB_USER:
+      secretKeyRef:
+        name: dev-backend-postgres
+        key: username
+    DB_PASSWORD:
+      secretKeyRef:
+        name: dev-backend-postgres
+        key: password
     DB_PORT: 5432
-    REDIS_URL: redis://default:pass@redis-master:6379/1
+    REDIS_URL: redis://user:pass@dev-backend-redis:6379/1
     STORAGES_STATICFILES_BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage
     {{- with .Values.livekit.keys }}
     {{- range $key, $value := . }}
@@ -74,8 +83,8 @@ backend:
     ALLOW_UNREGISTERED_ROOMS: False
     FRONTEND_SILENCE_LIVEKIT_DEBUG: False
     FRONTEND_SUPPORT: "{'id': '58ea6697-8eba-4492-bc59-ad6562585041'}"
-    AWS_S3_ENDPOINT_URL: http://minio.meet.svc.cluster.local:9000
+    AWS_S3_ENDPOINT_URL: http://dev-backend-minio.meet.svc.cluster.local:9000
-    AWS_S3_ACCESS_KEY_ID: meet
+    AWS_S3_ACCESS_KEY_ID: dinum
     AWS_S3_SECRET_ACCESS_KEY: password
     AWS_STORAGE_BUCKET_NAME: meet-media-storage
     AWS_S3_REGION_NAME: local

src/helm/extra/templates/kc-postgresql-sts.yaml

@@ -1,61 +0,0 @@
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: kc-postgres
-  namespace: {{ .Release.Namespace | quote }}
-spec:
-  ports:
-  - name: tcp-postgresql
-    port: 5432
-    protocol: TCP
-    targetPort: tcp-postgresql
-  selector:
-    app.kubernetes.io/instance: extra
-    app.kubernetes.io/name: kc-postgresql
-  type: ClusterIP
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: kc-postgresql
-  namespace: {{ .Release.Namespace | quote }}
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/instance: extra
-      app.kubernetes.io/name: kc-postgresql
-  serviceName: "kc-postgres"
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/instance: extra
-        app.kubernetes.io/name: kc-postgresql
-    spec:
-      terminationGracePeriodSeconds: 10
-      containers:
-      - name: pg
-        image: postgres:16-alpine
-        ports:
-        - containerPort: 5432
-          name: tcp-postgresql
-        env:
-          - name: POSTGRES_PASSWORD
-            value: pass
-          - name: POSTGRES_USER
-            value: dinum
-          - name: POSTGRES_DB
-            value: keycloak
-        volumeMounts:
-        - name: data
-          mountPath: /var/lib/postgresql
-  volumeClaimTemplates:
-  - metadata:
-      name: data
-    spec:
-      accessModes: [ "ReadWriteOnce" ]
-      resources:
-        requests:
-          storage: 1Gi
-

src/helm/extra/templates/keycloak-sts.yaml

@@ -1,104 +0,0 @@
----
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: keycloak
-spec:
-  rules:
-  - host: "keycloak.127.0.0.1.nip.io"
-    http:
-      paths:
-      - path: /
-        pathType: Prefix
-        backend:
-          service:
-            name: keycloak
-            port:
-              number: 8080
-  tls:
-  - hosts:
-    - keycloak.127.0.0.1.nip.io
-    secretName: meet-tls
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: keycloak
-  namespace: {{ .Release.Namespace | quote }}
-spec:
-  ports:
-  - name: tcp-keycloak
-    port: 8080
-    protocol: TCP
-    targetPort: tcp-keycloak
-  selector:
-    app.kubernetes.io/instance: extra
-    app.kubernetes.io/name: keycloak
-  type: ClusterIP
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: realm
-data:
-  meet.json: |
-{{ .Values.realm | indent 4 }}
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: keycloak
-  namespace: {{ .Release.Namespace | quote }}
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/instance: extra
-      app.kubernetes.io/name: keycloak
-  serviceName: "keycloak"
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/instance: extra
-        app.kubernetes.io/name: keycloak
-    spec:
-      terminationGracePeriodSeconds: 10
-      containers:
-      - name: keycloak
-        image: quay.io/keycloak/keycloak:20.0.1
-        args:
-          - start-dev
-          - --features=preview
-          - --import-realm
-          - --proxy=edge
-          - --hostname=keycloak.127.0.0.1.nip.io
-          - --hostname-strict=false
-          - --hostname-strict-https=false
-        ports:
-        - containerPort: 8080
-          name: tcp-keycloak
-        env:
-          - name: KEYCLOAK_ADMIN
-            value: admin
-          - name: KEYCLOAK_ADMIN_PASSWORD
-            value: admin
-          - name: PROXY_ADDRESS_FORWARDING
-            value: 'true'
-          - name: KC_DB_URL_HOST
-            value: kc_postgresql
-          - name: KC_DB_URL_DATABASE
-            value: keycloak
-          - name: KC_DB_PASSWORD
-            value: pass
-          - name: KC_DB_USERNAME
-            value: dinum
-          - name: KC_DB_SCHEMA
-            value: public
-        volumeMounts:
-        - name: realm
-          mountPath: "/opt/keycloak/data/import"
-          readOnly: true
-      volumes:
-      - name: realm
-        configMap:
-          name: realm

src/helm/extra/templates/minio.yaml

@@ -1,107 +0,0 @@
----
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: minio
-spec:
-  rules:
-  - host: "minio.127.0.0.1.nip.io"
-    http:
-      paths:
-      - path: /
-        pathType: Prefix
-        backend:
-          service:
-            name: minio
-            port:
-              number: 9001
-  tls:
-  - hosts:
-    - minio.127.0.0.1.nip.io
-    secretName: meet-tls
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: minio
-  namespace: {{ .Release.Namespace | quote }}
-spec:
-  ports:
-  - name: client
-    port: 9000
-    protocol: TCP
-    targetPort: 9000
-  - name: console
-    port: 9001
-    protocol: TCP
-    targetPort: 9001
-  selector:
-    app.kubernetes.io/instance: extra
-    app.kubernetes.io/name: minio
-  type: ClusterIP
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: minio
-  namespace: {{ .Release.Namespace | quote }}
-  labels:
-    app.kubernetes.io/instance: extra
-    app.kubernetes.io/name: minio
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/instance: extra
-      app.kubernetes.io/name: minio
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/instance: extra
-        app.kubernetes.io/name: minio
-    spec:
-      containers:
-      - name: minio
-        command:
-          - /bin/sh
-          - -c
-          - |
-            minio server --console-address :9001 /data
-        env:
-          - name: MINIO_ROOT_USER
-            value: meet
-          - name: MINIO_ROOT_PASSWORD
-            value: password
-        image: "minio/minio"
-        imagePullPolicy: IfNotPresent
-        ports:
-        - containerPort: 9000
-          name: client
-        - containerPort: 9001
-          name: console
-        volumeMounts:
-        - mountPath: /data
-          name: data
-      volumes:
-      - name: data
-        emptyDir:
----
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: minio-bucket
-spec:
-  template:
-    spec:
-      containers:
-      - name: mc
-        image: minio/mc
-        command:
-          - /bin/sh
-          - -c
-          - |
-            /usr/bin/mc alias set meet http://minio:9000 meet password && \
-            /usr/bin/mc mb meet/meet-media-storage && \
-            exit 0
-      restartPolicy: Never
-  backoffLimit: 1

src/helm/extra/templates/postgresql-sts.yaml

@@ -1,70 +0,0 @@
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: postgres
-  namespace: {{ .Release.Namespace | quote }}
-spec:
-  ports:
-  - name: tcp-postgresql
-    port: 5432
-    protocol: TCP
-    targetPort: tcp-postgresql
-  selector:
-    app.kubernetes.io/instance: extra
-    app.kubernetes.io/name: postgresql
-  type: ClusterIP
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: postgresql
-  namespace: {{ .Release.Namespace | quote }}
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/instance: extra
-      app.kubernetes.io/name: postgresql
-  serviceName: "postgres"
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/instance: extra
-        app.kubernetes.io/name: postgresql
-    spec:
-      terminationGracePeriodSeconds: 10
-      containers:
-      - name: pg
-        image: postgres:16-alpine
-        readinessProbe:
-          exec:
-            command: [ "pg_isready", "-U", "dinum", "-d", "meet", "-h", "127.0.0.1" ]
-          initialDelaySeconds: 5
-          periodSeconds: 5
-        livenessProbe:
-          exec:
-            command: [ "pg_isready", "-U", "dinum", "-d", "meet", "-h", "127.0.0.1" ]
-          initialDelaySeconds: 15
-          periodSeconds: 10
-        ports:
-        - containerPort: 5432
-          name: tcp-postgresql
-        env:
-          - name: POSTGRES_PASSWORD
-            value: pass
-          - name: POSTGRES_USER
-            value: dinum
-          - name: POSTGRES_DB
-            value: meet
-        volumeMounts:
-        - name: data
-          mountPath: /var/lib/postgresql/data
-  volumeClaimTemplates:
-  - metadata:
-      name: data
-    spec:
-      accessModes: [ "ReadWriteOnce" ]
-      resources:
-        requests:
-          storage: 1Gi

src/helm/extra/templates/redis.yaml

@@ -1,65 +0,0 @@
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: redis-master
-  namespace: {{ .Release.Namespace | quote }}
-spec:
-  ports:
-  - name: tcp-redis
-    port: 6379
-    protocol: TCP
-    targetPort: tcp-redis
-  selector:
-    app.kubernetes.io/instance: extra
-    app.kubernetes.io/name: redis
-  type: ClusterIP
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: redis
-data:
-  redis.conf: |
-    bind 0.0.0.0
-    port 6379
-    user default on >pass ~* &* +@all
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: redis
-  namespace: {{ .Release.Namespace | quote }}
-  labels:
-    app.kubernetes.io/instance: extra
-    app.kubernetes.io/name: redis
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/instance: extra
-      app.kubernetes.io/name: redis
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/instance: extra
-        app.kubernetes.io/name: redis
-    spec:
-      containers:
-      - name: redis
-        args:
-          - redis-server
-          - /usr/local/etc/redis/redis.conf
-        image: "redis:8.2-alpine"
-        imagePullPolicy: IfNotPresent
-        ports:
-        - containerPort: 6379
-          name: tcp-redis
-        volumeMounts:
-        - name: redis
-          mountPath: "/usr/local/etc/redis"
-          readOnly: true
-      volumes:
-      - name: redis
-        configMap:
-          name: redis

src/helm/helmfile.yaml

@@ -16,6 +16,9 @@ repositories:
 - name: livekit
   url: https://helm.livekit.io
 
+- name: dev-backends
+  url: https://suitenumerique.github.io/helm-dev-backend
+
 releases:
   - name: extra
     installed: {{ regexMatch "^dev.*" .Environment.Name | toYaml }}
@@ -65,3 +68,61 @@ releases:
       - env.d/{{ .Environment.Name }}/values.secrets.yaml
     secrets:
       - env.d/{{ .Environment.Name }}/secrets.enc.yaml
+
+  - name: dev-backend
+    namespace: {{ .Namespace }}
+    chart: dev-backends/dev-backend
+    version: 0.0.1
+    values:
+      - postgres:
+          enabled: true
+          name: postgres
+          #serviceNameOverride: postgres
+          image: postgres:16-alpine
+          username: dinum
+          password: pass
+          database: dinum
+          size: 1Gi
+      - redis:
+          enabled: true
+          name: redis
+          #serviceNameOverride: redis
+          image: redis:8.2-alpine
+          username: user
+          password: pass
+      - minio:
+          enabled: true
+          image: minio/minio
+          name: minio
+          #serviceNameOverride: minio
+          hostname: minio.127.0.0.1.nip.io
+          tls:
+            enabled: true
+            secretName: meet-tls
+          username: dinum
+          password: password
+          bucket: meet-media-storage
+          size: 1Gi
+      - keycloak:
+          enabled: true
+          image: quay.io/keycloak/keycloak:20.0.1
+          name: keycloak
+          #serviceNameOverride: keycloak
+          hostname: keycloak.127.0.0.1.nip.io
+          username: admin
+          password: pass
+          tls:
+            enabled: true
+            secretName: meet-tls
+          db:
+            username: dinum
+            password: pass
+            database: keycloak
+            size: 1Gi
+            image: postgres:16-alpine
+          realm:
+            name: meet
+            username: meet
+            password: meet
+            email: meet@example.com
+