From 3f9942a61d2b9cee8bd81fdc54e4e874bcfe46a0 Mon Sep 17 00:00:00 2001 From: briquet Date: Thu, 24 Sep 2026 19:16:35 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=A7(compose)=20replace=20MinIO=20by=20?= =?UTF-8?q?Garage=20for=20local=20development?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The development stacks now run Garage instead of MinIO which is deprecated. Garage is a bit stricter than MinIO: - key IDs and secrets must be at least 8 and 16 characters long - requests must be signed for its region, so every development env now sets AWS_S3_REGION_NAME=local; - cross-origin requests are denied unless the bucket CORS rules allow them, so a one-shot aws-cli container allows the frontend origin to upload files straight to the bucket. --- .github/workflows/ci.yml | 47 ++--- CHANGELOG.md | 1 + Makefile | 5 + bin/Tiltfile | 4 +- compose.yml | 54 +++--- devenv.nix | 2 +- docker/compose.d/compose.podman.yml | 2 +- docker/files/etc/garage/garage.toml | 15 ++ docker/files/etc/nginx/conf.d/default.conf | 6 +- docs/developping_locally.md | 2 +- docs/installation/compose.md | 2 +- env.d/development/common.dist | 7 +- env.d/development/garage.dist | 2 + env.d/development/metadata_collector.dist | 7 +- env.d/development/multi_user_transcriber.dist | 7 +- env.d/development/summary.dist | 7 +- .../core/tests/files/test_api_files_create.py | 4 +- src/helm/env.d/common.yaml.gotmpl | 30 +-- .../env.d/dev-dinum/values.egress.yaml.gotmpl | 6 +- .../dev-keycloak/values.egress.yaml.gotmpl | 6 +- src/helm/extra/templates/garage.yaml | 172 ++++++++++++++++++ src/helm/extra/templates/minio.yaml | 115 ------------ 22 files changed, 293 insertions(+), 210 deletions(-) create mode 100644 docker/files/etc/garage/garage.toml create mode 100644 env.d/development/garage.dist create mode 100644 src/helm/extra/templates/garage.yaml delete mode 100644 src/helm/extra/templates/minio.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8b501727..d1061e1a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -226,8 +226,9 @@ jobs: REDIS_URL: redis://localhost:6379/1 STORAGES_STATICFILES_BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage AWS_S3_ENDPOINT_URL: http://localhost:9000 - AWS_S3_ACCESS_KEY_ID: meet - AWS_S3_SECRET_ACCESS_KEY: password + AWS_S3_ACCESS_KEY_ID: meet-access-key + AWS_S3_SECRET_ACCESS_KEY: meet-secret-access-key + AWS_S3_REGION_NAME: local OIDC_RS_CLIENT_ID: meet OIDC_RS_CLIENT_SECRET: ThisIsAnExampleKeyForDevPurposeOnly OIDC_OP_INTROSPECTION_ENDPOINT: https://oidc.example.com/introspect @@ -250,34 +251,22 @@ jobs: path: "src/backend/core/templates/mail" key: mail-templates-${{ hashFiles('src/mail/mjml') }} - - name: Start MinIO + # Creates the access key and the bucket on startup + - name: Start Garage run: | - docker pull quay.io/minio/minio - docker run -d --name minio \ + docker run -d --name garage \ -p 9000:9000 \ - -e "MINIO_ACCESS_KEY=meet" \ - -e "MINIO_SECRET_KEY=password" \ - -v /data/media:/data \ - quay.io/minio/minio server --console-address :9001 /data + -v "${GITHUB_WORKSPACE}/docker/files/etc/garage/garage.toml:/etc/garage.toml:ro" \ + -e "GARAGE_RPC_SECRET=$(openssl rand -hex 32)" \ + -e "GARAGE_DEFAULT_ACCESS_KEY=meet-access-key" \ + -e "GARAGE_DEFAULT_SECRET_KEY=meet-secret-access-key" \ + -e "GARAGE_DEFAULT_BUCKET=meet-media-storage" \ + dxflrs/garage:v2.4.1 \ + /garage server --single-node --default-bucket - # Tool to wait for a service to be ready - - name: Install Dockerize + - name: Wait for Garage to be ready run: | - curl --proto "=https" --proto-redir "=https" --tlsv1.2 -sSLf \ - https://github.com/jwilder/dockerize/releases/download/v0.8.0/dockerize-linux-amd64-v0.8.0.tar.gz | - sudo tar -C /usr/local/bin -xzv - - - name: Wait for MinIO to be ready - run: | - dockerize -wait tcp://localhost:9000 -timeout 10s - - - name: Configure MinIO - run: | - MINIO=$(docker ps | grep minio/minio | sed -E 's/.*\s+([a-zA-Z0-9_-]+)$/\1/') - docker exec ${MINIO} sh -c \ - "mc alias set meet http://localhost:9000 meet password && \ - mc alias ls && \ - mc mb meet/meet-media-storage" + timeout 30 sh -c 'until docker exec garage /garage health; do sleep 1; done' - name: Install Python uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 @@ -310,9 +299,9 @@ jobs: env: AUTHORIZED_TENANTS: '[{"id": "test-tenant", "api_key": "test-api-token", "webhook_url": "https://example.com/webhook", "webhook_api_key": "test-webhook-api-key"}]' AWS_STORAGE_BUCKET_NAME: "http://meet-media-storage" - AWS_S3_ENDPOINT_URL: "minio:9000" - AWS_S3_ACCESS_KEY_ID: "meet" - AWS_S3_SECRET_ACCESS_KEY: "password" + AWS_S3_ENDPOINT_URL: "garage:9000" + AWS_S3_ACCESS_KEY_ID: "meet-access-key" + AWS_S3_SECRET_ACCESS_KEY: "meet-secret-access-key" WHISPERX_BASE_URL: "https://configure-your-url.com" WHISPERX_ASR_MODEL: "large-v2" WHISPERX_API_KEY: "test-whisperx-secret" diff --git a/CHANGELOG.md b/CHANGELOG.md index e802cf48..ac6c60d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ and this project adheres to - 🔖(helm) release chart 0.0.28 - ♻️(backend) decouple recording event handling from LiveKit egress statuses - ♻️(agents) replace the minio client by boto3 +- 🔧(compose) replace MinIO by Garage for local development ### Fixed diff --git a/Makefile b/Makefile index aafc5d56..ccaf1974 100644 --- a/Makefile +++ b/Makefile @@ -85,6 +85,7 @@ data/static: create-env-files: ## Copy the dist env files to env files create-env-files: \ env.d/development/common \ + env.d/development/garage \ env.d/development/crowdin \ env.d/development/postgresql \ env.d/development/kc_postgresql \ @@ -317,6 +318,10 @@ env.d/development/summary: env.d/development/kube-secret: cp -n env.d/development/kube-secret.dist env.d/development/kube-secret +env.d/development/garage: + sed "s/^GARAGE_RPC_SECRET=.*/GARAGE_RPC_SECRET=$$(openssl rand -hex 32)/" \ + env.d/development/garage.dist > env.d/development/garage + env.d/development/multi_user_transcriber: cp -n env.d/development/multi_user_transcriber.dist env.d/development/multi_user_transcriber diff --git a/bin/Tiltfile b/bin/Tiltfile index 1990a9c0..bd42416e 100644 --- a/bin/Tiltfile +++ b/bin/Tiltfile @@ -104,8 +104,8 @@ k8s_yaml(secret_yaml_generic( k8s_yaml(local('cd ../src/helm && helmfile -n meet -e ${DEV_ENV:-dev-keycloak} template .')) -k8s_resource('minio-bucket', resource_deps=['minio']) -k8s_resource('meet-backend', resource_deps=['postgresql', 'minio', 'redis', 'livekit-livekit-server']) +k8s_resource('garage-cors', resource_deps=['garage']) +k8s_resource('meet-backend', resource_deps=['postgresql', 'garage-cors', 'redis', 'livekit-livekit-server']) k8s_resource('meet-celery-backend', resource_deps=['redis']) k8s_resource('meet-celery-summarize', resource_deps=['redis']) k8s_resource('meet-celery-summary-backend', resource_deps=['redis']) diff --git a/compose.yml b/compose.yml index db4a5f17..0baffcfd 100644 --- a/compose.yml +++ b/compose.yml @@ -15,36 +15,44 @@ services: ports: - "1081:1080" - minio: + garage: user: ${DOCKER_USER:-1000} - image: quay.io/minio/minio + image: dxflrs/garage:v2.4.1 + command: /garage server --single-node --default-bucket + env_file: + - env.d/development/garage environment: - - MINIO_ROOT_USER=meet - - MINIO_ROOT_PASSWORD=password + - GARAGE_DEFAULT_ACCESS_KEY=meet-access-key + - GARAGE_DEFAULT_SECRET_KEY=meet-secret-access-key + - GARAGE_DEFAULT_BUCKET=meet-media-storage ports: - - '9000:9000' - - '9001:9001' + - '127.0.0.1:9000:9000' healthcheck: - test: [ "CMD", "mc", "ready", "local" ] + test: [ "CMD", "/garage", "health" ] interval: 1s timeout: 20s retries: 300 - entrypoint: "" - command: minio server --console-address :9001 /data volumes: - - ./data/media:/data + - ./docker/files/etc/garage/garage.toml:/etc/garage.toml:ro + - ./data/media:/var/lib/garage - createbuckets: - image: quay.io/minio/mc + # Garage denies cross-origin requests by default: allow the frontend to upload files + garage-cors: + image: amazon/aws-cli:2.37.1 + environment: + - AWS_ACCESS_KEY_ID=meet-access-key + - AWS_SECRET_ACCESS_KEY=meet-secret-access-key + - AWS_DEFAULT_REGION=local depends_on: - minio: + garage: condition: service_healthy restart: true - entrypoint: > - sh -c " - /usr/bin/mc alias set meet http://minio:9000 meet password && \ - /usr/bin/mc mb meet/meet-media-storage && \ - exit 0;" + command: + - s3api + - put-bucket-cors + - --endpoint-url=http://garage:9000 + - --bucket=meet-media-storage + - '--cors-configuration={"CORSRules": [{"AllowedOrigins": ["http://localhost:3000"], "AllowedMethods": ["GET", "HEAD", "PUT"], "AllowedHeaders": ["*"], "ExposeHeaders": ["ETag"]}]}' app-dev: build: @@ -70,7 +78,7 @@ services: - postgresql - mailcatcher - redis - - createbuckets + - garage-cors extra_hosts: - "127.0.0.1.nip.io:host-gateway" networks: @@ -110,7 +118,7 @@ services: - postgresql - redis - livekit - - minio + - garage celery: user: ${DOCKER_USER:-1000} @@ -244,7 +252,7 @@ services: - /app/.venv depends_on: - livekit - - minio + - garage develop: watch: - action: rebuild @@ -297,7 +305,7 @@ services: depends_on: - redis-summary - app-summary-dev - - minio + - garage develop: watch: - action: rebuild @@ -317,7 +325,7 @@ services: depends_on: - redis-summary - app-summary-dev - - minio + - garage develop: watch: - action: rebuild diff --git a/devenv.nix b/devenv.nix index 5addcb19..e7f808f5 100644 --- a/devenv.nix +++ b/devenv.nix @@ -163,7 +163,7 @@ in REDIS_URL = "redis://127.0.0.1:6379/1"; CELERY_BROKER_URL = "redis://127.0.0.1:6379/0"; - # S3 / MinIO + # S3 / Garage AWS_S3_ENDPOINT_URL = "http://127.0.0.1:9000"; # OIDC diff --git a/docker/compose.d/compose.podman.yml b/docker/compose.d/compose.podman.yml index 4c40b435..42a0871b 100644 --- a/docker/compose.d/compose.podman.yml +++ b/docker/compose.d/compose.podman.yml @@ -19,7 +19,7 @@ services: <<: *keep-id celery-dev: <<: *keep-id - minio: + garage: <<: *keep-id node: <<: *keep-id diff --git a/docker/files/etc/garage/garage.toml b/docker/files/etc/garage/garage.toml new file mode 100644 index 00000000..9e1c4f36 --- /dev/null +++ b/docker/files/etc/garage/garage.toml @@ -0,0 +1,15 @@ +# Garage configuration for local development only: single node, no replication. +# See https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/ +metadata_dir = "/var/lib/garage/meta" +data_dir = "/var/lib/garage/data" +db_engine = "lmdb" + +replication_factor = 1 + +rpc_bind_addr = "127.0.0.1:3901" +rpc_public_addr = "127.0.0.1:3901" + +[s3_api] +api_bind_addr = "[::]:9000" +# Clients must sign their requests for this region (AWS_S3_REGION_NAME) +s3_region = "local" diff --git a/docker/files/etc/nginx/conf.d/default.conf b/docker/files/etc/nginx/conf.d/default.conf index 627bf256..d508dde2 100644 --- a/docker/files/etc/nginx/conf.d/default.conf +++ b/docker/files/etc/nginx/conf.d/default.conf @@ -17,9 +17,9 @@ server { proxy_set_header X-Amz-Date $authDate; proxy_set_header X-Amz-Content-SHA256 $authContentSha256; - # Get resource from Minio - proxy_pass http://minio:9000/meet-media-storage/; - proxy_set_header Host minio:9000; + # Get resource from Garage + proxy_pass http://garage:9000/meet-media-storage/; + proxy_set_header Host garage:9000; # To use with ds_proxy # proxy_pass http://ds-proxy:4444/upstream/meet-media-storage/; # proxy_set_header Host ds-proxy:4444; diff --git a/docs/developping_locally.md b/docs/developping_locally.md index f72f58f4..b6f4330b 100644 --- a/docs/developping_locally.md +++ b/docs/developping_locally.md @@ -13,7 +13,7 @@ These components rely on a few key services: - PostgreSQL for storing data (users, rooms, recordings) - Redis for caching and inter-service communication -- MinIO for storing files (room recordings) +- Garage for storing files (room recordings) - Celery workers for meeting transcript (optional, required for AI beta features) We provide two stack options for getting Visio up and running for development: diff --git a/docs/installation/compose.md b/docs/installation/compose.md index 71fec3d5..75f11c81 100644 --- a/docs/installation/compose.md +++ b/docs/installation/compose.md @@ -14,7 +14,7 @@ All services are required to run the minimalist instance of LaSuite Meet. Click | **OIDC Provider** | User authentication | [Keycloak setup](../examples/compose/keycloak/README.md) | | **SMTP Service** | Email notifications | - | -> [!NOTE] Some advanced features, as Recording and transcription, require additional services (MinIO, email). See `/features` folder for details. +> [!NOTE] Some advanced features, as Recording and transcription, require additional services (S3-compatible object storage, email). See `/features` folder for details. ## Software Requirements diff --git a/env.d/development/common.dist b/env.d/development/common.dist index 870c81f9..8d9131d1 100644 --- a/env.d/development/common.dist +++ b/env.d/development/common.dist @@ -24,9 +24,10 @@ MEET_BASE_URL="http://localhost:8072" # Media STORAGES_STATICFILES_BACKEND=django.contrib.staticfiles.storage.StaticFilesStorage AWS_S3_DOMAIN_REPLACE=http://localhost:9000 -AWS_S3_ENDPOINT_URL=http://minio:9000 -AWS_S3_ACCESS_KEY_ID=meet -AWS_S3_SECRET_ACCESS_KEY=password +AWS_S3_ENDPOINT_URL=http://garage:9000 +AWS_S3_ACCESS_KEY_ID=meet-access-key +AWS_S3_SECRET_ACCESS_KEY=meet-secret-access-key +AWS_S3_REGION_NAME=local MEDIA_BASE_URL=http://localhost:3000 FILE_UPLOAD_ENABLED=True diff --git a/env.d/development/garage.dist b/env.d/development/garage.dist new file mode 100644 index 00000000..2cb6d421 --- /dev/null +++ b/env.d/development/garage.dist @@ -0,0 +1,2 @@ +# Filled with a random value by `make create-env-files` +GARAGE_RPC_SECRET= diff --git a/env.d/development/metadata_collector.dist b/env.d/development/metadata_collector.dist index ef6e2669..edcb1e4c 100644 --- a/env.d/development/metadata_collector.dist +++ b/env.d/development/metadata_collector.dist @@ -2,8 +2,9 @@ LIVEKIT_URL=ws://livekit:7880 LIVEKIT_API_KEY=devkey LIVEKIT_API_SECRET=secret -AWS_S3_ENDPOINT_URL=minio:9000 -AWS_S3_ACCESS_KEY_ID=meet -AWS_S3_SECRET_ACCESS_KEY=password +AWS_S3_ENDPOINT_URL=garage:9000 +AWS_S3_ACCESS_KEY_ID=meet-access-key +AWS_S3_SECRET_ACCESS_KEY=meet-secret-access-key +AWS_S3_REGION_NAME=local AWS_STORAGE_BUCKET_NAME=meet-media-storage AWS_S3_SECURE_ACCESS=False diff --git a/env.d/development/multi_user_transcriber.dist b/env.d/development/multi_user_transcriber.dist index 8b8e20ce..591f5915 100644 --- a/env.d/development/multi_user_transcriber.dist +++ b/env.d/development/multi_user_transcriber.dist @@ -1,6 +1,7 @@ -AWS_S3_ENDPOINT_URL=minio:9000 -AWS_S3_ACCESS_KEY_ID=meet -AWS_S3_SECRET_ACCESS_KEY=password +AWS_S3_ENDPOINT_URL=garage:9000 +AWS_S3_ACCESS_KEY_ID=meet-access-key +AWS_S3_SECRET_ACCESS_KEY=meet-secret-access-key +AWS_S3_REGION_NAME=local LIVEKIT_URL=ws://livekit:7880 LIVEKIT_API_KEY=devkey diff --git a/env.d/development/summary.dist b/env.d/development/summary.dist index bfd56404..6a674931 100644 --- a/env.d/development/summary.dist +++ b/env.d/development/summary.dist @@ -2,11 +2,12 @@ APP_NAME="meet-app-summary-dev" APP_API_TOKEN="password" AWS_STORAGE_BUCKET_NAME="meet-media-storage" -AWS_S3_ENDPOINT_URL="minio:9000" +AWS_S3_ENDPOINT_URL="garage:9000" AWS_S3_SECURE_ACCESS=false -AWS_S3_ACCESS_KEY_ID="meet" -AWS_S3_SECRET_ACCESS_KEY="password" +AWS_S3_ACCESS_KEY_ID="meet-access-key" +AWS_S3_SECRET_ACCESS_KEY="meet-secret-access-key" +AWS_S3_REGION_NAME="local" WHISPERX_BASE_URL="https://configure-your-url.com" WHISPERX_ASR_MODEL="large-v2" diff --git a/src/backend/core/tests/files/test_api_files_create.py b/src/backend/core/tests/files/test_api_files_create.py index 3bfbae5a..f08e1b69 100644 --- a/src/backend/core/tests/files/test_api_files_create.py +++ b/src/backend/core/tests/files/test_api_files_create.py @@ -117,14 +117,14 @@ def test_api_files_create_file_authenticated_success(): policy_parsed = urlparse(policy) assert policy_parsed.scheme == "http" - assert policy_parsed.netloc in ["minio:9000", "localhost:9000"] + assert policy_parsed.netloc in ["garage:9000", "localhost:9000"] assert policy_parsed.path == f"/meet-media-storage/tmp/files/{file.id!s}.png" query_params = parse_qs(policy_parsed.query) assert query_params.pop("X-Amz-Algorithm") == ["AWS4-HMAC-SHA256"] assert query_params.pop("X-Amz-Credential") == [ - f"meet/{now.strftime('%Y%m%d')}/us-east-1/s3/aws4_request" + f"meet-access-key/{now.strftime('%Y%m%d')}/local/s3/aws4_request" ] assert query_params.pop("X-Amz-Date") == [now.strftime("%Y%m%dT%H%M%SZ")] assert query_params.pop("X-Amz-Expires") == ["60"] diff --git a/src/helm/env.d/common.yaml.gotmpl b/src/helm/env.d/common.yaml.gotmpl index d73bd96d..4dab0d29 100644 --- a/src/helm/env.d/common.yaml.gotmpl +++ b/src/helm/env.d/common.yaml.gotmpl @@ -24,10 +24,11 @@ _summaryEnvVars: &summaryEnvVars APP_NAME: summary-microservice APP_API_TOKEN: password AWS_STORAGE_BUCKET_NAME: meet-media-storage - AWS_S3_ENDPOINT_URL: http://minio.meet.svc.cluster.local:9000/ - AWS_S3_ACCESS_KEY_ID: meet - AWS_S3_SECRET_ACCESS_KEY: password + AWS_S3_ENDPOINT_URL: http://garage.meet.svc.cluster.local:9000/ + AWS_S3_ACCESS_KEY_ID: meet-access-key + AWS_S3_SECRET_ACCESS_KEY: meet-secret-access-key AWS_S3_SECURE_ACCESS: False + AWS_S3_REGION_NAME: local AUTHORIZED_TENANTS: > [ { @@ -161,9 +162,9 @@ backend: FRONTEND_TRANSCRIPTION_DESTINATION: "https://docs.numerique.gouv.fr" FRONTEND_IS_SILENT_LOGIN_ENABLED: False # S3 Storage - AWS_S3_ENDPOINT_URL: http://minio.meet.svc.cluster.local:9000 - AWS_S3_ACCESS_KEY_ID: meet - AWS_S3_SECRET_ACCESS_KEY: password + AWS_S3_ENDPOINT_URL: http://garage.meet.svc.cluster.local:9000 + AWS_S3_ACCESS_KEY_ID: meet-access-key + AWS_S3_SECRET_ACCESS_KEY: meet-secret-access-key AWS_STORAGE_BUCKET_NAME: meet-media-storage # Telephony ROOM_TELEPHONY_ENABLED: True @@ -180,7 +181,7 @@ backend: # Custom Background AWS_S3_REGION_NAME: local AWS_S3_SIGNATURE_VERSION: s3v4 - AWS_S3_DOMAIN_REPLACE: https://minio.127.0.0.1.nip.io + AWS_S3_DOMAIN_REPLACE: https://garage.127.0.0.1.nip.io MEDIA_BASE_URL: https://meet.127.0.0.1.nip.io FILE_UPLOAD_ENABLED: True CELERY_ENABLED: True @@ -264,11 +265,11 @@ ingressMedia: annotations: nginx.ingress.kubernetes.io/auth-url: https://meet.127.0.0.1.nip.io/api/v1.0/recordings/media-auth/ nginx.ingress.kubernetes.io/auth-response-headers: "Authorization, X-Amz-Date, X-Amz-Content-SHA256" - nginx.ingress.kubernetes.io/upstream-vhost: minio.meet.svc.cluster.local:9000 + nginx.ingress.kubernetes.io/upstream-vhost: garage.meet.svc.cluster.local:9000 nginx.ingress.kubernetes.io/rewrite-target: /meet-media-storage/$1 serviceMedia: - host: minio.meet.svc.cluster.local + host: garage.meet.svc.cluster.local port: 9000 # ---- Extra ingress/service for background file uploads ------------ @@ -280,11 +281,11 @@ ingressMediaFiles: annotations: nginx.ingress.kubernetes.io/auth-url: https://meet.127.0.0.1.nip.io/api/v1.0/files/media-auth/ nginx.ingress.kubernetes.io/auth-response-headers: "Authorization, X-Amz-Date, X-Amz-Content-SHA256" - nginx.ingress.kubernetes.io/upstream-vhost: minio.meet.svc.cluster.local:9000 + nginx.ingress.kubernetes.io/upstream-vhost: garage.meet.svc.cluster.local:9000 nginx.ingress.kubernetes.io/rewrite-target: /meet-media-storage/files/$1 serviceMediaFiles: - host: minio.meet.svc.cluster.local + host: garage.meet.svc.cluster.local port: 9000 # ---- STT Orchestration Microservice Components -------------------- @@ -362,10 +363,11 @@ agentMetadata: {{- end }} {{- end }} ENABLE_SILERO_VAD: "false" - AWS_S3_ENDPOINT_URL: minio.meet.svc.cluster.local:9000 - AWS_S3_ACCESS_KEY_ID: meet - AWS_S3_SECRET_ACCESS_KEY: password + AWS_S3_ENDPOINT_URL: garage.meet.svc.cluster.local:9000 + AWS_S3_ACCESS_KEY_ID: meet-access-key + AWS_S3_SECRET_ACCESS_KEY: meet-secret-access-key AWS_S3_SECURE_ACCESS: False + AWS_S3_REGION_NAME: local AWS_STORAGE_BUCKET_NAME: meet-media-storage AWS_S3_OUTPUT_FOLDER: metadata diff --git a/src/helm/env.d/dev-dinum/values.egress.yaml.gotmpl b/src/helm/env.d/dev-dinum/values.egress.yaml.gotmpl index 5d09dfca..185f05ba 100644 --- a/src/helm/env.d/dev-dinum/values.egress.yaml.gotmpl +++ b/src/helm/env.d/dev-dinum/values.egress.yaml.gotmpl @@ -16,11 +16,11 @@ egress: address: redis-master:6379 password: pass s3: - access_key: meet - secret: password + access_key: meet-access-key + secret: meet-secret-access-key region: local bucket: meet-media-storage - endpoint: http://minio:9000 + endpoint: http://garage:9000 force_path_style: true loadBalancer: diff --git a/src/helm/env.d/dev-keycloak/values.egress.yaml.gotmpl b/src/helm/env.d/dev-keycloak/values.egress.yaml.gotmpl index b295b5af..707e3991 100644 --- a/src/helm/env.d/dev-keycloak/values.egress.yaml.gotmpl +++ b/src/helm/env.d/dev-keycloak/values.egress.yaml.gotmpl @@ -19,11 +19,11 @@ egress: address: redis-master:6379 password: pass s3: - access_key: meet - secret: password + access_key: meet-access-key + secret: meet-secret-access-key region: local bucket: meet-media-storage - endpoint: http://minio:9000 + endpoint: http://garage:9000 force_path_style: true loadBalancer: diff --git a/src/helm/extra/templates/garage.yaml b/src/helm/extra/templates/garage.yaml new file mode 100644 index 00000000..19955c32 --- /dev/null +++ b/src/helm/extra/templates/garage.yaml @@ -0,0 +1,172 @@ +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: garage + annotations: + nginx.ingress.kubernetes.io/proxy-body-size: 10m +spec: + rules: + - host: "garage.127.0.0.1.nip.io" + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: garage + port: + number: 9000 + tls: + - hosts: + - garage.127.0.0.1.nip.io + secretName: meet-tls +--- +apiVersion: v1 +kind: Service +metadata: + name: garage + namespace: {{ .Release.Namespace | quote }} +spec: + ports: + - name: client + port: 9000 + protocol: TCP + targetPort: 9000 + selector: + app.kubernetes.io/instance: extra + app.kubernetes.io/name: garage + type: ClusterIP +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: garage-config + namespace: {{ .Release.Namespace | quote }} +data: + garage.toml: | + metadata_dir = "/var/lib/garage/meta" + data_dir = "/var/lib/garage/data" + db_engine = "lmdb" + + replication_factor = 1 + + rpc_bind_addr = "127.0.0.1:3901" + rpc_public_addr = "127.0.0.1:3901" + + [s3_api] + api_bind_addr = "[::]:9000" + # Clients must sign their requests for this region (AWS_S3_REGION_NAME) + s3_region = "local" +--- +apiVersion: v1 +kind: Secret +metadata: + name: garage-dev + namespace: {{ .Release.Namespace | quote }} +type: Opaque +data: + GARAGE_RPC_SECRET: {{ printf "%s/garage-rpc-secret" .Release.Namespace | sha256sum | b64enc }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: garage + namespace: {{ .Release.Namespace | quote }} + labels: + app.kubernetes.io/instance: extra + app.kubernetes.io/name: garage +spec: + selector: + matchLabels: + app.kubernetes.io/instance: extra + app.kubernetes.io/name: garage + replicas: 1 + template: + metadata: + labels: + app.kubernetes.io/instance: extra + app.kubernetes.io/name: garage + spec: + containers: + - name: garage + command: + - /garage + - server + - --single-node + - --default-bucket + env: + - name: GARAGE_RPC_SECRET + valueFrom: + secretKeyRef: + name: garage-dev + key: GARAGE_RPC_SECRET + - name: GARAGE_DEFAULT_ACCESS_KEY + value: meet-access-key + - name: GARAGE_DEFAULT_SECRET_KEY + value: meet-secret-access-key + - name: GARAGE_DEFAULT_BUCKET + value: meet-media-storage + image: "dxflrs/garage:v2.4.1" + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9000 + name: client + readinessProbe: + exec: + command: + - /garage + - health + volumeMounts: + - mountPath: /etc/garage.toml + name: config + subPath: garage.toml + - mountPath: /var/lib/garage + name: data + volumes: + - name: config + configMap: + name: garage-config + - name: data + emptyDir: +--- +# Garage denies cross-origin requests by default: allow the frontend to upload +# files straight to the bucket +apiVersion: batch/v1 +kind: Job +metadata: + name: garage-cors +spec: + template: + spec: + containers: + - name: aws-cli + image: amazon/aws-cli:2.37.1 + env: + - name: AWS_ACCESS_KEY_ID + value: meet-access-key + - name: AWS_SECRET_ACCESS_KEY + value: meet-secret-access-key + - name: AWS_DEFAULT_REGION + value: local + - name: AWS_ENDPOINT_URL + value: http://garage:9000 + - name: BUCKET + value: meet-media-storage + command: + - /bin/sh + - -c + - | + deadline=$(($(date +%s) + 300)) + until aws s3api head-bucket --bucket="$BUCKET" --cli-connect-timeout=5; do + if [ "$(date +%s)" -ge "$deadline" ]; then + echo "Bucket $BUCKET still unavailable on $AWS_ENDPOINT_URL" >&2 + exit 1 + fi + echo "Waiting for bucket $BUCKET on $AWS_ENDPOINT_URL" + sleep 5 + done + exec aws s3api put-bucket-cors --bucket="$BUCKET" \ + --cors-configuration='{"CORSRules": [{"AllowedOrigins": ["https://meet.127.0.0.1.nip.io"], "AllowedMethods": ["GET", "HEAD", "PUT"], "AllowedHeaders": ["*"], "ExposeHeaders": ["ETag"]}]}' + restartPolicy: Never + backoffLimit: 3 diff --git a/src/helm/extra/templates/minio.yaml b/src/helm/extra/templates/minio.yaml deleted file mode 100644 index 7e6a73bb..00000000 --- a/src/helm/extra/templates/minio.yaml +++ /dev/null @@ -1,115 +0,0 @@ ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: minio - annotations: - nginx.ingress.kubernetes.io/proxy-body-size: 10m -spec: - rules: - - host: "minio.127.0.0.1.nip.io" - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: minio - port: - number: 9000 - tls: - - hosts: - - minio.127.0.0.1.nip.io - secretName: meet-tls ---- -apiVersion: v1 -kind: Service -metadata: - name: minio - namespace: {{ .Release.Namespace | quote }} -spec: - ports: - - name: client - port: 9000 - protocol: TCP - targetPort: 9000 - - name: console - port: 9001 - protocol: TCP - targetPort: 9001 - selector: - app.kubernetes.io/instance: extra - app.kubernetes.io/name: minio - type: ClusterIP ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: minio - namespace: {{ .Release.Namespace | quote }} - labels: - app.kubernetes.io/instance: extra - app.kubernetes.io/name: minio -spec: - selector: - matchLabels: - app.kubernetes.io/instance: extra - app.kubernetes.io/name: minio - replicas: 1 - template: - metadata: - labels: - app.kubernetes.io/instance: extra - app.kubernetes.io/name: minio - spec: - containers: - - name: minio - command: - - /bin/sh - - -c - - | - minio server --console-address :9001 /data - env: - - name: MINIO_ROOT_USER - value: meet - - name: MINIO_ROOT_PASSWORD - value: password - image: "quay.io/minio/minio" - imagePullPolicy: IfNotPresent - ports: - - containerPort: 9000 - name: client - - containerPort: 9001 - name: console - volumeMounts: - - mountPath: /data - name: data - - mountPath: /etc/ssl/certs/mkcert-ca.pem - name: mkcert - subPath: rootCA.pem - volumes: - - name: data - emptyDir: - - name: mkcert - secret: - secretName: mkcert ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: minio-bucket -spec: - template: - spec: - containers: - - name: mc - image: quay.io/minio/mc - command: - - /bin/sh - - -c - - | - /usr/bin/mc alias set meet http://minio:9000 meet password && \ - /usr/bin/mc mb meet/meet-media-storage && \ - exit 0 - restartPolicy: Never - backoffLimit: 3