From c9207318890009b0dff52bc2ffb50b2f25a30990 Mon Sep 17 00:00:00 2001 From: briquet Date: Mon, 12 Oct 2026 22:52:22 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9A=A1=EF=B8=8F(backend)=20add=20opt-in=20ps?= =?UTF-8?q?ycopg=20connection=20pool?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each request opened and closed its own Postgres connection which added a big overhead. The goal is to add Django's native psycopg pool behind a new DB_PSYCOPG_POOL_ENABLED settings with explicit min/max size and keepalive. Each pool lives in one process, and gunicorn's sync workers serve a single request at a time which means that a given worker won't use more than one connection. The default pool minimum size is set to 1, which keeps always one connection open and the maximum size to 3 as headroom for code touching the database from an extra thread. Pooled connections can sit idle for hours, so TCP keepalives are enabled to keep load balancers and NAT from dropping them silently, and each connection is health checked on checkout so a broken one is replaced instead of failing the request. --- CHANGELOG.md | 1 + docs/installation/kubernetes.md | 7 +- env.d/development/postgresql.dist | 3 +- .../core/tests/files/test_api_files_create.py | 28 ++-- .../files/test_api_files_upload_ended.py | 10 +- .../core/tests/test_settings_database_pool.py | 130 ++++++++++++++++++ src/backend/meet/settings.py | 63 ++++++++- src/backend/pyproject.toml | 2 +- src/backend/uv.lock | 19 ++- 9 files changed, 242 insertions(+), 21 deletions(-) create mode 100644 src/backend/core/tests/test_settings_database_pool.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d6f8f7eeb..a568a53da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ and this project adheres to - ✨(backend) introduce a token exchange endpoint for iframe embeds - ⚡️(backend) send recording notification emails from a Celery task - ✨(backend) allow static feature flags without an analytics backend #1809 +- ⚡️(backend) add an opt-in psycopg connection pool ### Changed diff --git a/docs/installation/kubernetes.md b/docs/installation/kubernetes.md index 48bf7264e..bfe3d95e7 100644 --- a/docs/installation/kubernetes.md +++ b/docs/installation/kubernetes.md @@ -318,12 +318,17 @@ These are the environmental options available on meet backend. | DJANGO_SECRET_KEY | Secret key used for Django security | | | DJANGO_SILENCED_SYSTEM_CHECKS | Silence Django system checks | [] | | DJANGO_ALLOW_UNSECURE_USER_LISTING | Allow unsecure user listing | false | -| DB_ENGINE | Database engine used | django.db.backends.postgresql_psycopg2 | +| DB_ENGINE | Database engine used | django.db.backends.postgresql | | DB_NAME | Name of the database | meet | | DB_USER | User used to connect to database | dinum | | DB_PASSWORD | Password used to connect to the database | pass | | DB_HOST | Hostname of the database | localhost | | DB_PORT | Port to connect to database | 5432 | +| DB_PSYCOPG_POOL_ENABLED | Enable the psycopg connection pool | false | +| DB_PSYCOPG_POOL_MIN_SIZE | Connections each pool keeps open | 1 | +| DB_PSYCOPG_POOL_MAX_SIZE | Maximum connections per pool | 3 | +| DB_PSYCOPG_POOL_TIMEOUT | Seconds a request waits for a free pool connection before failing | 30 | +| DB_PSYCOPG_POOL_MAX_IDLE | Seconds an idle connection above the minimum stays open | 600 | | STORAGES_STATICFILES_BACKEND | Static file serving engine | whitenoise.storage.CompressedManifestStaticFilesStorage | | AWS_S3_ENDPOINT_URL | S3 host endpoint | | | AWS_S3_ACCESS_KEY_ID | S3 access key | | diff --git a/env.d/development/postgresql.dist b/env.d/development/postgresql.dist index b1c3e3387..fdb8c676f 100644 --- a/env.d/development/postgresql.dist +++ b/env.d/development/postgresql.dist @@ -8,4 +8,5 @@ DB_HOST=postgresql DB_NAME=meet DB_USER=dinum DB_PASSWORD=pass -DB_PORT=5432 \ No newline at end of file +DB_PORT=5432 +DB_PSYCOPG_POOL_ENABLED=True diff --git a/src/backend/core/tests/files/test_api_files_create.py b/src/backend/core/tests/files/test_api_files_create.py index f08e1b69b..6fade97f1 100644 --- a/src/backend/core/tests/files/test_api_files_create.py +++ b/src/backend/core/tests/files/test_api_files_create.py @@ -6,6 +6,7 @@ from concurrent.futures import ThreadPoolExecutor from urllib.parse import parse_qs, urlparse from uuid import uuid4 +from django.db import connection from django.utils import timezone import pytest @@ -361,18 +362,21 @@ def test_api_files_create_file_race_condition(): """ def create_item(title): - user = factories.UserFactory() - client = APIClient() - client.force_login(user) - return client.post( - "/api/v1.0/files/", - { - "title": title, - "type": FileTypeChoices.BACKGROUND_IMAGE, - "filename": "my_file.png", - }, - format="json", - ) + try: + user = factories.UserFactory() + client = APIClient() + client.force_login(user) + return client.post( + "/api/v1.0/files/", + { + "title": title, + "type": FileTypeChoices.BACKGROUND_IMAGE, + "filename": "my_file.png", + }, + format="json", + ) + finally: + connection.close() with ThreadPoolExecutor(max_workers=2) as executor: future1 = executor.submit(create_item, "my item 1") diff --git a/src/backend/core/tests/files/test_api_files_upload_ended.py b/src/backend/core/tests/files/test_api_files_upload_ended.py index a89c578cb..cd3f7c165 100644 --- a/src/backend/core/tests/files/test_api_files_upload_ended.py +++ b/src/backend/core/tests/files/test_api_files_upload_ended.py @@ -5,6 +5,7 @@ from concurrent.futures import ThreadPoolExecutor from io import BytesIO from django.core.files.storage import default_storage +from django.db import connection import pytest from rest_framework.test import APIClient @@ -295,9 +296,12 @@ def test_api_file_upload_ended_concurrent_calls_are_serialized(settings): default_storage.save(file.temporary_file_key, BytesIO(b"my prose")) def call_upload_ended(): - client = APIClient() - client.force_login(user) - return client.post(f"/api/v1.0/files/{file.id!s}/upload-ended/") + try: + client = APIClient() + client.force_login(user) + return client.post(f"/api/v1.0/files/{file.id!s}/upload-ended/") + finally: + connection.close() with ThreadPoolExecutor(max_workers=2) as executor: futures = [ diff --git a/src/backend/core/tests/test_settings_database_pool.py b/src/backend/core/tests/test_settings_database_pool.py new file mode 100644 index 000000000..7925d2974 --- /dev/null +++ b/src/backend/core/tests/test_settings_database_pool.py @@ -0,0 +1,130 @@ +"""Unit tests for the opt-in psycopg connection pool settings.""" + +# pylint: disable=protected-access + +from django.core.exceptions import ImproperlyConfigured + +import pytest + +from meet import settings + + +def make_configuration(**overrides): + """Build a throwaway configuration with plain values and its own DATABASES.""" + attributes = { + "DATABASES": { + "default": { + "ENGINE": "django.db.backends.postgresql", + "OPTIONS": {"sslmode": "require"}, + } + }, + "DB_PSYCOPG_POOL_ENABLED": True, + "DB_PSYCOPG_POOL_MIN_SIZE": 1, + "DB_PSYCOPG_POOL_MAX_SIZE": 3, + "DB_PSYCOPG_POOL_TIMEOUT": 30.0, + "DB_PSYCOPG_POOL_MAX_IDLE": 600.0, + **overrides, + } + return type("PoolConfiguration", (settings.Test,), attributes) + + +def test_database_pool_disabled_leaves_options_untouched(): + """Without the flag, no connection pool is configured.""" + configuration = make_configuration(DB_PSYCOPG_POOL_ENABLED=False) + + configuration._configure_database_pool() + + assert configuration.DATABASES["default"]["OPTIONS"] == {"sslmode": "require"} + assert "CONN_HEALTH_CHECKS" not in configuration.DATABASES["default"] + + +def test_database_pool_enabled_merges_pool_options(): + """The pool and keepalive options are added next to the existing ones.""" + configuration = make_configuration() + + configuration._configure_database_pool() + + assert configuration.DATABASES["default"]["OPTIONS"] == { + "sslmode": "require", + "keepalives_idle": 60, + "keepalives_interval": 10, + "keepalives_count": 5, + "pool": {"min_size": 1, "max_size": 3, "timeout": 30.0, "max_idle": 600.0}, + } + + +def test_database_pool_enabled_keeps_explicit_keepalives(): + """Keepalive values already set, e.g. from DATABASE_URL, are not overridden.""" + configuration = make_configuration( + DATABASES={ + "default": { + "ENGINE": "django.db.backends.postgresql", + "OPTIONS": {"keepalives_idle": "30", "keepalives": "0"}, + } + } + ) + + configuration._configure_database_pool() + + options = configuration.DATABASES["default"]["OPTIONS"] + assert options["keepalives_idle"] == "30" + assert options["keepalives"] == "0" + assert options["keepalives_interval"] == 10 + + +@pytest.mark.parametrize("health_checks", [None, False]) +def test_database_pool_enabled_forces_health_checks(health_checks): + """Pooled connections are checked on checkout, even if DATABASE_URL said no.""" + database = {"ENGINE": "django.db.backends.postgresql"} + if health_checks is not None: + database["CONN_HEALTH_CHECKS"] = health_checks + configuration = make_configuration(DATABASES={"default": database}) + + configuration._configure_database_pool() + + assert configuration.DATABASES["default"]["CONN_HEALTH_CHECKS"] is True + + +def test_database_pool_enabled_without_options(): + """A database without OPTIONS, as built from DATABASE_URL, gets them created.""" + configuration = make_configuration( + DATABASES={"default": {"ENGINE": "django.db.backends.postgresql_psycopg2"}} + ) + + configuration._configure_database_pool() + + assert configuration.DATABASES["default"]["OPTIONS"]["pool"]["max_size"] == 3 + + +@pytest.mark.parametrize( + "overrides,match", + [ + ( + {"DATABASES": {"default": {"ENGINE": "django.db.backends.sqlite3"}}}, + "requires the django.db.backends.postgresql engine", + ), + ( + { + "DATABASES": { + "default": { + "ENGINE": "django.db.backends.postgresql", + "CONN_MAX_AGE": 60, + } + } + }, + "cannot be combined with persistent connections", + ), + ( + {"DB_PSYCOPG_POOL_MIN_SIZE": 6}, + "MIN_SIZE cannot be greater than", + ), + ({"DB_PSYCOPG_POOL_TIMEOUT": 0.0}, "must be positive"), + ({"DB_PSYCOPG_POOL_MAX_IDLE": -1.0}, "must be positive"), + ], +) +def test_database_pool_invalid_configuration(overrides, match): + """An invalid pool setup fails at boot.""" + configuration = make_configuration(**overrides) + + with pytest.raises(ImproperlyConfigured, match=match): + configuration._configure_database_pool() diff --git a/src/backend/meet/settings.py b/src/backend/meet/settings.py index 1f29edbfc..e76950a55 100755 --- a/src/backend/meet/settings.py +++ b/src/backend/meet/settings.py @@ -17,6 +17,7 @@ import warnings from os import path from socket import gethostbyname, gethostname +from django.core.exceptions import ImproperlyConfigured from django.utils.translation import gettext_lazy as _ import dj_database_url @@ -139,7 +140,7 @@ class Base(Configuration): ) else { "ENGINE": values.Value( - "django.db.backends.postgresql_psycopg2", + "django.db.backends.postgresql", environ_name="DB_ENGINE", environ_prefix=None, ), @@ -154,6 +155,21 @@ class Base(Configuration): "PORT": values.Value(5432, environ_name="DB_PORT", environ_prefix=None), } } + DB_PSYCOPG_POOL_ENABLED = values.BooleanValue( + False, environ_name="DB_PSYCOPG_POOL_ENABLED", environ_prefix=None + ) + DB_PSYCOPG_POOL_MIN_SIZE = values.PositiveIntegerValue( + 1, environ_name="DB_PSYCOPG_POOL_MIN_SIZE", environ_prefix=None + ) + DB_PSYCOPG_POOL_MAX_SIZE = values.PositiveIntegerValue( + 3, environ_name="DB_PSYCOPG_POOL_MAX_SIZE", environ_prefix=None + ) + DB_PSYCOPG_POOL_TIMEOUT = values.FloatValue( + 30.0, environ_name="DB_PSYCOPG_POOL_TIMEOUT", environ_prefix=None + ) + DB_PSYCOPG_POOL_MAX_IDLE = values.FloatValue( + 600.0, environ_name="DB_PSYCOPG_POOL_MAX_IDLE", environ_prefix=None + ) DEFAULT_AUTO_FIELD = "django.db.models.AutoField" # Static files (CSS, JavaScript, Images) @@ -1540,6 +1556,50 @@ class Base(Configuration): f"'{cls.AUDIT_LOG_DATA_STREAM_NAMESPACE}' must not contain '-'." ) + @classmethod + def _configure_database_pool(cls): + """Inject the psycopg pool options into the default database when enabled.""" + if not cls.DB_PSYCOPG_POOL_ENABLED: + return + + database = cls.DATABASES["default"] + if database.get("ENGINE") not in ( + "django.db.backends.postgresql", + "django.db.backends.postgresql_psycopg2", + ): + raise ImproperlyConfigured( + "DB_PSYCOPG_POOL_ENABLED requires the django.db.backends.postgresql " + f"engine, got {database.get('ENGINE')!r}." + ) + if database.get("CONN_MAX_AGE"): + raise ImproperlyConfigured( + "DB_PSYCOPG_POOL_ENABLED cannot be combined with persistent " + "connections; set CONN_MAX_AGE to 0." + ) + if cls.DB_PSYCOPG_POOL_MIN_SIZE > cls.DB_PSYCOPG_POOL_MAX_SIZE: + raise ImproperlyConfigured( + "DB_PSYCOPG_POOL_MIN_SIZE cannot be greater than " + "DB_PSYCOPG_POOL_MAX_SIZE." + ) + if cls.DB_PSYCOPG_POOL_TIMEOUT <= 0 or cls.DB_PSYCOPG_POOL_MAX_IDLE <= 0: + raise ImproperlyConfigured( + "DB_PSYCOPG_POOL_TIMEOUT and DB_PSYCOPG_POOL_MAX_IDLE must be positive." + ) + + database["OPTIONS"] = { + "keepalives_idle": 60, + "keepalives_interval": 10, + "keepalives_count": 5, + **database.get("OPTIONS", {}), + "pool": { + "min_size": cls.DB_PSYCOPG_POOL_MIN_SIZE, + "max_size": cls.DB_PSYCOPG_POOL_MAX_SIZE, + "timeout": cls.DB_PSYCOPG_POOL_TIMEOUT, + "max_idle": cls.DB_PSYCOPG_POOL_MAX_IDLE, + }, + } + database["CONN_HEALTH_CHECKS"] = True + @classmethod def post_setup(cls): """Post setup configuration. @@ -1555,6 +1615,7 @@ class Base(Configuration): cls._check_recording_encoding_maps() cls._check_audit_log_data_stream_namespace() + cls._configure_database_pool() if ( cls.SUMMARY_SERVICE_VERSION == 1 diff --git a/src/backend/pyproject.toml b/src/backend/pyproject.toml index 62c816692..e0880408f 100644 --- a/src/backend/pyproject.toml +++ b/src/backend/pyproject.toml @@ -52,7 +52,7 @@ dependencies = [ "menshen-client==0.2.0", "nested-multipart-parser==1.6.0", "posthog==7.44.0", - "psycopg[binary]==3.3.4", + "psycopg[binary,pool]==3.3.4", "pydantic==2.13.4", "PyJWT==2.14.0", "python-frontmatter==1.3.0", diff --git a/src/backend/uv.lock b/src/backend/uv.lock index 55a36b692..881176510 100644 --- a/src/backend/uv.lock +++ b/src/backend/uv.lock @@ -1332,7 +1332,7 @@ dependencies = [ { name = "nested-multipart-parser" }, { name = "phonenumbers" }, { name = "posthog" }, - { name = "psycopg", extra = ["binary"] }, + { name = "psycopg", extra = ["binary", "pool"] }, { name = "pydantic" }, { name = "pyjwt" }, { name = "python-frontmatter" }, @@ -1398,7 +1398,7 @@ requires-dist = [ { name = "nested-multipart-parser", specifier = "==1.6.0" }, { name = "phonenumbers", specifier = "==9.0.37" }, { name = "posthog", specifier = "==7.44.0" }, - { name = "psycopg", extras = ["binary"], specifier = "==3.3.4" }, + { name = "psycopg", extras = ["binary", "pool"], specifier = "==3.3.4" }, { name = "pydantic", specifier = "==2.13.4" }, { name = "pyjwt", specifier = "==2.14.0" }, { name = "python-frontmatter", specifier = "==1.3.0" }, @@ -1889,6 +1889,9 @@ wheels = [ binary = [ { name = "psycopg-binary", marker = "implementation_name != 'pypy'" }, ] +pool = [ + { name = "psycopg-pool" }, +] [[package]] name = "psycopg-binary" @@ -1919,6 +1922,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/eb/e6/5fff07a70d1f945ed90ae131c3bd76cab32beff7c58c6db15ad5820b6d1f/psycopg_binary-3.3.4-cp314-cp314-win_amd64.whl", hash = "sha256:c37e024c07308cd06cf3ec51bfd0e7f6157585a4d84d1bce4a7f5f7913719bf8", size = 3666849, upload-time = "2026-05-01T23:31:51.165Z" }, ] +[[package]] +name = "psycopg-pool" +version = "3.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/74/5e/c0664b968b102ff68b811d999c728546c48d5c1eec03e3bbaf88c0cb4472/psycopg_pool-3.3.3.tar.gz", hash = "sha256:df87b5d9d0ad7db37f6cdad4fa8ce113d250f5997f6db38e9a99192fb67f9e1d", size = 32006, upload-time = "2026-09-22T15:53:24.947Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/b4/452c6607a0f479465cd8a9b0d9956919fcb150050c1f83f9f11e6b8ee8dc/psycopg_pool-3.3.3-py3-none-any.whl", hash = "sha256:9b9cd6a4fcec47a410f7e82d408540e7f77b478509e91b44c1a5457a13e5ff37", size = 40304, upload-time = "2026-09-22T15:53:23.712Z" }, +] + [[package]] name = "ptyprocess" version = "0.7.0"