diff --git a/CHANGELOG.md b/CHANGELOG.md index d6f8f7eeb..a568a53da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ and this project adheres to - ✨(backend) introduce a token exchange endpoint for iframe embeds - ⚡️(backend) send recording notification emails from a Celery task - ✨(backend) allow static feature flags without an analytics backend #1809 +- ⚡️(backend) add an opt-in psycopg connection pool ### Changed diff --git a/docs/installation/kubernetes.md b/docs/installation/kubernetes.md index 48bf7264e..bfe3d95e7 100644 --- a/docs/installation/kubernetes.md +++ b/docs/installation/kubernetes.md @@ -318,12 +318,17 @@ These are the environmental options available on meet backend. | DJANGO_SECRET_KEY | Secret key used for Django security | | | DJANGO_SILENCED_SYSTEM_CHECKS | Silence Django system checks | [] | | DJANGO_ALLOW_UNSECURE_USER_LISTING | Allow unsecure user listing | false | -| DB_ENGINE | Database engine used | django.db.backends.postgresql_psycopg2 | +| DB_ENGINE | Database engine used | django.db.backends.postgresql | | DB_NAME | Name of the database | meet | | DB_USER | User used to connect to database | dinum | | DB_PASSWORD | Password used to connect to the database | pass | | DB_HOST | Hostname of the database | localhost | | DB_PORT | Port to connect to database | 5432 | +| DB_PSYCOPG_POOL_ENABLED | Enable the psycopg connection pool | false | +| DB_PSYCOPG_POOL_MIN_SIZE | Connections each pool keeps open | 1 | +| DB_PSYCOPG_POOL_MAX_SIZE | Maximum connections per pool | 3 | +| DB_PSYCOPG_POOL_TIMEOUT | Seconds a request waits for a free pool connection before failing | 30 | +| DB_PSYCOPG_POOL_MAX_IDLE | Seconds an idle connection above the minimum stays open | 600 | | STORAGES_STATICFILES_BACKEND | Static file serving engine | whitenoise.storage.CompressedManifestStaticFilesStorage | | AWS_S3_ENDPOINT_URL | S3 host endpoint | | | AWS_S3_ACCESS_KEY_ID | S3 access key | | diff --git a/env.d/development/postgresql.dist b/env.d/development/postgresql.dist index b1c3e3387..fdb8c676f 100644 --- a/env.d/development/postgresql.dist +++ b/env.d/development/postgresql.dist @@ -8,4 +8,5 @@ DB_HOST=postgresql DB_NAME=meet DB_USER=dinum DB_PASSWORD=pass -DB_PORT=5432 \ No newline at end of file +DB_PORT=5432 +DB_PSYCOPG_POOL_ENABLED=True diff --git a/src/backend/core/tests/files/test_api_files_create.py b/src/backend/core/tests/files/test_api_files_create.py index f08e1b69b..6fade97f1 100644 --- a/src/backend/core/tests/files/test_api_files_create.py +++ b/src/backend/core/tests/files/test_api_files_create.py @@ -6,6 +6,7 @@ from concurrent.futures import ThreadPoolExecutor from urllib.parse import parse_qs, urlparse from uuid import uuid4 +from django.db import connection from django.utils import timezone import pytest @@ -361,18 +362,21 @@ def test_api_files_create_file_race_condition(): """ def create_item(title): - user = factories.UserFactory() - client = APIClient() - client.force_login(user) - return client.post( - "/api/v1.0/files/", - { - "title": title, - "type": FileTypeChoices.BACKGROUND_IMAGE, - "filename": "my_file.png", - }, - format="json", - ) + try: + user = factories.UserFactory() + client = APIClient() + client.force_login(user) + return client.post( + "/api/v1.0/files/", + { + "title": title, + "type": FileTypeChoices.BACKGROUND_IMAGE, + "filename": "my_file.png", + }, + format="json", + ) + finally: + connection.close() with ThreadPoolExecutor(max_workers=2) as executor: future1 = executor.submit(create_item, "my item 1") diff --git a/src/backend/core/tests/files/test_api_files_upload_ended.py b/src/backend/core/tests/files/test_api_files_upload_ended.py index a89c578cb..cd3f7c165 100644 --- a/src/backend/core/tests/files/test_api_files_upload_ended.py +++ b/src/backend/core/tests/files/test_api_files_upload_ended.py @@ -5,6 +5,7 @@ from concurrent.futures import ThreadPoolExecutor from io import BytesIO from django.core.files.storage import default_storage +from django.db import connection import pytest from rest_framework.test import APIClient @@ -295,9 +296,12 @@ def test_api_file_upload_ended_concurrent_calls_are_serialized(settings): default_storage.save(file.temporary_file_key, BytesIO(b"my prose")) def call_upload_ended(): - client = APIClient() - client.force_login(user) - return client.post(f"/api/v1.0/files/{file.id!s}/upload-ended/") + try: + client = APIClient() + client.force_login(user) + return client.post(f"/api/v1.0/files/{file.id!s}/upload-ended/") + finally: + connection.close() with ThreadPoolExecutor(max_workers=2) as executor: futures = [ diff --git a/src/backend/core/tests/test_settings_database_pool.py b/src/backend/core/tests/test_settings_database_pool.py new file mode 100644 index 000000000..7925d2974 --- /dev/null +++ b/src/backend/core/tests/test_settings_database_pool.py @@ -0,0 +1,130 @@ +"""Unit tests for the opt-in psycopg connection pool settings.""" + +# pylint: disable=protected-access + +from django.core.exceptions import ImproperlyConfigured + +import pytest + +from meet import settings + + +def make_configuration(**overrides): + """Build a throwaway configuration with plain values and its own DATABASES.""" + attributes = { + "DATABASES": { + "default": { + "ENGINE": "django.db.backends.postgresql", + "OPTIONS": {"sslmode": "require"}, + } + }, + "DB_PSYCOPG_POOL_ENABLED": True, + "DB_PSYCOPG_POOL_MIN_SIZE": 1, + "DB_PSYCOPG_POOL_MAX_SIZE": 3, + "DB_PSYCOPG_POOL_TIMEOUT": 30.0, + "DB_PSYCOPG_POOL_MAX_IDLE": 600.0, + **overrides, + } + return type("PoolConfiguration", (settings.Test,), attributes) + + +def test_database_pool_disabled_leaves_options_untouched(): + """Without the flag, no connection pool is configured.""" + configuration = make_configuration(DB_PSYCOPG_POOL_ENABLED=False) + + configuration._configure_database_pool() + + assert configuration.DATABASES["default"]["OPTIONS"] == {"sslmode": "require"} + assert "CONN_HEALTH_CHECKS" not in configuration.DATABASES["default"] + + +def test_database_pool_enabled_merges_pool_options(): + """The pool and keepalive options are added next to the existing ones.""" + configuration = make_configuration() + + configuration._configure_database_pool() + + assert configuration.DATABASES["default"]["OPTIONS"] == { + "sslmode": "require", + "keepalives_idle": 60, + "keepalives_interval": 10, + "keepalives_count": 5, + "pool": {"min_size": 1, "max_size": 3, "timeout": 30.0, "max_idle": 600.0}, + } + + +def test_database_pool_enabled_keeps_explicit_keepalives(): + """Keepalive values already set, e.g. from DATABASE_URL, are not overridden.""" + configuration = make_configuration( + DATABASES={ + "default": { + "ENGINE": "django.db.backends.postgresql", + "OPTIONS": {"keepalives_idle": "30", "keepalives": "0"}, + } + } + ) + + configuration._configure_database_pool() + + options = configuration.DATABASES["default"]["OPTIONS"] + assert options["keepalives_idle"] == "30" + assert options["keepalives"] == "0" + assert options["keepalives_interval"] == 10 + + +@pytest.mark.parametrize("health_checks", [None, False]) +def test_database_pool_enabled_forces_health_checks(health_checks): + """Pooled connections are checked on checkout, even if DATABASE_URL said no.""" + database = {"ENGINE": "django.db.backends.postgresql"} + if health_checks is not None: + database["CONN_HEALTH_CHECKS"] = health_checks + configuration = make_configuration(DATABASES={"default": database}) + + configuration._configure_database_pool() + + assert configuration.DATABASES["default"]["CONN_HEALTH_CHECKS"] is True + + +def test_database_pool_enabled_without_options(): + """A database without OPTIONS, as built from DATABASE_URL, gets them created.""" + configuration = make_configuration( + DATABASES={"default": {"ENGINE": "django.db.backends.postgresql_psycopg2"}} + ) + + configuration._configure_database_pool() + + assert configuration.DATABASES["default"]["OPTIONS"]["pool"]["max_size"] == 3 + + +@pytest.mark.parametrize( + "overrides,match", + [ + ( + {"DATABASES": {"default": {"ENGINE": "django.db.backends.sqlite3"}}}, + "requires the django.db.backends.postgresql engine", + ), + ( + { + "DATABASES": { + "default": { + "ENGINE": "django.db.backends.postgresql", + "CONN_MAX_AGE": 60, + } + } + }, + "cannot be combined with persistent connections", + ), + ( + {"DB_PSYCOPG_POOL_MIN_SIZE": 6}, + "MIN_SIZE cannot be greater than", + ), + ({"DB_PSYCOPG_POOL_TIMEOUT": 0.0}, "must be positive"), + ({"DB_PSYCOPG_POOL_MAX_IDLE": -1.0}, "must be positive"), + ], +) +def test_database_pool_invalid_configuration(overrides, match): + """An invalid pool setup fails at boot.""" + configuration = make_configuration(**overrides) + + with pytest.raises(ImproperlyConfigured, match=match): + configuration._configure_database_pool() diff --git a/src/backend/meet/settings.py b/src/backend/meet/settings.py index 1f29edbfc..e76950a55 100755 --- a/src/backend/meet/settings.py +++ b/src/backend/meet/settings.py @@ -17,6 +17,7 @@ import warnings from os import path from socket import gethostbyname, gethostname +from django.core.exceptions import ImproperlyConfigured from django.utils.translation import gettext_lazy as _ import dj_database_url @@ -139,7 +140,7 @@ class Base(Configuration): ) else { "ENGINE": values.Value( - "django.db.backends.postgresql_psycopg2", + "django.db.backends.postgresql", environ_name="DB_ENGINE", environ_prefix=None, ), @@ -154,6 +155,21 @@ class Base(Configuration): "PORT": values.Value(5432, environ_name="DB_PORT", environ_prefix=None), } } + DB_PSYCOPG_POOL_ENABLED = values.BooleanValue( + False, environ_name="DB_PSYCOPG_POOL_ENABLED", environ_prefix=None + ) + DB_PSYCOPG_POOL_MIN_SIZE = values.PositiveIntegerValue( + 1, environ_name="DB_PSYCOPG_POOL_MIN_SIZE", environ_prefix=None + ) + DB_PSYCOPG_POOL_MAX_SIZE = values.PositiveIntegerValue( + 3, environ_name="DB_PSYCOPG_POOL_MAX_SIZE", environ_prefix=None + ) + DB_PSYCOPG_POOL_TIMEOUT = values.FloatValue( + 30.0, environ_name="DB_PSYCOPG_POOL_TIMEOUT", environ_prefix=None + ) + DB_PSYCOPG_POOL_MAX_IDLE = values.FloatValue( + 600.0, environ_name="DB_PSYCOPG_POOL_MAX_IDLE", environ_prefix=None + ) DEFAULT_AUTO_FIELD = "django.db.models.AutoField" # Static files (CSS, JavaScript, Images) @@ -1540,6 +1556,50 @@ class Base(Configuration): f"'{cls.AUDIT_LOG_DATA_STREAM_NAMESPACE}' must not contain '-'." ) + @classmethod + def _configure_database_pool(cls): + """Inject the psycopg pool options into the default database when enabled.""" + if not cls.DB_PSYCOPG_POOL_ENABLED: + return + + database = cls.DATABASES["default"] + if database.get("ENGINE") not in ( + "django.db.backends.postgresql", + "django.db.backends.postgresql_psycopg2", + ): + raise ImproperlyConfigured( + "DB_PSYCOPG_POOL_ENABLED requires the django.db.backends.postgresql " + f"engine, got {database.get('ENGINE')!r}." + ) + if database.get("CONN_MAX_AGE"): + raise ImproperlyConfigured( + "DB_PSYCOPG_POOL_ENABLED cannot be combined with persistent " + "connections; set CONN_MAX_AGE to 0." + ) + if cls.DB_PSYCOPG_POOL_MIN_SIZE > cls.DB_PSYCOPG_POOL_MAX_SIZE: + raise ImproperlyConfigured( + "DB_PSYCOPG_POOL_MIN_SIZE cannot be greater than " + "DB_PSYCOPG_POOL_MAX_SIZE." + ) + if cls.DB_PSYCOPG_POOL_TIMEOUT <= 0 or cls.DB_PSYCOPG_POOL_MAX_IDLE <= 0: + raise ImproperlyConfigured( + "DB_PSYCOPG_POOL_TIMEOUT and DB_PSYCOPG_POOL_MAX_IDLE must be positive." + ) + + database["OPTIONS"] = { + "keepalives_idle": 60, + "keepalives_interval": 10, + "keepalives_count": 5, + **database.get("OPTIONS", {}), + "pool": { + "min_size": cls.DB_PSYCOPG_POOL_MIN_SIZE, + "max_size": cls.DB_PSYCOPG_POOL_MAX_SIZE, + "timeout": cls.DB_PSYCOPG_POOL_TIMEOUT, + "max_idle": cls.DB_PSYCOPG_POOL_MAX_IDLE, + }, + } + database["CONN_HEALTH_CHECKS"] = True + @classmethod def post_setup(cls): """Post setup configuration. @@ -1555,6 +1615,7 @@ class Base(Configuration): cls._check_recording_encoding_maps() cls._check_audit_log_data_stream_namespace() + cls._configure_database_pool() if ( cls.SUMMARY_SERVICE_VERSION == 1 diff --git a/src/backend/pyproject.toml b/src/backend/pyproject.toml index 62c816692..e0880408f 100644 --- a/src/backend/pyproject.toml +++ b/src/backend/pyproject.toml @@ -52,7 +52,7 @@ dependencies = [ "menshen-client==0.2.0", "nested-multipart-parser==1.6.0", "posthog==7.44.0", - "psycopg[binary]==3.3.4", + "psycopg[binary,pool]==3.3.4", "pydantic==2.13.4", "PyJWT==2.14.0", "python-frontmatter==1.3.0", diff --git a/src/backend/uv.lock b/src/backend/uv.lock index 55a36b692..881176510 100644 --- a/src/backend/uv.lock +++ b/src/backend/uv.lock @@ -1332,7 +1332,7 @@ dependencies = [ { name = "nested-multipart-parser" }, { name = "phonenumbers" }, { name = "posthog" }, - { name = "psycopg", extra = ["binary"] }, + { name = "psycopg", extra = ["binary", "pool"] }, { name = "pydantic" }, { name = "pyjwt" }, { name = "python-frontmatter" }, @@ -1398,7 +1398,7 @@ requires-dist = [ { name = "nested-multipart-parser", specifier = "==1.6.0" }, { name = "phonenumbers", specifier = "==9.0.37" }, { name = "posthog", specifier = "==7.44.0" }, - { name = "psycopg", extras = ["binary"], specifier = "==3.3.4" }, + { name = "psycopg", extras = ["binary", "pool"], specifier = "==3.3.4" }, { name = "pydantic", specifier = "==2.13.4" }, { name = "pyjwt", specifier = "==2.14.0" }, { name = "python-frontmatter", specifier = "==1.3.0" }, @@ -1889,6 +1889,9 @@ wheels = [ binary = [ { name = "psycopg-binary", marker = "implementation_name != 'pypy'" }, ] +pool = [ + { name = "psycopg-pool" }, +] [[package]] name = "psycopg-binary" @@ -1919,6 +1922,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/eb/e6/5fff07a70d1f945ed90ae131c3bd76cab32beff7c58c6db15ad5820b6d1f/psycopg_binary-3.3.4-cp314-cp314-win_amd64.whl", hash = "sha256:c37e024c07308cd06cf3ec51bfd0e7f6157585a4d84d1bce4a7f5f7913719bf8", size = 3666849, upload-time = "2026-05-01T23:31:51.165Z" }, ] +[[package]] +name = "psycopg-pool" +version = "3.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/74/5e/c0664b968b102ff68b811d999c728546c48d5c1eec03e3bbaf88c0cb4472/psycopg_pool-3.3.3.tar.gz", hash = "sha256:df87b5d9d0ad7db37f6cdad4fa8ce113d250f5997f6db38e9a99192fb67f9e1d", size = 32006, upload-time = "2026-09-22T15:53:24.947Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/b4/452c6607a0f479465cd8a9b0d9956919fcb150050c1f83f9f11e6b8ee8dc/psycopg_pool-3.3.3-py3-none-any.whl", hash = "sha256:9b9cd6a4fcec47a410f7e82d408540e7f77b478509e91b44c1a5457a13e5ff37", size = 40304, upload-time = "2026-09-22T15:53:23.712Z" }, +] + [[package]] name = "ptyprocess" version = "0.7.0"