gsadmin/meet
1
0
Fork 0
mirror of https://github.com/suitenumerique/meet.git synced 2026-10-20 11:41:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

⚡️(backend) add opt-in psycopg connection pool

Browse Source 
Each request opened and closed its own Postgres connection which added a
big overhead. The goal is to add Django's native psycopg pool behind a new
DB_PSYCOPG_POOL_ENABLED settings with explicit min/max size and keepalive.

Each pool lives in one process, and gunicorn's sync workers serve a single
request at a time which means that a given worker won't use more than one
connection. The default pool minimum size is set to 1, which keeps always
one connection open and the maximum size to 3 as headroom for code touching
the database from an extra thread.

Pooled connections can sit idle for hours, so TCP keepalives are enabled
to keep load balancers and NAT from dropping them silently, and each
connection is health checked on checkout so a broken one is replaced
instead of failing the request.
This commit is contained in:
briquet
2026-10-12 22:52:22 +02:00
parent 5c371e8ad8
commit c920731889
9 changed files with 242 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+1
View File
@@ -29,6 +29,7 @@ and this project adheres to
- ✨(backend) introduce a token exchange endpoint for iframe embeds
- ⚡️(backend) send recording notification emails from a Celery task
- ✨(backend) allow static feature flags without an analytics backend #1809
- ⚡️(backend) add an opt-in psycopg connection pool
### Changed
docs/installation/kubernetes.md
+6 -1
View File
@@ -318,12 +318,17 @@ These are the environmental options available on meet backend.
| DJANGO_SECRET_KEY | Secret key used for Django security | |
| DJANGO_SILENCED_SYSTEM_CHECKS | Silence Django system checks | [] |
| DJANGO_ALLOW_UNSECURE_USER_LISTING | Allow unsecure user listing | false |
| DB_ENGINE | Database engine used | django.db.backends.postgresql_psycopg2 |
| DB_ENGINE | Database engine used | django.db.backends.postgresql |
| DB_NAME | Name of the database | meet |
| DB_USER | User used to connect to database | dinum |
| DB_PASSWORD | Password used to connect to the database | pass |
| DB_HOST | Hostname of the database | localhost |
| DB_PORT | Port to connect to database | 5432 |
| DB_PSYCOPG_POOL_ENABLED | Enable the psycopg connection pool | false |
| DB_PSYCOPG_POOL_MIN_SIZE | Connections each pool keeps open | 1 |
| DB_PSYCOPG_POOL_MAX_SIZE | Maximum connections per pool | 3 |
| DB_PSYCOPG_POOL_TIMEOUT | Seconds a request waits for a free pool connection before failing | 30 |
| DB_PSYCOPG_POOL_MAX_IDLE | Seconds an idle connection above the minimum stays open | 600 |
| STORAGES_STATICFILES_BACKEND | Static file serving engine | whitenoise.storage.CompressedManifestStaticFilesStorage |
| AWS_S3_ENDPOINT_URL | S3 host endpoint | |
| AWS_S3_ACCESS_KEY_ID | S3 access key | |
env.d/development/postgresql.dist
+2 -1
View File
@@ -8,4 +8,5 @@ DB_HOST=postgresql
DB_NAME=meet
DB_USER=dinum
DB_PASSWORD=pass
DB_PORT=5432
DB_PORT=5432
DB_PSYCOPG_POOL_ENABLED=True
src/backend/core/tests/files/test_api_files_create.py
+16 -12
View File
@@ -6,6 +6,7 @@ from concurrent.futures import ThreadPoolExecutor
from urllib.parse import parse_qs, urlparse
from uuid import uuid4
from django.db import connection
from django.utils import timezone
import pytest
@@ -361,18 +362,21 @@ def test_api_files_create_file_race_condition():
"""
def create_item(title):
user = factories.UserFactory()
client = APIClient()
client.force_login(user)
return client.post(
"/api/v1.0/files/",
{
"title": title,
"type": FileTypeChoices.BACKGROUND_IMAGE,
"filename": "my_file.png",
},
format="json",
)
try:
user = factories.UserFactory()
client = APIClient()
client.force_login(user)
return client.post(
"/api/v1.0/files/",
{
"title": title,
"type": FileTypeChoices.BACKGROUND_IMAGE,
"filename": "my_file.png",
},
format="json",
)
finally:
connection.close()
with ThreadPoolExecutor(max_workers=2) as executor:
future1 = executor.submit(create_item, "my item 1")
src/backend/core/tests/files/test_api_files_upload_ended.py
+7 -3
View File
@@ -5,6 +5,7 @@ from concurrent.futures import ThreadPoolExecutor
from io import BytesIO
from django.core.files.storage import default_storage
from django.db import connection
import pytest
from rest_framework.test import APIClient
@@ -295,9 +296,12 @@ def test_api_file_upload_ended_concurrent_calls_are_serialized(settings):
default_storage.save(file.temporary_file_key, BytesIO(b"my prose"))
def call_upload_ended():
client = APIClient()
client.force_login(user)
return client.post(f"/api/v1.0/files/{file.id!s}/upload-ended/")
try:
client = APIClient()
client.force_login(user)
return client.post(f"/api/v1.0/files/{file.id!s}/upload-ended/")
finally:
connection.close()
with ThreadPoolExecutor(max_workers=2) as executor:
futures = [
src/backend/core/tests/test_settings_database_pool.py
+130
View File
@@ -0,0 +1,130 @@
"""Unit tests for the opt-in psycopg connection pool settings."""
# pylint: disable=protected-access
from django.core.exceptions import ImproperlyConfigured
import pytest
from meet import settings
def make_configuration(**overrides):
"""Build a throwaway configuration with plain values and its own DATABASES."""
attributes = {
"DATABASES": {
"default": {
"ENGINE": "django.db.backends.postgresql",
"OPTIONS": {"sslmode": "require"},
}
},
"DB_PSYCOPG_POOL_ENABLED": True,
"DB_PSYCOPG_POOL_MIN_SIZE": 1,
"DB_PSYCOPG_POOL_MAX_SIZE": 3,
"DB_PSYCOPG_POOL_TIMEOUT": 30.0,
"DB_PSYCOPG_POOL_MAX_IDLE": 600.0,
**overrides,
}
return type("PoolConfiguration", (settings.Test,), attributes)
def test_database_pool_disabled_leaves_options_untouched():
"""Without the flag, no connection pool is configured."""
configuration = make_configuration(DB_PSYCOPG_POOL_ENABLED=False)
configuration._configure_database_pool()
assert configuration.DATABASES["default"]["OPTIONS"] == {"sslmode": "require"}
assert "CONN_HEALTH_CHECKS" not in configuration.DATABASES["default"]
def test_database_pool_enabled_merges_pool_options():
"""The pool and keepalive options are added next to the existing ones."""
configuration = make_configuration()
configuration._configure_database_pool()
assert configuration.DATABASES["default"]["OPTIONS"] == {
"sslmode": "require",
"keepalives_idle": 60,
"keepalives_interval": 10,
"keepalives_count": 5,
"pool": {"min_size": 1, "max_size": 3, "timeout": 30.0, "max_idle": 600.0},
}
def test_database_pool_enabled_keeps_explicit_keepalives():
"""Keepalive values already set, e.g. from DATABASE_URL, are not overridden."""
configuration = make_configuration(
DATABASES={
"default": {
"ENGINE": "django.db.backends.postgresql",
"OPTIONS": {"keepalives_idle": "30", "keepalives": "0"},
}
}
)
configuration._configure_database_pool()
options = configuration.DATABASES["default"]["OPTIONS"]
assert options["keepalives_idle"] == "30"
assert options["keepalives"] == "0"
assert options["keepalives_interval"] == 10
@pytest.mark.parametrize("health_checks", [None, False])
def test_database_pool_enabled_forces_health_checks(health_checks):
"""Pooled connections are checked on checkout, even if DATABASE_URL said no."""
database = {"ENGINE": "django.db.backends.postgresql"}
if health_checks is not None:
database["CONN_HEALTH_CHECKS"] = health_checks
configuration = make_configuration(DATABASES={"default": database})
configuration._configure_database_pool()
assert configuration.DATABASES["default"]["CONN_HEALTH_CHECKS"] is True
def test_database_pool_enabled_without_options():
"""A database without OPTIONS, as built from DATABASE_URL, gets them created."""
configuration = make_configuration(
DATABASES={"default": {"ENGINE": "django.db.backends.postgresql_psycopg2"}}
)
configuration._configure_database_pool()
assert configuration.DATABASES["default"]["OPTIONS"]["pool"]["max_size"] == 3
@pytest.mark.parametrize(
"overrides,match",
[
(
{"DATABASES": {"default": {"ENGINE": "django.db.backends.sqlite3"}}},
"requires the django.db.backends.postgresql engine",
),
(
{
"DATABASES": {
"default": {
"ENGINE": "django.db.backends.postgresql",
"CONN_MAX_AGE": 60,
}
}
},
"cannot be combined with persistent connections",
),
(
{"DB_PSYCOPG_POOL_MIN_SIZE": 6},
"MIN_SIZE cannot be greater than",
),
({"DB_PSYCOPG_POOL_TIMEOUT": 0.0}, "must be positive"),
({"DB_PSYCOPG_POOL_MAX_IDLE": -1.0}, "must be positive"),
],
)
def test_database_pool_invalid_configuration(overrides, match):
"""An invalid pool setup fails at boot."""
configuration = make_configuration(**overrides)
with pytest.raises(ImproperlyConfigured, match=match):
configuration._configure_database_pool()
src/backend/meet/settings.py
+62 -1
View File
@@ -17,6 +17,7 @@ import warnings
from os import path
from socket import gethostbyname, gethostname
from django.core.exceptions import ImproperlyConfigured
from django.utils.translation import gettext_lazy as _
import dj_database_url
@@ -139,7 +140,7 @@ class Base(Configuration):
)
else {
"ENGINE": values.Value(
"django.db.backends.postgresql_psycopg2",
"django.db.backends.postgresql",
environ_name="DB_ENGINE",
environ_prefix=None,
),
@@ -154,6 +155,21 @@ class Base(Configuration):
"PORT": values.Value(5432, environ_name="DB_PORT", environ_prefix=None),
}
}
DB_PSYCOPG_POOL_ENABLED = values.BooleanValue(
False, environ_name="DB_PSYCOPG_POOL_ENABLED", environ_prefix=None
)
DB_PSYCOPG_POOL_MIN_SIZE = values.PositiveIntegerValue(
1, environ_name="DB_PSYCOPG_POOL_MIN_SIZE", environ_prefix=None
)
DB_PSYCOPG_POOL_MAX_SIZE = values.PositiveIntegerValue(
3, environ_name="DB_PSYCOPG_POOL_MAX_SIZE", environ_prefix=None
)
DB_PSYCOPG_POOL_TIMEOUT = values.FloatValue(
30.0, environ_name="DB_PSYCOPG_POOL_TIMEOUT", environ_prefix=None
)
DB_PSYCOPG_POOL_MAX_IDLE = values.FloatValue(
600.0, environ_name="DB_PSYCOPG_POOL_MAX_IDLE", environ_prefix=None
)
DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
# Static files (CSS, JavaScript, Images)
@@ -1540,6 +1556,50 @@ class Base(Configuration):
f"'{cls.AUDIT_LOG_DATA_STREAM_NAMESPACE}' must not contain '-'."
)
@classmethod
def _configure_database_pool(cls):
"""Inject the psycopg pool options into the default database when enabled."""
if not cls.DB_PSYCOPG_POOL_ENABLED:
return
database = cls.DATABASES["default"]
if database.get("ENGINE") not in (
"django.db.backends.postgresql",
"django.db.backends.postgresql_psycopg2",
):
raise ImproperlyConfigured(
"DB_PSYCOPG_POOL_ENABLED requires the django.db.backends.postgresql "
f"engine, got {database.get('ENGINE')!r}."
)
if database.get("CONN_MAX_AGE"):
raise ImproperlyConfigured(
"DB_PSYCOPG_POOL_ENABLED cannot be combined with persistent "
"connections; set CONN_MAX_AGE to 0."
)
if cls.DB_PSYCOPG_POOL_MIN_SIZE > cls.DB_PSYCOPG_POOL_MAX_SIZE:
raise ImproperlyConfigured(
"DB_PSYCOPG_POOL_MIN_SIZE cannot be greater than "
"DB_PSYCOPG_POOL_MAX_SIZE."
)
if cls.DB_PSYCOPG_POOL_TIMEOUT <= 0 or cls.DB_PSYCOPG_POOL_MAX_IDLE <= 0:
raise ImproperlyConfigured(
"DB_PSYCOPG_POOL_TIMEOUT and DB_PSYCOPG_POOL_MAX_IDLE must be positive."
)
database["OPTIONS"] = {
"keepalives_idle": 60,
"keepalives_interval": 10,
"keepalives_count": 5,
**database.get("OPTIONS", {}),
"pool": {
"min_size": cls.DB_PSYCOPG_POOL_MIN_SIZE,
"max_size": cls.DB_PSYCOPG_POOL_MAX_SIZE,
"timeout": cls.DB_PSYCOPG_POOL_TIMEOUT,
"max_idle": cls.DB_PSYCOPG_POOL_MAX_IDLE,
},
}
database["CONN_HEALTH_CHECKS"] = True
@classmethod
def post_setup(cls):
"""Post setup configuration.
@@ -1555,6 +1615,7 @@ class Base(Configuration):
cls._check_recording_encoding_maps()
cls._check_audit_log_data_stream_namespace()
cls._configure_database_pool()
if (
cls.SUMMARY_SERVICE_VERSION == 1
src/backend/pyproject.toml
+1 -1
View File
@@ -52,7 +52,7 @@ dependencies = [
"menshen-client==0.2.0",
"nested-multipart-parser==1.6.0",
"posthog==7.44.0",
"psycopg[binary]==3.3.4",
"psycopg[binary,pool]==3.3.4",
"pydantic==2.13.4",
"PyJWT==2.14.0",
"python-frontmatter==1.3.0",
src/backend/uv.lock
Generated
+17 -2
View File
@@ -1332,7 +1332,7 @@ dependencies = [
{ name = "nested-multipart-parser" },
{ name = "phonenumbers" },
{ name = "posthog" },
{ name = "psycopg", extra = ["binary"] },
{ name = "psycopg", extra = ["binary", "pool"] },
{ name = "pydantic" },
{ name = "pyjwt" },
{ name = "python-frontmatter" },
@@ -1398,7 +1398,7 @@ requires-dist = [
{ name = "nested-multipart-parser", specifier = "==1.6.0" },
{ name = "phonenumbers", specifier = "==9.0.37" },
{ name = "posthog", specifier = "==7.44.0" },
{ name = "psycopg", extras = ["binary"], specifier = "==3.3.4" },
{ name = "psycopg", extras = ["binary", "pool"], specifier = "==3.3.4" },
{ name = "pydantic", specifier = "==2.13.4" },
{ name = "pyjwt", specifier = "==2.14.0" },
{ name = "python-frontmatter", specifier = "==1.3.0" },
@@ -1889,6 +1889,9 @@ wheels = [
binary = [
{ name = "psycopg-binary", marker = "implementation_name != 'pypy'" },
]
pool = [
{ name = "psycopg-pool" },
]
[[package]]
name = "psycopg-binary"
@@ -1919,6 +1922,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/eb/e6/5fff07a70d1f945ed90ae131c3bd76cab32beff7c58c6db15ad5820b6d1f/psycopg_binary-3.3.4-cp314-cp314-win_amd64.whl", hash = "sha256:c37e024c07308cd06cf3ec51bfd0e7f6157585a4d84d1bce4a7f5f7913719bf8", size = 3666849, upload-time = "2026-05-01T23:31:51.165Z" },
]
[[package]]
name = "psycopg-pool"
version = "3.3.3"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "typing-extensions" },
]
sdist = { url = "https://files.pythonhosted.org/packages/74/5e/c0664b968b102ff68b811d999c728546c48d5c1eec03e3bbaf88c0cb4472/psycopg_pool-3.3.3.tar.gz", hash = "sha256:df87b5d9d0ad7db37f6cdad4fa8ce113d250f5997f6db38e9a99192fb67f9e1d", size = 32006, upload-time = "2026-09-22T15:53:24.947Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/5d/b4/452c6607a0f479465cd8a9b0d9956919fcb150050c1f83f9f11e6b8ee8dc/psycopg_pool-3.3.3-py3-none-any.whl", hash = "sha256:9b9cd6a4fcec47a410f7e82d408540e7f77b478509e91b44c1a5457a13e5ff37", size = 40304, upload-time = "2026-09-22T15:53:23.712Z" },
]
[[package]]
name = "ptyprocess"
version = "0.7.0"