mirror of
https://github.com/suitenumerique/meet.git
synced 2026-10-15 01:13:43 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 briquet
|
e3b6b7e213 |
@@ -20,6 +20,7 @@ and this project adheres to
|
||||
- ✨(backend) introduce a token exchange endpoint for iframe embeds
|
||||
- ⚡️(backend) send recording notification emails from a Celery task
|
||||
- ✨(backend) allow static feature flags without an analytics backend #1809
|
||||
- ⚡️(backend) add an opt-in psycopg connection pool
|
||||
|
||||
### Changed
|
||||
|
||||
|
||||
@@ -318,12 +318,17 @@ These are the environmental options available on meet backend.
|
||||
| DJANGO_SECRET_KEY | Secret key used for Django security | |
|
||||
| DJANGO_SILENCED_SYSTEM_CHECKS | Silence Django system checks | [] |
|
||||
| DJANGO_ALLOW_UNSECURE_USER_LISTING | Allow unsecure user listing | false |
|
||||
| DB_ENGINE | Database engine used | django.db.backends.postgresql_psycopg2 |
|
||||
| DB_ENGINE | Database engine used | django.db.backends.postgresql |
|
||||
| DB_NAME | Name of the database | meet |
|
||||
| DB_USER | User used to connect to database | dinum |
|
||||
| DB_PASSWORD | Password used to connect to the database | pass |
|
||||
| DB_HOST | Hostname of the database | localhost |
|
||||
| DB_PORT | Port to connect to database | 5432 |
|
||||
| DB_PSYCOPG_POOL_ENABLED | Enable the psycopg connection pool | false |
|
||||
| DB_PSYCOPG_POOL_MIN_SIZE | Connections each pool keeps open | 1 |
|
||||
| DB_PSYCOPG_POOL_MAX_SIZE | Maximum connections per pool | 3 |
|
||||
| DB_PSYCOPG_POOL_TIMEOUT | Seconds a request waits for a free pool connection before failing | 30 |
|
||||
| DB_PSYCOPG_POOL_MAX_IDLE | Seconds an idle connection above the minimum stays open | 600 |
|
||||
| STORAGES_STATICFILES_BACKEND | Static file serving engine | whitenoise.storage.CompressedManifestStaticFilesStorage |
|
||||
| AWS_S3_ENDPOINT_URL | S3 host endpoint | |
|
||||
| AWS_S3_ACCESS_KEY_ID | S3 access key | |
|
||||
|
||||
@@ -8,4 +8,5 @@ DB_HOST=postgresql
|
||||
DB_NAME=meet
|
||||
DB_USER=dinum
|
||||
DB_PASSWORD=pass
|
||||
DB_PORT=5432
|
||||
DB_PORT=5432
|
||||
DB_PSYCOPG_POOL_ENABLED=True
|
||||
|
||||
@@ -6,6 +6,7 @@ from concurrent.futures import ThreadPoolExecutor
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
from uuid import uuid4
|
||||
|
||||
from django.db import connection
|
||||
from django.utils import timezone
|
||||
|
||||
import pytest
|
||||
@@ -361,18 +362,21 @@ def test_api_files_create_file_race_condition():
|
||||
"""
|
||||
|
||||
def create_item(title):
|
||||
user = factories.UserFactory()
|
||||
client = APIClient()
|
||||
client.force_login(user)
|
||||
return client.post(
|
||||
"/api/v1.0/files/",
|
||||
{
|
||||
"title": title,
|
||||
"type": FileTypeChoices.BACKGROUND_IMAGE,
|
||||
"filename": "my_file.png",
|
||||
},
|
||||
format="json",
|
||||
)
|
||||
try:
|
||||
user = factories.UserFactory()
|
||||
client = APIClient()
|
||||
client.force_login(user)
|
||||
return client.post(
|
||||
"/api/v1.0/files/",
|
||||
{
|
||||
"title": title,
|
||||
"type": FileTypeChoices.BACKGROUND_IMAGE,
|
||||
"filename": "my_file.png",
|
||||
},
|
||||
format="json",
|
||||
)
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
future1 = executor.submit(create_item, "my item 1")
|
||||
|
||||
@@ -5,6 +5,7 @@ from concurrent.futures import ThreadPoolExecutor
|
||||
from io import BytesIO
|
||||
|
||||
from django.core.files.storage import default_storage
|
||||
from django.db import connection
|
||||
|
||||
import pytest
|
||||
from rest_framework.test import APIClient
|
||||
@@ -295,9 +296,12 @@ def test_api_file_upload_ended_concurrent_calls_are_serialized(settings):
|
||||
default_storage.save(file.temporary_file_key, BytesIO(b"my prose"))
|
||||
|
||||
def call_upload_ended():
|
||||
client = APIClient()
|
||||
client.force_login(user)
|
||||
return client.post(f"/api/v1.0/files/{file.id!s}/upload-ended/")
|
||||
try:
|
||||
client = APIClient()
|
||||
client.force_login(user)
|
||||
return client.post(f"/api/v1.0/files/{file.id!s}/upload-ended/")
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
futures = [
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
"""Unit tests for the opt-in psycopg connection pool settings."""
|
||||
|
||||
# pylint: disable=protected-access
|
||||
|
||||
from django.core.exceptions import ImproperlyConfigured
|
||||
|
||||
import pytest
|
||||
|
||||
from meet import settings
|
||||
|
||||
|
||||
def make_configuration(**overrides):
|
||||
"""Build a throwaway configuration with plain values and its own DATABASES."""
|
||||
attributes = {
|
||||
"DATABASES": {
|
||||
"default": {
|
||||
"ENGINE": "django.db.backends.postgresql",
|
||||
"OPTIONS": {"sslmode": "require"},
|
||||
}
|
||||
},
|
||||
"DB_PSYCOPG_POOL_ENABLED": True,
|
||||
"DB_PSYCOPG_POOL_MIN_SIZE": 1,
|
||||
"DB_PSYCOPG_POOL_MAX_SIZE": 3,
|
||||
"DB_PSYCOPG_POOL_TIMEOUT": 30.0,
|
||||
"DB_PSYCOPG_POOL_MAX_IDLE": 600.0,
|
||||
**overrides,
|
||||
}
|
||||
return type("PoolConfiguration", (settings.Test,), attributes)
|
||||
|
||||
|
||||
def test_database_pool_disabled_leaves_options_untouched():
|
||||
"""Without the flag, no connection pool is configured."""
|
||||
configuration = make_configuration(DB_PSYCOPG_POOL_ENABLED=False)
|
||||
|
||||
configuration._configure_database_pool()
|
||||
|
||||
assert configuration.DATABASES["default"]["OPTIONS"] == {"sslmode": "require"}
|
||||
assert "CONN_HEALTH_CHECKS" not in configuration.DATABASES["default"]
|
||||
|
||||
|
||||
def test_database_pool_enabled_merges_pool_options():
|
||||
"""The pool and keepalive options are added next to the existing ones."""
|
||||
configuration = make_configuration()
|
||||
|
||||
configuration._configure_database_pool()
|
||||
|
||||
assert configuration.DATABASES["default"]["OPTIONS"] == {
|
||||
"sslmode": "require",
|
||||
"keepalives_idle": 60,
|
||||
"keepalives_interval": 10,
|
||||
"keepalives_count": 5,
|
||||
"pool": {"min_size": 1, "max_size": 3, "timeout": 30.0, "max_idle": 600.0},
|
||||
}
|
||||
|
||||
|
||||
def test_database_pool_enabled_keeps_explicit_keepalives():
|
||||
"""Keepalive values already set, e.g. from DATABASE_URL, are not overridden."""
|
||||
configuration = make_configuration(
|
||||
DATABASES={
|
||||
"default": {
|
||||
"ENGINE": "django.db.backends.postgresql",
|
||||
"OPTIONS": {"keepalives_idle": "30", "keepalives": "0"},
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
configuration._configure_database_pool()
|
||||
|
||||
options = configuration.DATABASES["default"]["OPTIONS"]
|
||||
assert options["keepalives_idle"] == "30"
|
||||
assert options["keepalives"] == "0"
|
||||
assert options["keepalives_interval"] == 10
|
||||
|
||||
|
||||
@pytest.mark.parametrize("health_checks", [None, False])
|
||||
def test_database_pool_enabled_forces_health_checks(health_checks):
|
||||
"""Pooled connections are checked on checkout, even if DATABASE_URL said no."""
|
||||
database = {"ENGINE": "django.db.backends.postgresql"}
|
||||
if health_checks is not None:
|
||||
database["CONN_HEALTH_CHECKS"] = health_checks
|
||||
configuration = make_configuration(DATABASES={"default": database})
|
||||
|
||||
configuration._configure_database_pool()
|
||||
|
||||
assert configuration.DATABASES["default"]["CONN_HEALTH_CHECKS"] is True
|
||||
|
||||
|
||||
def test_database_pool_enabled_without_options():
|
||||
"""A database without OPTIONS, as built from DATABASE_URL, gets them created."""
|
||||
configuration = make_configuration(
|
||||
DATABASES={"default": {"ENGINE": "django.db.backends.postgresql_psycopg2"}}
|
||||
)
|
||||
|
||||
configuration._configure_database_pool()
|
||||
|
||||
assert configuration.DATABASES["default"]["OPTIONS"]["pool"]["max_size"] == 3
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"overrides,match",
|
||||
[
|
||||
(
|
||||
{"DATABASES": {"default": {"ENGINE": "django.db.backends.sqlite3"}}},
|
||||
"requires the django.db.backends.postgresql engine",
|
||||
),
|
||||
(
|
||||
{
|
||||
"DATABASES": {
|
||||
"default": {
|
||||
"ENGINE": "django.db.backends.postgresql",
|
||||
"CONN_MAX_AGE": 60,
|
||||
}
|
||||
}
|
||||
},
|
||||
"cannot be combined with persistent connections",
|
||||
),
|
||||
(
|
||||
{"DB_PSYCOPG_POOL_MIN_SIZE": 6},
|
||||
"MIN_SIZE cannot be greater than",
|
||||
),
|
||||
({"DB_PSYCOPG_POOL_TIMEOUT": 0.0}, "must be positive"),
|
||||
({"DB_PSYCOPG_POOL_MAX_IDLE": -1.0}, "must be positive"),
|
||||
],
|
||||
)
|
||||
def test_database_pool_invalid_configuration(overrides, match):
|
||||
"""An invalid pool setup fails at boot."""
|
||||
configuration = make_configuration(**overrides)
|
||||
|
||||
with pytest.raises(ImproperlyConfigured, match=match):
|
||||
configuration._configure_database_pool()
|
||||
@@ -17,6 +17,7 @@ import warnings
|
||||
from os import path
|
||||
from socket import gethostbyname, gethostname
|
||||
|
||||
from django.core.exceptions import ImproperlyConfigured
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
|
||||
import dj_database_url
|
||||
@@ -139,7 +140,7 @@ class Base(Configuration):
|
||||
)
|
||||
else {
|
||||
"ENGINE": values.Value(
|
||||
"django.db.backends.postgresql_psycopg2",
|
||||
"django.db.backends.postgresql",
|
||||
environ_name="DB_ENGINE",
|
||||
environ_prefix=None,
|
||||
),
|
||||
@@ -154,6 +155,21 @@ class Base(Configuration):
|
||||
"PORT": values.Value(5432, environ_name="DB_PORT", environ_prefix=None),
|
||||
}
|
||||
}
|
||||
DB_PSYCOPG_POOL_ENABLED = values.BooleanValue(
|
||||
False, environ_name="DB_PSYCOPG_POOL_ENABLED", environ_prefix=None
|
||||
)
|
||||
DB_PSYCOPG_POOL_MIN_SIZE = values.PositiveIntegerValue(
|
||||
1, environ_name="DB_PSYCOPG_POOL_MIN_SIZE", environ_prefix=None
|
||||
)
|
||||
DB_PSYCOPG_POOL_MAX_SIZE = values.PositiveIntegerValue(
|
||||
3, environ_name="DB_PSYCOPG_POOL_MAX_SIZE", environ_prefix=None
|
||||
)
|
||||
DB_PSYCOPG_POOL_TIMEOUT = values.FloatValue(
|
||||
30.0, environ_name="DB_PSYCOPG_POOL_TIMEOUT", environ_prefix=None
|
||||
)
|
||||
DB_PSYCOPG_POOL_MAX_IDLE = values.FloatValue(
|
||||
600.0, environ_name="DB_PSYCOPG_POOL_MAX_IDLE", environ_prefix=None
|
||||
)
|
||||
DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
|
||||
|
||||
# Static files (CSS, JavaScript, Images)
|
||||
@@ -1446,6 +1462,52 @@ class Base(Configuration):
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _configure_database_pool(cls):
|
||||
"""Inject the psycopg pool options into the default database when enabled."""
|
||||
if not cls.DB_PSYCOPG_POOL_ENABLED:
|
||||
return
|
||||
|
||||
database = cls.DATABASES["default"]
|
||||
if database.get("ENGINE") not in (
|
||||
"django.db.backends.postgresql",
|
||||
"django.db.backends.postgresql_psycopg2",
|
||||
):
|
||||
raise ImproperlyConfigured(
|
||||
"DB_PSYCOPG_POOL_ENABLED requires the django.db.backends.postgresql "
|
||||
f"engine, got {database.get('ENGINE')!r}."
|
||||
)
|
||||
if database.get("CONN_MAX_AGE"):
|
||||
raise ImproperlyConfigured(
|
||||
"DB_PSYCOPG_POOL_ENABLED cannot be combined with persistent "
|
||||
"connections; set CONN_MAX_AGE to 0."
|
||||
)
|
||||
if cls.DB_PSYCOPG_POOL_MIN_SIZE > cls.DB_PSYCOPG_POOL_MAX_SIZE:
|
||||
raise ImproperlyConfigured(
|
||||
"DB_PSYCOPG_POOL_MIN_SIZE cannot be greater than "
|
||||
"DB_PSYCOPG_POOL_MAX_SIZE."
|
||||
)
|
||||
if cls.DB_PSYCOPG_POOL_TIMEOUT <= 0 or cls.DB_PSYCOPG_POOL_MAX_IDLE <= 0:
|
||||
raise ImproperlyConfigured(
|
||||
"DB_PSYCOPG_POOL_TIMEOUT and DB_PSYCOPG_POOL_MAX_IDLE must be positive."
|
||||
)
|
||||
|
||||
database["OPTIONS"] = {
|
||||
"keepalives_idle": 60,
|
||||
"keepalives_interval": 10,
|
||||
"keepalives_count": 5,
|
||||
**database.get("OPTIONS", {}),
|
||||
"pool": {
|
||||
"min_size": cls.DB_PSYCOPG_POOL_MIN_SIZE,
|
||||
"max_size": cls.DB_PSYCOPG_POOL_MAX_SIZE,
|
||||
"timeout": cls.DB_PSYCOPG_POOL_TIMEOUT,
|
||||
"max_idle": cls.DB_PSYCOPG_POOL_MAX_IDLE,
|
||||
},
|
||||
}
|
||||
# The pool does not watch idle connections: check each one on checkout
|
||||
# so a broken connection is replaced instead of failing the request.
|
||||
database["CONN_HEALTH_CHECKS"] = True
|
||||
|
||||
@classmethod
|
||||
def post_setup(cls):
|
||||
"""Post setup configuration.
|
||||
@@ -1460,6 +1522,7 @@ class Base(Configuration):
|
||||
)
|
||||
|
||||
cls._check_recording_encoding_maps()
|
||||
cls._configure_database_pool()
|
||||
|
||||
if (
|
||||
cls.SUMMARY_SERVICE_VERSION == 1
|
||||
|
||||
@@ -51,7 +51,7 @@ dependencies = [
|
||||
"markdown==3.10.3",
|
||||
"nested-multipart-parser==1.6.0",
|
||||
"posthog==7.44.0",
|
||||
"psycopg[binary]==3.3.4",
|
||||
"psycopg[binary,pool]==3.3.4",
|
||||
"pydantic==2.13.4",
|
||||
"PyJWT==2.14.0",
|
||||
"python-frontmatter==1.3.0",
|
||||
|
||||
Generated
+17
-2
@@ -1331,7 +1331,7 @@ dependencies = [
|
||||
{ name = "nested-multipart-parser" },
|
||||
{ name = "phonenumbers" },
|
||||
{ name = "posthog" },
|
||||
{ name = "psycopg", extra = ["binary"] },
|
||||
{ name = "psycopg", extra = ["binary", "pool"] },
|
||||
{ name = "pydantic" },
|
||||
{ name = "pyjwt" },
|
||||
{ name = "python-frontmatter" },
|
||||
@@ -1396,7 +1396,7 @@ requires-dist = [
|
||||
{ name = "nested-multipart-parser", specifier = "==1.6.0" },
|
||||
{ name = "phonenumbers", specifier = "==9.0.37" },
|
||||
{ name = "posthog", specifier = "==7.44.0" },
|
||||
{ name = "psycopg", extras = ["binary"], specifier = "==3.3.4" },
|
||||
{ name = "psycopg", extras = ["binary", "pool"], specifier = "==3.3.4" },
|
||||
{ name = "pydantic", specifier = "==2.13.4" },
|
||||
{ name = "pyjwt", specifier = "==2.14.0" },
|
||||
{ name = "python-frontmatter", specifier = "==1.3.0" },
|
||||
@@ -1875,6 +1875,9 @@ wheels = [
|
||||
binary = [
|
||||
{ name = "psycopg-binary", marker = "implementation_name != 'pypy'" },
|
||||
]
|
||||
pool = [
|
||||
{ name = "psycopg-pool" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "psycopg-binary"
|
||||
@@ -1905,6 +1908,18 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/eb/e6/5fff07a70d1f945ed90ae131c3bd76cab32beff7c58c6db15ad5820b6d1f/psycopg_binary-3.3.4-cp314-cp314-win_amd64.whl", hash = "sha256:c37e024c07308cd06cf3ec51bfd0e7f6157585a4d84d1bce4a7f5f7913719bf8", size = 3666849, upload-time = "2026-05-01T23:31:51.165Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "psycopg-pool"
|
||||
version = "3.3.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/74/5e/c0664b968b102ff68b811d999c728546c48d5c1eec03e3bbaf88c0cb4472/psycopg_pool-3.3.3.tar.gz", hash = "sha256:df87b5d9d0ad7db37f6cdad4fa8ce113d250f5997f6db38e9a99192fb67f9e1d", size = 32006, upload-time = "2026-09-22T15:53:24.947Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/5d/b4/452c6607a0f479465cd8a9b0d9956919fcb150050c1f83f9f11e6b8ee8dc/psycopg_pool-3.3.3-py3-none-any.whl", hash = "sha256:9b9cd6a4fcec47a410f7e82d408540e7f77b478509e91b44c1a5457a13e5ff37", size = 40304, upload-time = "2026-09-22T15:53:23.712Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ptyprocess"
|
||||
version = "0.7.0"
|
||||
|
||||
Reference in New Issue
Block a user