mirror of
https://github.com/suitenumerique/meet.git
synced 2026-11-16 04:52:05 +00:00
Compare commits
10 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| b0e2972406 | |||
| fb0b4d4be1 | |||
| 851b39a33e | |||
| 7b451b67ce | |||
| 375e89f032 | |||
| 0cdfd783bf | |||
| 7e259d7e44 | |||
| 5344bcce2a | |||
| 59d8541261 | |||
| 536f9e9d2f |
@@ -444,6 +444,7 @@ class RecordingAdmin(admin.ModelAdmin):
|
||||
"status",
|
||||
"updated_at",
|
||||
"worker_id",
|
||||
"livekit_cluster",
|
||||
)
|
||||
actions = [resend_notification, mark_as_failed_to_stop]
|
||||
|
||||
|
||||
@@ -8,11 +8,16 @@ from rest_framework import views as drf_views
|
||||
from rest_framework.decorators import api_view
|
||||
from rest_framework.response import Response
|
||||
|
||||
from core.livekit.clusters import get_default_livekit_cluster
|
||||
from core.livekit.exceptions import LiveKitRoomUnavailableError
|
||||
from core.utils import build_telephony_config
|
||||
|
||||
from .exceptions import LiveKitRoomUnavailable
|
||||
|
||||
|
||||
def exception_handler(exc, context):
|
||||
"""Handle Django ValidationError as an accepted exception.
|
||||
"""Handle Django ValidationError as an accepted exception, and answer a
|
||||
LiveKit room that cannot be joined for now with a 503.
|
||||
|
||||
For the parameters, see ``exception_handler``
|
||||
This code comes from twidi's gist:
|
||||
@@ -30,6 +35,9 @@ def exception_handler(exc, context):
|
||||
|
||||
exc = drf_exceptions.ValidationError(detail=detail)
|
||||
|
||||
elif isinstance(exc, LiveKitRoomUnavailableError):
|
||||
exc = LiveKitRoomUnavailable()
|
||||
|
||||
return drf_views.exception_handler(exc, context)
|
||||
|
||||
|
||||
@@ -81,7 +89,7 @@ def get_frontend_configuration(request):
|
||||
"subtitle": {"enabled": settings.ROOM_SUBTITLE_ENABLED},
|
||||
"diagnostics": {"connection_test_enabled": settings.CONNECTION_TEST_ENABLED},
|
||||
"livekit": {
|
||||
"url": settings.LIVEKIT_CONFIGURATION["url"],
|
||||
"url": get_default_livekit_cluster().url,
|
||||
"force_wss_protocol": settings.LIVEKIT_FORCE_WSS_PROTOCOL,
|
||||
"enable_firefox_proxy_workaround": settings.LIVEKIT_ENABLE_FIREFOX_PROXY_WORKAROUND,
|
||||
"default_sources": settings.LIVEKIT_DEFAULT_SOURCES,
|
||||
|
||||
@@ -15,6 +15,15 @@ class RoomSoftDeleted(exceptions.APIException):
|
||||
default_code = "room_deleted"
|
||||
|
||||
|
||||
class LiveKitRoomUnavailable(exceptions.APIException):
|
||||
"""Raised when the LiveKit room to hand out credentials for cannot be
|
||||
joined for now: it cannot be placed, or created on its LiveKit cluster."""
|
||||
|
||||
status_code = status.HTTP_503_SERVICE_UNAVAILABLE
|
||||
default_detail = _("This room cannot be joined for now, please try again.")
|
||||
default_code = "livekit_room_unavailable"
|
||||
|
||||
|
||||
def ensure_room_not_deleted(resource):
|
||||
"""Raise a 410 Gone if the resource is a soft-deleted room.
|
||||
|
||||
|
||||
@@ -143,6 +143,18 @@ class ConnectionTestAnonRateThrottle(MonitoredAnonRateThrottle):
|
||||
scope = "connection_test"
|
||||
|
||||
|
||||
class UnregisteredRoomPlacementRateThrottle(MonitoredUserRateThrottle):
|
||||
"""Throttle the joins placing unregistered rooms.
|
||||
|
||||
Counted per authenticated user, and per IP address for anonymous callers.
|
||||
Such a join creates a placement and a LiveKit room for any slug it is
|
||||
given. The view only counts the joins of unregistered rooms that have no
|
||||
placement yet, so joining a room already placed is never throttled.
|
||||
"""
|
||||
|
||||
scope = "unregistered_room_placement"
|
||||
|
||||
|
||||
class ExchangeAccessTokenAnonRateThrottle(MonitoredAnonRateThrottle):
|
||||
"""Throttle anonymous transit code exchange attempts.
|
||||
|
||||
|
||||
@@ -46,6 +46,7 @@ from core.api.exceptions import ensure_room_not_deleted
|
||||
from core.api.filters import ListFileFilter
|
||||
from core.api.locks import room_locked_for_update
|
||||
from core.enums import MEDIA_STORAGE_URL_PATTERN
|
||||
from core.livekit.placements import PlacementService
|
||||
from core.recording.enums import FileExtension
|
||||
from core.recording.event.authentication import RecordingProcessWebhookAuthentication
|
||||
from core.recording.services.metadata_collector import (
|
||||
@@ -88,7 +89,6 @@ from core.services.subtitle import SubtitleException, SubtitleService
|
||||
from core.services.transit_code import TransitCodeService
|
||||
from core.tasks.connection_test import delete_connection_test_room
|
||||
from core.tasks.file import process_file_deletion
|
||||
from core.utils import generate_token
|
||||
|
||||
from ..authentication.livekit import LiveKitTokenAuthentication
|
||||
from ..models import RoomAccessLevel
|
||||
@@ -310,15 +310,38 @@ class RoomViewSet( # pylint: disable=too-many-public-methods
|
||||
return obj
|
||||
|
||||
def _get_object_or_unregistered_slug(self):
|
||||
"""Return the requested room, or the slug of an unregistered room."""
|
||||
"""Return the requested room, or the slug of an unregistered room.
|
||||
|
||||
A slug no room could have, empty, a UUID or longer than a room's slug
|
||||
may be, is no unregistered room either. Nor is one starting like the
|
||||
names of connection test rooms, which never get a placement.
|
||||
"""
|
||||
try:
|
||||
return self.get_object(), None
|
||||
except Http404:
|
||||
slug = slugify(self.kwargs["pk"])
|
||||
if not settings.ALLOW_UNREGISTERED_ROOMS or not slug or utils.is_uuid(slug):
|
||||
if (
|
||||
not settings.ALLOW_UNREGISTERED_ROOMS
|
||||
or not slug
|
||||
or utils.is_uuid(slug)
|
||||
or len(slug) > models.Room._meta.get_field("slug").max_length # noqa: SLF001
|
||||
or slug.startswith(settings.CONNECTION_TEST_ROOM_PREFIX)
|
||||
):
|
||||
raise
|
||||
return None, slug
|
||||
|
||||
def _throttle_unregistered_room_placement(self, request, slug):
|
||||
"""Count a join about to place an unregistered room, or refuse it.
|
||||
|
||||
Joins of an unregistered room that is placed already are not counted:
|
||||
its participants may ask for credentials again as often as needed.
|
||||
"""
|
||||
if models.Placement.objects.filter(room_name=slug).exists():
|
||||
return
|
||||
throttle = throttling.UnregisteredRoomPlacementRateThrottle()
|
||||
if not throttle.allow_request(request, self):
|
||||
self.throttled(request, throttle.wait())
|
||||
|
||||
@staticmethod
|
||||
def _get_unregistered_room_data(slug):
|
||||
"""Describe an unregistered room, which only exists by its slug."""
|
||||
@@ -486,6 +509,7 @@ class RoomViewSet( # pylint: disable=too-many-public-methods
|
||||
room=room,
|
||||
mode=mode,
|
||||
options=options_data,
|
||||
livekit_cluster=PlacementService.cluster_for(str(room.id)).name,
|
||||
)
|
||||
recording.grant_initial_accesses(starter=self.request.user)
|
||||
|
||||
@@ -503,7 +527,9 @@ class RoomViewSet( # pylint: disable=too-many-public-methods
|
||||
status=drf_status.HTTP_409_CONFLICT,
|
||||
)
|
||||
|
||||
worker_service = get_worker_service(mode=recording.mode)
|
||||
worker_service = get_worker_service(
|
||||
mode=recording.mode, cluster=recording.get_livekit_cluster()
|
||||
)
|
||||
worker_manager = WorkerServiceMediator(worker_service=worker_service)
|
||||
|
||||
try:
|
||||
@@ -557,7 +583,9 @@ class RoomViewSet( # pylint: disable=too-many-public-methods
|
||||
|
||||
self.audit_target = recording
|
||||
|
||||
worker_service = get_worker_service(mode=recording.mode)
|
||||
worker_service = get_worker_service(
|
||||
mode=recording.mode, cluster=recording.get_livekit_cluster()
|
||||
)
|
||||
worker_manager = WorkerServiceMediator(worker_service=worker_service)
|
||||
|
||||
try:
|
||||
@@ -587,7 +615,12 @@ class RoomViewSet( # pylint: disable=too-many-public-methods
|
||||
going through the lobby. Otherwise, it has no livekit object and the
|
||||
client should request entry through the lobby.
|
||||
|
||||
Unregistered rooms, when allowed, are always joined directly.
|
||||
Unregistered rooms, when allowed, are always joined directly. The joins
|
||||
that place one are throttled, and answered with a 429 over the rate.
|
||||
|
||||
The credentials are for the LiveKit cluster the LiveKit room is placed
|
||||
on, where it is created first. When it cannot be, the response is a
|
||||
503, and asking again later may succeed.
|
||||
"""
|
||||
|
||||
serializer = serializers.JoinRoomSerializer(data=request.data)
|
||||
@@ -596,11 +629,16 @@ class RoomViewSet( # pylint: disable=too-many-public-methods
|
||||
|
||||
room, slug = self._get_object_or_unregistered_slug()
|
||||
if room is None:
|
||||
self._throttle_unregistered_room_placement(request, slug)
|
||||
cluster = PlacementService.place(slug)
|
||||
return drf_response.Response(
|
||||
{
|
||||
**self._get_unregistered_room_data(slug),
|
||||
"livekit": utils.generate_livekit_config(
|
||||
room_id=slug, user=request.user, username=username
|
||||
room_id=slug,
|
||||
user=request.user,
|
||||
username=username,
|
||||
cluster=cluster,
|
||||
),
|
||||
}
|
||||
)
|
||||
@@ -611,12 +649,14 @@ class RoomViewSet( # pylint: disable=too-many-public-methods
|
||||
).data
|
||||
|
||||
if LobbyService.can_bypass_lobby(room=room, user=request.user, role=role):
|
||||
cluster = PlacementService.place(str(room.id), room)
|
||||
data["livekit"] = utils.generate_livekit_config(
|
||||
room_id=str(room.id),
|
||||
user=request.user,
|
||||
username=username,
|
||||
configuration=data["configuration"],
|
||||
role=role,
|
||||
cluster=cluster,
|
||||
)
|
||||
|
||||
return drf_response.Response(data)
|
||||
@@ -1727,9 +1767,15 @@ class DiagnosticsViewSet(viewsets.ViewSet):
|
||||
Going through the room API is not an option here: it is tied to
|
||||
registered meetings, lobby rules and longer-lived tokens. Each call
|
||||
gets its own room so two people testing at the same time never meet.
|
||||
|
||||
The room is created on the LiveKit cluster the placement strategy
|
||||
picks, but has no placement: no one else ever joins it. When it cannot
|
||||
be created, the response is a 503.
|
||||
"""
|
||||
room = f"{settings.CONNECTION_TEST_ROOM_PREFIX}-{uuid4()}"
|
||||
expires_in = settings.CONNECTION_TEST_TOKEN_TTL_SECONDS
|
||||
cluster = PlacementService.choose_cluster(room)
|
||||
PlacementService.create_livekit_room(room, cluster)
|
||||
|
||||
# LiveKit refreshes tokens for connected clients, so JWT TTL alone does not
|
||||
# eject someone who stays connected. Schedule a hard DeleteRoom when Celery
|
||||
@@ -1740,20 +1786,21 @@ class DiagnosticsViewSet(viewsets.ViewSet):
|
||||
+ settings.CONNECTION_TEST_ROOM_EXTRA_AGE_SECONDS
|
||||
)
|
||||
delete_connection_test_room.apply_async(
|
||||
args=[room],
|
||||
args=[room, cluster.name],
|
||||
countdown=max_age,
|
||||
)
|
||||
|
||||
return drf_response.Response(
|
||||
{
|
||||
"livekit": {
|
||||
"url": settings.LIVEKIT_CONFIGURATION["url"],
|
||||
"url": cluster.url,
|
||||
"room": room,
|
||||
"token": generate_token(
|
||||
"token": utils.generate_token(
|
||||
room=room,
|
||||
user=request.user,
|
||||
username="Connection Test",
|
||||
ttl=timedelta(seconds=expires_in),
|
||||
cluster=cluster,
|
||||
),
|
||||
"expires_in": expires_in,
|
||||
},
|
||||
|
||||
@@ -1,19 +1,23 @@
|
||||
"""Authentication using LiveKit token for the Meet core app."""
|
||||
|
||||
from django.conf import settings
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.contrib.auth.models import AnonymousUser
|
||||
|
||||
from livekit.api import TokenVerifier
|
||||
from rest_framework import authentication, exceptions
|
||||
|
||||
from core.livekit.clusters import LiveKitClusterTokenVerifier
|
||||
|
||||
UserModel = get_user_model()
|
||||
|
||||
LIVEKIT_AUTH_SCHEME = "X-LiveKit-Token"
|
||||
|
||||
|
||||
class LiveKitTokenAuthentication(authentication.BaseAuthentication):
|
||||
"""Authenticate using LiveKit token and load the associated Django user."""
|
||||
"""Authenticate using LiveKit token and load the associated Django user.
|
||||
|
||||
The token may be signed by any registered LiveKit cluster. The request's
|
||||
auth is the token's claims, which carry that cluster as their `cluster`.
|
||||
"""
|
||||
|
||||
def authenticate(self, request):
|
||||
auth_header = request.headers.get("Authorization")
|
||||
@@ -35,11 +39,7 @@ class LiveKitTokenAuthentication(authentication.BaseAuthentication):
|
||||
token = parts[1]
|
||||
|
||||
try:
|
||||
verifier = TokenVerifier(
|
||||
api_key=settings.LIVEKIT_CONFIGURATION["api_key"],
|
||||
api_secret=settings.LIVEKIT_CONFIGURATION["api_secret"],
|
||||
)
|
||||
claims = verifier.verify(token)
|
||||
claims = LiveKitClusterTokenVerifier().verify(token)
|
||||
|
||||
user_id = claims.identity
|
||||
if not user_id:
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
"""Exception reporter filter hiding the settings that hold LiveKit credentials."""
|
||||
|
||||
import re
|
||||
|
||||
from django.utils.functional import SimpleLazyObject
|
||||
from django.views.debug import SafeExceptionReporterFilter
|
||||
|
||||
|
||||
class LiveKitSafeExceptionReporterFilter(SafeExceptionReporterFilter):
|
||||
"""Hide LIVEKIT_CLUSTERS, whose value is a JSON string holding the API secret of
|
||||
every LiveKit cluster, on top of the settings Django hides by name."""
|
||||
|
||||
hidden_settings = SimpleLazyObject(
|
||||
lambda: re.compile(
|
||||
f"{SafeExceptionReporterFilter.hidden_settings.pattern}|LIVEKIT_CLUSTERS$",
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1 @@
|
||||
"""The LiveKit clusters on which Django places rooms."""
|
||||
@@ -0,0 +1,150 @@
|
||||
"""Registry of the LiveKit clusters on which Django places rooms.
|
||||
|
||||
The first registered cluster is the default one. Each cluster has an API key of
|
||||
its own, so the issuer of a token or webhook tells which cluster signed it.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from functools import lru_cache
|
||||
|
||||
from django.conf import settings
|
||||
|
||||
import jwt
|
||||
from livekit.api import TokenVerifier
|
||||
from livekit.api.access_token import Claims
|
||||
|
||||
from .exceptions import UnknownLiveKitClusterError
|
||||
from .specs import parse_livekit_clusters
|
||||
|
||||
FALLBACK_LIVEKIT_CLUSTER_NAME = "default"
|
||||
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class LiveKitClusterConfig:
|
||||
"""How browsers and Django reach a LiveKit cluster.
|
||||
|
||||
Browsers connect to `url` and Django calls `api_url`. The API secret is
|
||||
left out of the representation, which may end up in logs.
|
||||
"""
|
||||
|
||||
name: str
|
||||
url: str
|
||||
api_url: str
|
||||
api_key: str
|
||||
api_secret: str = field(repr=False)
|
||||
weight: int = 1
|
||||
|
||||
|
||||
@lru_cache(maxsize=8)
|
||||
def _clusters_from_json(raw: str) -> tuple[LiveKitClusterConfig, ...]:
|
||||
"""Build the LiveKit clusters listed in a value of LIVEKIT_CLUSTERS."""
|
||||
return tuple(
|
||||
LiveKitClusterConfig(
|
||||
name=spec.name,
|
||||
url=spec.url,
|
||||
api_url=spec.api_url or spec.url,
|
||||
api_key=spec.api_key,
|
||||
api_secret=spec.api_secret.get_secret_value(),
|
||||
weight=spec.weight,
|
||||
)
|
||||
for spec in parse_livekit_clusters(raw)
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(maxsize=8)
|
||||
def _clusters_from_configuration(
|
||||
url: str, api_key: str, api_secret: str
|
||||
) -> tuple[LiveKitClusterConfig, ...]:
|
||||
"""Build the single LiveKit cluster that LIVEKIT_CONFIGURATION describes."""
|
||||
return (
|
||||
LiveKitClusterConfig(
|
||||
name=FALLBACK_LIVEKIT_CLUSTER_NAME,
|
||||
url=url,
|
||||
api_url=url,
|
||||
api_key=api_key,
|
||||
api_secret=api_secret,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def get_livekit_clusters() -> tuple[LiveKitClusterConfig, ...]:
|
||||
"""Return the registered LiveKit clusters, the default one first.
|
||||
|
||||
LIVEKIT_CLUSTERS settings lists them. When it is unset, a single cluster named
|
||||
"default" is built from LIVEKIT_CONFIGURATION.
|
||||
"""
|
||||
if settings.LIVEKIT_CLUSTERS is not None:
|
||||
return _clusters_from_json(settings.LIVEKIT_CLUSTERS)
|
||||
configuration = settings.LIVEKIT_CONFIGURATION
|
||||
return _clusters_from_configuration(
|
||||
configuration["url"],
|
||||
configuration["api_key"],
|
||||
configuration["api_secret"],
|
||||
)
|
||||
|
||||
|
||||
def get_default_livekit_cluster() -> LiveKitClusterConfig:
|
||||
"""Return the default LiveKit cluster: the first registered one."""
|
||||
return get_livekit_clusters()[0]
|
||||
|
||||
|
||||
def get_livekit_cluster(name: str) -> LiveKitClusterConfig:
|
||||
"""Return the registered LiveKit cluster with this name."""
|
||||
for cluster in get_livekit_clusters():
|
||||
if cluster.name == name:
|
||||
return cluster
|
||||
raise UnknownLiveKitClusterError(f"No LiveKit cluster is named {name!r}.")
|
||||
|
||||
|
||||
def get_livekit_cluster_by_api_key(api_key: str | None) -> LiveKitClusterConfig | None:
|
||||
"""Return the registered LiveKit cluster using this API key, if any."""
|
||||
if not api_key:
|
||||
return None
|
||||
for cluster in get_livekit_clusters():
|
||||
if cluster.api_key == api_key:
|
||||
return cluster
|
||||
return None
|
||||
|
||||
|
||||
def _get_livekit_cluster_by_issuer(token: str) -> LiveKitClusterConfig:
|
||||
"""Return the registered LiveKit cluster whose API key is a token's issuer.
|
||||
|
||||
The issuer is read without verifying the token. A missing issuer, or one
|
||||
that is no registered cluster's API key, raises the error PyJWT raises when
|
||||
an issuer is missing or is not the expected one.
|
||||
"""
|
||||
payload = jwt.decode(token, options={"verify_signature": False})
|
||||
if "iss" not in payload:
|
||||
raise jwt.MissingRequiredClaimError("iss")
|
||||
cluster = get_livekit_cluster_by_api_key(payload["iss"])
|
||||
if cluster is None:
|
||||
raise jwt.InvalidIssuerError("Invalid issuer")
|
||||
return cluster
|
||||
|
||||
|
||||
class LiveKitClusterTokenVerifier(TokenVerifier):
|
||||
"""Verify tokens signed by any registered LiveKit cluster.
|
||||
|
||||
A token's issuer is the API key it was signed with, and each cluster has an
|
||||
API key of its own, so the issuer names the cluster whose API key and secret
|
||||
verify the token. A token without issuer, or issued by no registered
|
||||
cluster, is refused as a TokenVerifier refuses a token without issuer or
|
||||
issued with another API key than its own.
|
||||
|
||||
The claims returned carry that cluster as their `cluster`. The verifier
|
||||
also keeps the cluster of the last token it accepted as its own `cluster`,
|
||||
for callers that never see the claims, such as those of a webhook receiver,
|
||||
which returns the event alone.
|
||||
"""
|
||||
|
||||
cluster: LiveKitClusterConfig | None = None
|
||||
|
||||
def verify(self, token: str, *, verify_signature: bool = True) -> Claims:
|
||||
"""Return the claims of a token, verified for the cluster that signed it."""
|
||||
cluster = _get_livekit_cluster_by_issuer(token)
|
||||
claims = TokenVerifier(
|
||||
cluster.api_key, cluster.api_secret, leeway=self._leeway
|
||||
).verify(token, verify_signature=verify_signature)
|
||||
claims.cluster = cluster
|
||||
self.cluster = cluster
|
||||
return claims
|
||||
@@ -0,0 +1,26 @@
|
||||
"""LiveKit clusters specific exceptions."""
|
||||
|
||||
|
||||
class UnknownLiveKitClusterError(LookupError):
|
||||
"""Raised when no registered LiveKit cluster has the requested name."""
|
||||
|
||||
|
||||
class LiveKitRoomUnavailableError(Exception):
|
||||
"""Raised when a LiveKit room cannot be joined for now.
|
||||
|
||||
Either it cannot be placed on any LiveKit cluster, or it cannot be created
|
||||
on the one it is placed on. Asking again later may succeed.
|
||||
"""
|
||||
|
||||
|
||||
class NoCandidateLiveKitClusterError(LiveKitRoomUnavailableError):
|
||||
"""Raised when no LiveKit cluster may receive new placements."""
|
||||
|
||||
|
||||
class LiveKitRoomCreationError(LiveKitRoomUnavailableError):
|
||||
"""Raised when a LiveKit cluster fails to create a LiveKit room."""
|
||||
|
||||
|
||||
class PlacementChangedError(LiveKitRoomUnavailableError):
|
||||
"""Raised when the placement of a LiveKit room keeps changing while
|
||||
credentials are handed out for it."""
|
||||
@@ -0,0 +1,370 @@
|
||||
"""Placements of LiveKit rooms on the LiveKit clusters.
|
||||
|
||||
A placement records the LiveKit cluster a LiveKit room lives on, from the first
|
||||
credentials handed out for it until the LiveKit room finishes.
|
||||
"""
|
||||
|
||||
from logging import getLogger
|
||||
|
||||
from django.conf import settings
|
||||
from django.utils import timezone
|
||||
from django.utils.text import slugify
|
||||
|
||||
import aiohttp
|
||||
from asgiref.sync import async_to_sync
|
||||
from livekit.api import CreateRoomRequest, TwirpError # pylint: disable=E0611
|
||||
|
||||
from core import utils
|
||||
from core.models import Placement, Room
|
||||
|
||||
from .clusters import LiveKitClusterConfig, get_livekit_clusters
|
||||
from .exceptions import (
|
||||
LiveKitRoomCreationError,
|
||||
NoCandidateLiveKitClusterError,
|
||||
PlacementChangedError,
|
||||
)
|
||||
from .strategies import PlacementContext, get_placement_strategy
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
PLACEMENT_CREATION_ATTEMPTS = 3
|
||||
PLACEMENT_UPDATE_ATTEMPTS = 2
|
||||
|
||||
|
||||
def _fits_placement(field_name: str, value: str) -> bool:
|
||||
"""Tell whether a value fits in a field of placements.
|
||||
|
||||
bulk_create skips validation, so lengths are checked beforehand.
|
||||
"""
|
||||
return len(value) <= Placement._meta.get_field(field_name).max_length # noqa: SLF001
|
||||
|
||||
|
||||
def _insert_placement(new_placement: Placement) -> Placement | None:
|
||||
"""Insert a placement, unless its LiveKit room is placed already, and
|
||||
return the LiveKit room's placement read back.
|
||||
|
||||
bulk_create(ignore_conflicts=True) does not tell whether it inserted the
|
||||
placement: the one read back is the new placement only if it has its id.
|
||||
When another request placed the LiveKit room meanwhile and its placement is
|
||||
gone by the time it is read back, the insertion is tried again, a bounded
|
||||
number of times, after which None is returned.
|
||||
"""
|
||||
for _attempt in range(PLACEMENT_CREATION_ATTEMPTS):
|
||||
Placement.objects.bulk_create([new_placement], ignore_conflicts=True)
|
||||
placement = Placement.objects.filter(room_name=new_placement.room_name).first()
|
||||
if placement is not None:
|
||||
return placement
|
||||
return None
|
||||
|
||||
|
||||
class PlacementService:
|
||||
"""Service for the placements of LiveKit rooms on LiveKit clusters."""
|
||||
|
||||
@classmethod
|
||||
def cluster_for(cls, room_name: str) -> LiveKitClusterConfig:
|
||||
"""Return the LiveKit cluster on which a LiveKit room is managed.
|
||||
|
||||
With a single registered cluster, every room lives there, and the
|
||||
database is not queried. Otherwise it is the cluster of the room's
|
||||
placement, as long as that cluster is registered. Rooms without
|
||||
placement, either live since before placements existed or not placed
|
||||
yet, and rooms placed on a cluster since removed from the registry, are
|
||||
managed on the default cluster.
|
||||
"""
|
||||
clusters = get_livekit_clusters()
|
||||
default_cluster = clusters[0]
|
||||
if len(clusters) == 1:
|
||||
return default_cluster
|
||||
|
||||
placed_on = (
|
||||
Placement.objects.filter(room_name=room_name)
|
||||
.values_list("cluster", flat=True)
|
||||
.first()
|
||||
)
|
||||
if placed_on is None:
|
||||
return default_cluster
|
||||
|
||||
for cluster in clusters:
|
||||
if cluster.name == placed_on:
|
||||
return cluster
|
||||
|
||||
logger.warning(
|
||||
"LiveKit room %s is placed on LiveKit cluster %s, which is not "
|
||||
"registered: managing it on the default cluster %s",
|
||||
room_name,
|
||||
placed_on,
|
||||
default_cluster.name,
|
||||
)
|
||||
return default_cluster
|
||||
|
||||
@classmethod
|
||||
def candidates(cls) -> tuple[LiveKitClusterConfig, ...]:
|
||||
"""Return the LiveKit clusters that may receive new placements.
|
||||
|
||||
They are the registered clusters, in registry order, but for those
|
||||
weighing 0, which keep their live rooms and receive no new placement.
|
||||
"""
|
||||
return tuple(
|
||||
cluster for cluster in get_livekit_clusters() if cluster.weight > 0
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def choose_cluster(
|
||||
cls, room_name: str, room: Room | None = None
|
||||
) -> LiveKitClusterConfig:
|
||||
"""Return the LiveKit cluster on which to place a LiveKit room.
|
||||
|
||||
The placement strategy named by LIVEKIT_PLACEMENT_STRATEGY picks it
|
||||
among the candidates, knowing the LiveKit room's name and the room it is
|
||||
named after, if any. Without any candidate, no LiveKit room can be
|
||||
placed anywhere: this is logged as critical and raises
|
||||
NoCandidateLiveKitClusterError.
|
||||
"""
|
||||
candidates = cls.candidates()
|
||||
if not candidates:
|
||||
logger.critical(
|
||||
"No LiveKit cluster may receive new placements: LiveKit room %s "
|
||||
"cannot be placed",
|
||||
room_name,
|
||||
)
|
||||
raise NoCandidateLiveKitClusterError(
|
||||
"No LiveKit cluster may receive new placements."
|
||||
)
|
||||
context = PlacementContext(room_name=room_name, room=room)
|
||||
return get_placement_strategy().choose(candidates, context)
|
||||
|
||||
@classmethod
|
||||
def place(cls, room_name: str, room: Room | None = None) -> LiveKitClusterConfig:
|
||||
"""Make a LiveKit room ready to be joined and return its LiveKit cluster.
|
||||
|
||||
It is called each time credentials are handed out for the LiveKit room,
|
||||
which must be for the cluster returned. A LiveKit room without placement
|
||||
is placed on the cluster the placement strategy picks, knowing the room
|
||||
it is named after, if any. A placement on a cluster no longer registered
|
||||
is deleted, and the LiveKit room placed again.
|
||||
|
||||
LiveKit clusters need not create rooms on their own, so the LiveKit room
|
||||
is then created on its cluster, unless it is live there already, and
|
||||
its SID recorded on its placement, which is no longer absent from its
|
||||
cluster. That is done only as long as the placement is still the one
|
||||
read, with the same epoch, and with the SID read or the very SID the
|
||||
cluster just returned, which another request creating the LiveKit room
|
||||
at the same time records too. Otherwise, as when it was relocated or
|
||||
deleted meanwhile, or got another SID, the placement is read again and
|
||||
the LiveKit room created once more.
|
||||
|
||||
Raise LiveKitRoomUnavailableError, which means trying again later:
|
||||
NoCandidateLiveKitClusterError when no cluster may receive the LiveKit
|
||||
room, LiveKitRoomCreationError when its cluster fails to create it, and
|
||||
PlacementChangedError when its placement keeps changing. A placement
|
||||
created by this call is deleted when its LiveKit room cannot be created,
|
||||
since no one was handed out credentials for it, so that the next call
|
||||
may place the LiveKit room elsewhere. An older placement stays.
|
||||
"""
|
||||
for _attempt in range(PLACEMENT_UPDATE_ATTEMPTS):
|
||||
placement, cluster, created = cls._get_or_create_placement(room_name, room)
|
||||
try:
|
||||
sid = cls.create_livekit_room(room_name, cluster)
|
||||
except LiveKitRoomCreationError:
|
||||
if created:
|
||||
cls._delete_created_placement(placement, cluster)
|
||||
raise
|
||||
|
||||
if Placement.objects.filter(
|
||||
pk=placement.pk, epoch=placement.epoch, sid__in=(placement.sid, sid)
|
||||
).update(sid=sid, absent_since=None, updated_at=timezone.now()):
|
||||
return cluster
|
||||
|
||||
logger.info(
|
||||
"Placement of LiveKit room %s changed while LiveKit cluster %s "
|
||||
"created it: reading it again",
|
||||
room_name,
|
||||
cluster.name,
|
||||
)
|
||||
|
||||
logger.warning(
|
||||
"Placement of LiveKit room %s kept changing while its LiveKit room "
|
||||
"was created",
|
||||
room_name,
|
||||
)
|
||||
raise PlacementChangedError(
|
||||
f"The placement of LiveKit room {room_name} kept changing."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _get_or_create_placement(
|
||||
cls, room_name: str, room: Room | None
|
||||
) -> tuple[Placement, LiveKitClusterConfig, bool]:
|
||||
"""Return the placement of a LiveKit room, placing it if need be.
|
||||
|
||||
Return the placement, its registered LiveKit cluster and whether this
|
||||
call created the placement. Another request may place the LiveKit room
|
||||
at the same time: both then get the one placement read back. A
|
||||
placement on a cluster no longer registered is deleted, provided it did
|
||||
not change meanwhile, and the LiveKit room placed again, a bounded
|
||||
number of times.
|
||||
"""
|
||||
clusters = {cluster.name: cluster for cluster in get_livekit_clusters()}
|
||||
for _attempt in range(PLACEMENT_CREATION_ATTEMPTS):
|
||||
created = False
|
||||
placement = Placement.objects.filter(room_name=room_name).first()
|
||||
if placement is None:
|
||||
new_placement = Placement(
|
||||
room_name=room_name,
|
||||
room=room,
|
||||
cluster=cls.choose_cluster(room_name, room).name,
|
||||
)
|
||||
placement = _insert_placement(new_placement)
|
||||
if placement is None:
|
||||
break
|
||||
created = placement.pk == new_placement.pk
|
||||
if created:
|
||||
logger.info(
|
||||
"Placed LiveKit room %s on LiveKit cluster %s",
|
||||
room_name,
|
||||
placement.cluster,
|
||||
)
|
||||
|
||||
cluster = clusters.get(placement.cluster)
|
||||
if cluster is not None:
|
||||
return placement, cluster, created
|
||||
|
||||
logger.warning(
|
||||
"LiveKit room %s is placed on LiveKit cluster %s, which is not "
|
||||
"registered: placing it again",
|
||||
room_name,
|
||||
placement.cluster,
|
||||
)
|
||||
Placement.objects.filter(
|
||||
pk=placement.pk, epoch=placement.epoch, cluster=placement.cluster
|
||||
).delete()
|
||||
|
||||
raise PlacementChangedError(
|
||||
f"The placement of LiveKit room {room_name} kept changing."
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _delete_created_placement(placement: Placement, cluster: LiveKitClusterConfig):
|
||||
"""Delete a placement whose LiveKit room its cluster failed to create.
|
||||
|
||||
It is deleted only while it still has the epoch read and no SID: no one
|
||||
was handed out credentials for it then.
|
||||
"""
|
||||
deleted, _ = Placement.objects.filter(
|
||||
pk=placement.pk, epoch=placement.epoch, sid=""
|
||||
).delete()
|
||||
if deleted:
|
||||
logger.info(
|
||||
"Deleted the placement of LiveKit room %s on LiveKit cluster %s, "
|
||||
"which failed to create it",
|
||||
placement.room_name,
|
||||
cluster.name,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def create_livekit_room(cls, room_name: str, cluster: LiveKitClusterConfig) -> str:
|
||||
"""Create a LiveKit room on a LiveKit cluster and return its SID.
|
||||
|
||||
A LiveKit room live there already is kept as it is, and its SID
|
||||
returned. No metadata is sent: the cluster would store it in place of
|
||||
a live LiveKit room's, but leave the live room with its own. The
|
||||
cluster is given LIVEKIT_CREATE_ROOM_TIMEOUT_SECONDS to answer.
|
||||
|
||||
Raise LiveKitRoomCreationError when the cluster cannot be reached, does
|
||||
not answer in time or answers with an error.
|
||||
"""
|
||||
return cls._create_livekit_room(room_name, cluster)
|
||||
|
||||
@staticmethod
|
||||
@async_to_sync
|
||||
async def _create_livekit_room(
|
||||
room_name: str, cluster: LiveKitClusterConfig
|
||||
) -> str:
|
||||
"""Create a LiveKit room on a LiveKit cluster and return its SID."""
|
||||
lkapi = utils.create_livekit_client(
|
||||
cluster=cluster, timeout=settings.LIVEKIT_CREATE_ROOM_TIMEOUT_SECONDS
|
||||
)
|
||||
try:
|
||||
livekit_room = await lkapi.room.create_room(
|
||||
CreateRoomRequest(name=room_name)
|
||||
)
|
||||
except (TwirpError, TimeoutError, aiohttp.ClientError) as error:
|
||||
logger.exception(
|
||||
"LiveKit cluster %s failed to create LiveKit room %s",
|
||||
cluster.name,
|
||||
room_name,
|
||||
)
|
||||
raise LiveKitRoomCreationError(
|
||||
f"LiveKit cluster {cluster.name} failed to create LiveKit room "
|
||||
f"{room_name}."
|
||||
) from error
|
||||
finally:
|
||||
await lkapi.aclose()
|
||||
return livekit_room.sid
|
||||
|
||||
@classmethod
|
||||
def adopt(
|
||||
cls, room_name: str, cluster: LiveKitClusterConfig, sid: str
|
||||
) -> Placement | None:
|
||||
"""Place a live LiveKit room on the LiveKit cluster it lives on.
|
||||
|
||||
It is meant for a LiveKit room live without placement. Only a LiveKit
|
||||
room Django hands out credentials for is adopted: one named after the
|
||||
id of an active room, or, while unregistered rooms are allowed, after a
|
||||
slug that is not a UUID and no longer than a room's slug may be.
|
||||
Connection test rooms never are. A LiveKit room named after a UUID that
|
||||
is no active room's id, or whose name or SID does not fit in a
|
||||
placement, is logged.
|
||||
|
||||
Return the placement of the LiveKit room read back, which may have been
|
||||
created meanwhile by another request, or None when it is not adopted.
|
||||
When another request placed the LiveKit room meanwhile and its placement
|
||||
is gone by the time it is read back, adoption is tried again, a bounded
|
||||
number of times.
|
||||
"""
|
||||
if room_name.startswith(settings.CONNECTION_TEST_ROOM_PREFIX):
|
||||
return None
|
||||
|
||||
room = None
|
||||
if utils.is_uuid(room_name):
|
||||
room = Room.objects.filter(pk=room_name).first()
|
||||
if room is None:
|
||||
logger.warning(
|
||||
"LiveKit room %s, live on LiveKit cluster %s, is named after "
|
||||
"no active room: leaving it without placement",
|
||||
room_name,
|
||||
cluster.name,
|
||||
)
|
||||
return None
|
||||
elif not (
|
||||
settings.ALLOW_UNREGISTERED_ROOMS
|
||||
and room_name
|
||||
and slugify(room_name) == room_name
|
||||
and len(room_name) <= Room._meta.get_field("slug").max_length # noqa: SLF001
|
||||
):
|
||||
return None
|
||||
|
||||
if not (
|
||||
_fits_placement("room_name", room_name) and _fits_placement("sid", sid)
|
||||
):
|
||||
logger.warning(
|
||||
"LiveKit room %s, live on LiveKit cluster %s with SID %s, has a "
|
||||
"name or SID too long for a placement",
|
||||
room_name,
|
||||
cluster.name,
|
||||
sid,
|
||||
)
|
||||
return None
|
||||
|
||||
adopted = Placement(
|
||||
room_name=room_name, room=room, cluster=cluster.name, sid=sid
|
||||
)
|
||||
placement = _insert_placement(adopted)
|
||||
if placement is not None and placement.pk == adopted.pk:
|
||||
logger.info(
|
||||
"Adopted LiveKit room %s, live on LiveKit cluster %s with SID %s",
|
||||
room_name,
|
||||
cluster.name,
|
||||
sid,
|
||||
)
|
||||
return placement
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Specification of the LiveKit clusters listed in the LIVEKIT_CLUSTERS setting.
|
||||
|
||||
Free of Django, so that the settings can validate the value they read.
|
||||
"""
|
||||
|
||||
import re
|
||||
from typing import Annotated
|
||||
|
||||
from pydantic import (
|
||||
AfterValidator,
|
||||
AnyUrl,
|
||||
BaseModel,
|
||||
Field,
|
||||
NonNegativeInt,
|
||||
SecretStr,
|
||||
StringConstraints,
|
||||
TypeAdapter,
|
||||
UrlConstraints,
|
||||
ValidationError,
|
||||
)
|
||||
from pydantic_core import PydanticCustomError
|
||||
|
||||
LIVEKIT_CLUSTER_NAME_PATTERN = r"^[a-z0-9-]{1,63}$"
|
||||
|
||||
_LIVEKIT_URL_ADAPTER = TypeAdapter(
|
||||
Annotated[AnyUrl, UrlConstraints(allowed_schemes=["http", "https", "ws", "wss"])]
|
||||
)
|
||||
|
||||
|
||||
_LIVEKIT_URL_AUTHORITY = re.compile(r"(?i)(https?|wss?)://[^/\\]")
|
||||
|
||||
|
||||
def _check_livekit_url(url: str) -> str:
|
||||
"""Ensure a URL reaches LiveKit over HTTP/WebSocket, and stay unchanged."""
|
||||
if " " in url or not url.isprintable():
|
||||
raise PydanticCustomError(
|
||||
"url_characters",
|
||||
"URL should not contain whitespace or non-printable characters",
|
||||
)
|
||||
_LIVEKIT_URL_ADAPTER.validate_python(url)
|
||||
if not _LIVEKIT_URL_AUTHORITY.match(url):
|
||||
raise PydanticCustomError(
|
||||
"url_authority", "URL should be written scheme://host"
|
||||
)
|
||||
return url
|
||||
|
||||
|
||||
LiveKitUrl = Annotated[str, AfterValidator(_check_livekit_url)]
|
||||
|
||||
|
||||
class LiveKitClusterSpec(BaseModel):
|
||||
"""A LiveKit cluster, as listed in LIVEKIT_CLUSTERS.
|
||||
|
||||
Browsers connect to `url` and Django calls `api_url`, which defaults to
|
||||
`url`. Each cluster has its own API key, which tells it apart.
|
||||
"""
|
||||
|
||||
name: Annotated[str, StringConstraints(pattern=LIVEKIT_CLUSTER_NAME_PATTERN)]
|
||||
url: LiveKitUrl
|
||||
api_url: LiveKitUrl | None = None
|
||||
api_key: Annotated[str, StringConstraints(min_length=1)]
|
||||
api_secret: Annotated[SecretStr, Field(min_length=1)]
|
||||
weight: NonNegativeInt = 1
|
||||
|
||||
model_config = {"extra": "forbid"}
|
||||
|
||||
|
||||
_LIVEKIT_CLUSTERS_ADAPTER = TypeAdapter(
|
||||
Annotated[list[LiveKitClusterSpec], Field(min_length=1)]
|
||||
)
|
||||
|
||||
|
||||
def _locate(loc) -> str:
|
||||
"""Name a place in LIVEKIT_CLUSTERS, such as LIVEKIT_CLUSTERS[0].url."""
|
||||
return "LIVEKIT_CLUSTERS" + "".join(
|
||||
f"[{part}]" if isinstance(part, int) else f".{part}" for part in loc
|
||||
)
|
||||
|
||||
|
||||
def _find_duplicates(specs: list[LiveKitClusterSpec]) -> list[str]:
|
||||
"""Describe each cluster reusing the name or the API key of an earlier one."""
|
||||
problems = []
|
||||
first_by_name = {}
|
||||
first_by_api_key = {}
|
||||
for index, spec in enumerate(specs):
|
||||
first = first_by_name.setdefault(spec.name, index)
|
||||
if first != index:
|
||||
problems.append(
|
||||
f"{_locate((index, 'name'))}: {spec.name!r} is already the name "
|
||||
f"of {_locate((first,))}"
|
||||
)
|
||||
first = first_by_api_key.setdefault(spec.api_key, index)
|
||||
if first != index:
|
||||
problems.append(
|
||||
f"{_locate((index, 'api_key'))}: {_locate((first,))} already uses "
|
||||
"this API key, each LiveKit cluster needs its own"
|
||||
)
|
||||
return problems
|
||||
|
||||
|
||||
def parse_livekit_clusters(raw: str) -> list[LiveKitClusterSpec]:
|
||||
"""Parse LIVEKIT_CLUSTERS, a non-empty JSON list of LiveKit clusters.
|
||||
|
||||
The first cluster is the default one. Names and API keys must be unique.
|
||||
Every problem found is reported in a single ValueError.
|
||||
"""
|
||||
specs = []
|
||||
problems = []
|
||||
try:
|
||||
specs = _LIVEKIT_CLUSTERS_ADAPTER.validate_json(raw, strict=True)
|
||||
except ValidationError as error:
|
||||
problems = [
|
||||
f"{_locate(problem['loc'])}: {problem['msg']}"
|
||||
for problem in error.errors(include_input=False, include_url=False)
|
||||
]
|
||||
problems += _find_duplicates(specs)
|
||||
if problems:
|
||||
raise ValueError("; ".join(problems))
|
||||
return specs
|
||||
@@ -0,0 +1,145 @@
|
||||
"""Placement strategies, the rules picking the LiveKit cluster of a new placement.
|
||||
|
||||
LIVEKIT_PLACEMENT_STRATEGY names the strategy in use. It picks among candidates:
|
||||
the LiveKit clusters that may receive new placements.
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from bisect import bisect_right
|
||||
from dataclasses import dataclass
|
||||
from fractions import Fraction
|
||||
from itertools import accumulate
|
||||
from secrets import randbelow
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.cache import cache
|
||||
from django.db.models import Count
|
||||
from django.utils.module_loading import import_string
|
||||
|
||||
from core.models import Placement, Room
|
||||
|
||||
from .clusters import LiveKitClusterConfig
|
||||
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class PlacementContext:
|
||||
"""What a placement strategy is told about the LiveKit room to place.
|
||||
|
||||
`room` is the room the LiveKit room is named after, or None for an
|
||||
unregistered room or a connection test room.
|
||||
"""
|
||||
|
||||
room_name: str
|
||||
room: Room | None = None
|
||||
|
||||
|
||||
class PlacementStrategy(ABC):
|
||||
"""A placement strategy, which picks the LiveKit cluster of a new placement.
|
||||
|
||||
A strategy is instantiated each time a cluster is chosen, so whatever it
|
||||
has to remember goes to the cache or the database.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def choose(
|
||||
self,
|
||||
candidates: tuple[LiveKitClusterConfig, ...],
|
||||
context: PlacementContext,
|
||||
) -> LiveKitClusterConfig:
|
||||
"""Return the candidate on which to place a LiveKit room.
|
||||
|
||||
The candidates are the LiveKit clusters that may receive new placements,
|
||||
in registry order. There is at least one, and each weighs more than 0.
|
||||
"""
|
||||
|
||||
|
||||
class WeightedRandomStrategy(PlacementStrategy):
|
||||
"""Pick LiveKit clusters at random, in proportion to their weights.
|
||||
|
||||
Clusters of equal weights receive as many new placements on average, and
|
||||
with weights of 1 and 3, the second cluster receives three quarters of them.
|
||||
"""
|
||||
|
||||
def choose(
|
||||
self,
|
||||
candidates: tuple[LiveKitClusterConfig, ...],
|
||||
context: PlacementContext,
|
||||
) -> LiveKitClusterConfig:
|
||||
"""Return a candidate drawn with a probability proportional to its weight."""
|
||||
cumulative_weights = list(accumulate(cluster.weight for cluster in candidates))
|
||||
draw = randbelow(cumulative_weights[-1])
|
||||
return candidates[bisect_right(cumulative_weights, draw)]
|
||||
|
||||
|
||||
class RoundRobinStrategy(PlacementStrategy):
|
||||
"""Pick the LiveKit clusters in turn, whatever their weights.
|
||||
|
||||
The first new placement among candidates goes to the first of them, the
|
||||
next one to the second, and so on. Turns are counted in the Django cache,
|
||||
which all processes share, with a counter for each set of candidates.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def counter_key(candidates: tuple[LiveKitClusterConfig, ...]) -> str:
|
||||
"""Return the cache key counting the placements made among candidates."""
|
||||
names = ",".join(cluster.name for cluster in candidates)
|
||||
return f"livekit-placement-round-robin_{names}"
|
||||
|
||||
def choose(
|
||||
self,
|
||||
candidates: tuple[LiveKitClusterConfig, ...],
|
||||
context: PlacementContext,
|
||||
) -> LiveKitClusterConfig:
|
||||
"""Return the candidate whose turn it is.
|
||||
|
||||
Should the counter vanish between its creation and its increment, the
|
||||
turn goes to the first candidate.
|
||||
"""
|
||||
key = self.counter_key(candidates)
|
||||
cache.add(key, 0, timeout=None)
|
||||
try:
|
||||
count = cache.incr(key)
|
||||
except ValueError:
|
||||
count = 1
|
||||
return candidates[(count - 1) % len(candidates)]
|
||||
|
||||
|
||||
class LeastPlacementsStrategy(PlacementStrategy):
|
||||
"""Pick the LiveKit cluster holding the fewest placements for its weight.
|
||||
|
||||
A cluster weighing twice as much as another is given twice as many
|
||||
placements. On a tie, the cluster registered first is picked. Rooms placed
|
||||
at the same moment may all go to the same cluster, as each of them is
|
||||
placed before the placements of the others are recorded.
|
||||
"""
|
||||
|
||||
def choose(
|
||||
self,
|
||||
candidates: tuple[LiveKitClusterConfig, ...],
|
||||
context: PlacementContext,
|
||||
) -> LiveKitClusterConfig:
|
||||
"""Return the candidate holding the fewest placements for its weight."""
|
||||
placement_counts = dict(
|
||||
Placement.objects.filter(
|
||||
cluster__in=[cluster.name for cluster in candidates]
|
||||
)
|
||||
.values_list("cluster")
|
||||
.annotate(Count("pk"))
|
||||
.order_by()
|
||||
)
|
||||
return min(
|
||||
candidates,
|
||||
key=lambda cluster: Fraction(
|
||||
placement_counts.get(cluster.name, 0), cluster.weight
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def get_placement_strategy() -> PlacementStrategy:
|
||||
"""Return the placement strategy named by LIVEKIT_PLACEMENT_STRATEGY.
|
||||
|
||||
It is looked up on every call and never while the settings load, so that a
|
||||
strategy may need applications that are not ready then, and a change of
|
||||
settings applies at once.
|
||||
"""
|
||||
return import_string(settings.LIVEKIT_PLACEMENT_STRATEGY)()
|
||||
@@ -0,0 +1,70 @@
|
||||
# Generated by Django 5.2.17 on 2026-11-06 22:46
|
||||
|
||||
import django.db.models.deletion
|
||||
import uuid
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
SET_NULL_ON_ROOM_DELETE = """
|
||||
DO $$
|
||||
DECLARE
|
||||
fk_name text;
|
||||
BEGIN
|
||||
SELECT conname INTO fk_name
|
||||
FROM pg_constraint
|
||||
WHERE contype = 'f'
|
||||
AND conrelid = 'meet_placement'::regclass
|
||||
AND confrelid = 'meet_room'::regclass;
|
||||
EXECUTE format(
|
||||
'ALTER TABLE meet_placement DROP CONSTRAINT %I', fk_name
|
||||
);
|
||||
EXECUTE format(
|
||||
'ALTER TABLE meet_placement ADD CONSTRAINT %I '
|
||||
'FOREIGN KEY (room_id) REFERENCES meet_room (resource_id) '
|
||||
'ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED',
|
||||
fk_name
|
||||
);
|
||||
END $$;
|
||||
"""
|
||||
|
||||
|
||||
def set_null_on_room_delete(apps, schema_editor):
|
||||
"""Make PostgreSQL itself unset the room of a placement when the room is deleted."""
|
||||
for statement in schema_editor.deferred_sql:
|
||||
schema_editor.execute(statement)
|
||||
schema_editor.deferred_sql.clear()
|
||||
schema_editor.execute(SET_NULL_ON_ROOM_DELETE, params=None)
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('core', '0029_alter_application_client_secret_sha256'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='Placement',
|
||||
fields=[
|
||||
('id', models.UUIDField(default=uuid.uuid4, editable=False, help_text='primary key for the record as UUID', primary_key=True, serialize=False, verbose_name='id')),
|
||||
('created_at', models.DateTimeField(auto_now_add=True, help_text='date and time at which a record was created', verbose_name='created on')),
|
||||
('updated_at', models.DateTimeField(auto_now=True, help_text='date and time at which a record was last updated', verbose_name='updated on')),
|
||||
('room_name', models.CharField(help_text="The room's id, or the slug of an unregistered room.", max_length=255, unique=True, verbose_name='LiveKit room name')),
|
||||
('cluster', models.CharField(db_index=True, help_text='Name of the LiveKit cluster the LiveKit room lives on.', max_length=63, verbose_name='LiveKit cluster')),
|
||||
('sid', models.CharField(blank=True, default='', help_text='SID of the LiveKit room on its cluster, empty until known.', max_length=63, verbose_name='LiveKit room SID')),
|
||||
('epoch', models.PositiveIntegerField(default=0, help_text='Incremented on every relocation.', verbose_name='Epoch')),
|
||||
('previous_cluster', models.CharField(blank=True, default='', help_text='Name of the LiveKit cluster the room was last relocated from.', max_length=63, verbose_name='Previous LiveKit cluster')),
|
||||
('relocated_at', models.DateTimeField(blank=True, help_text='Date and time at which the room was last relocated.', null=True, verbose_name='Relocated at')),
|
||||
('absent_since', models.DateTimeField(blank=True, help_text='Date and time since which the LiveKit room is missing from its cluster.', null=True, verbose_name='Absent since')),
|
||||
('room', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='placements', to='core.room', verbose_name='Room')),
|
||||
],
|
||||
options={
|
||||
'verbose_name': 'Placement',
|
||||
'verbose_name_plural': 'Placements',
|
||||
'db_table': 'meet_placement',
|
||||
},
|
||||
),
|
||||
migrations.RunPython(
|
||||
set_null_on_room_delete, reverse_code=migrations.RunPython.noop
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,18 @@
|
||||
# Generated by Django 5.2.17 on 2026-11-07 15:03
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('core', '0030_placement'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='recording',
|
||||
name='livekit_cluster',
|
||||
field=models.CharField(blank=True, db_default='', default='', help_text='Name of the LiveKit cluster recording the room, the default cluster when empty.', max_length=63, verbose_name='LiveKit cluster'),
|
||||
),
|
||||
]
|
||||
@@ -27,6 +27,12 @@ from lasuite.tools.email import get_domain_from_email
|
||||
from timezone_field import TimeZoneField
|
||||
|
||||
from . import fields, hashers, utils
|
||||
from .livekit.clusters import (
|
||||
LiveKitClusterConfig,
|
||||
get_default_livekit_cluster,
|
||||
get_livekit_cluster,
|
||||
)
|
||||
from .livekit.exceptions import UnknownLiveKitClusterError
|
||||
from .recording.enums import FileExtension
|
||||
from .validators import sub_validator
|
||||
|
||||
@@ -707,6 +713,17 @@ class Recording(BaseModel):
|
||||
verbose_name=_("External Process ID"),
|
||||
help_text=_("ID of the external process associated with the recording."),
|
||||
)
|
||||
livekit_cluster = models.CharField(
|
||||
max_length=63,
|
||||
blank=True,
|
||||
default="",
|
||||
db_default="",
|
||||
verbose_name=_("LiveKit cluster"),
|
||||
help_text=_(
|
||||
"Name of the LiveKit cluster recording the room, the default "
|
||||
"cluster when empty."
|
||||
),
|
||||
)
|
||||
|
||||
class Meta:
|
||||
db_table = "meet_recording"
|
||||
@@ -775,6 +792,30 @@ class Recording(BaseModel):
|
||||
for user_id in room_privileged_user_ids
|
||||
)
|
||||
|
||||
def get_livekit_cluster(self) -> LiveKitClusterConfig:
|
||||
"""Return the LiveKit cluster recording the room.
|
||||
|
||||
It is the cluster the room was placed on when the recording started,
|
||||
wherever the room was placed since. A recording naming no cluster, as
|
||||
one started before recordings kept theirs, is on the default cluster.
|
||||
So is a recording on a cluster no longer registered, which is logged.
|
||||
"""
|
||||
if not self.livekit_cluster:
|
||||
return get_default_livekit_cluster()
|
||||
|
||||
try:
|
||||
return get_livekit_cluster(self.livekit_cluster)
|
||||
except UnknownLiveKitClusterError:
|
||||
cluster = get_default_livekit_cluster()
|
||||
logger.warning(
|
||||
"Recording %s is on LiveKit cluster %s, which is not registered: "
|
||||
"managing it on the default cluster %s",
|
||||
self.id,
|
||||
self.livekit_cluster,
|
||||
cluster.name,
|
||||
)
|
||||
return cluster
|
||||
|
||||
def is_savable(self) -> bool:
|
||||
"""Determine if the recording can be saved based on its current status."""
|
||||
|
||||
@@ -1262,3 +1303,73 @@ class File(BaseModel):
|
||||
|
||||
self.hard_deleted_at = timezone.now()
|
||||
self.save(update_fields=["hard_deleted_at"])
|
||||
|
||||
|
||||
class Placement(BaseModel):
|
||||
"""The record that a LiveKit room lives on a given LiveKit cluster.
|
||||
|
||||
It lasts from the first credentials handed out for the LiveKit room until
|
||||
the LiveKit room finishes.
|
||||
"""
|
||||
|
||||
room_name = models.CharField(
|
||||
max_length=255,
|
||||
unique=True,
|
||||
verbose_name=_("LiveKit room name"),
|
||||
help_text=_("The room's id, or the slug of an unregistered room."),
|
||||
)
|
||||
room = models.ForeignKey(
|
||||
Room,
|
||||
on_delete=models.SET_NULL,
|
||||
null=True,
|
||||
blank=True,
|
||||
related_name="placements",
|
||||
verbose_name=_("Room"),
|
||||
)
|
||||
cluster = models.CharField(
|
||||
max_length=63,
|
||||
db_index=True,
|
||||
verbose_name=_("LiveKit cluster"),
|
||||
help_text=_("Name of the LiveKit cluster the LiveKit room lives on."),
|
||||
)
|
||||
sid = models.CharField(
|
||||
max_length=63,
|
||||
blank=True,
|
||||
default="",
|
||||
verbose_name=_("LiveKit room SID"),
|
||||
help_text=_("SID of the LiveKit room on its cluster, empty until known."),
|
||||
)
|
||||
epoch = models.PositiveIntegerField(
|
||||
default=0,
|
||||
verbose_name=_("Epoch"),
|
||||
help_text=_("Incremented on every relocation."),
|
||||
)
|
||||
previous_cluster = models.CharField(
|
||||
max_length=63,
|
||||
blank=True,
|
||||
default="",
|
||||
verbose_name=_("Previous LiveKit cluster"),
|
||||
help_text=_("Name of the LiveKit cluster the room was last relocated from."),
|
||||
)
|
||||
relocated_at = models.DateTimeField(
|
||||
null=True,
|
||||
blank=True,
|
||||
verbose_name=_("Relocated at"),
|
||||
help_text=_("Date and time at which the room was last relocated."),
|
||||
)
|
||||
absent_since = models.DateTimeField(
|
||||
null=True,
|
||||
blank=True,
|
||||
verbose_name=_("Absent since"),
|
||||
help_text=_(
|
||||
"Date and time since which the LiveKit room is missing from its cluster."
|
||||
),
|
||||
)
|
||||
|
||||
class Meta:
|
||||
db_table = "meet_placement"
|
||||
verbose_name = _("Placement")
|
||||
verbose_name_plural = _("Placements")
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.room_name} on {self.cluster}"
|
||||
|
||||
@@ -121,7 +121,7 @@ class NotificationService:
|
||||
return not has_failures
|
||||
|
||||
@staticmethod
|
||||
async def _get_recording_timestamps(worker_id):
|
||||
async def _get_recording_timestamps(worker_id, cluster):
|
||||
"""Fetch FileInfo.started_at and ended_at from LiveKit's egress API.
|
||||
|
||||
FileInfo.started_at is more accurate than EgressInfo.started_at because
|
||||
@@ -130,6 +130,9 @@ class NotificationService:
|
||||
corresponds to when the egress service received the request, not the moment
|
||||
the egress worker effectively joined the room.
|
||||
|
||||
The egress is looked up on `cluster`, the LiveKit cluster that recorded
|
||||
the room.
|
||||
|
||||
Returns:
|
||||
Tuple of (started_at, ended_at) datetimes, either may be None.
|
||||
"""
|
||||
@@ -137,7 +140,7 @@ class NotificationService:
|
||||
if not worker_id:
|
||||
return None, None
|
||||
|
||||
lkapi = utils.create_livekit_client()
|
||||
lkapi = utils.create_livekit_client(cluster=cluster)
|
||||
try:
|
||||
egress_list = await lkapi.egress.list_egress(
|
||||
livekit_api.ListEgressRequest(egress_id=worker_id) # pylint: disable=no-member
|
||||
@@ -266,7 +269,7 @@ class NotificationService:
|
||||
|
||||
started_at, ended_at = async_to_sync(
|
||||
NotificationService._get_recording_timestamps
|
||||
)(recording.worker_id)
|
||||
)(recording.worker_id, recording.get_livekit_cluster())
|
||||
|
||||
payload = {
|
||||
"owner_id": str(recipient.id),
|
||||
@@ -335,7 +338,7 @@ class NotificationService:
|
||||
|
||||
started_at, ended_at = async_to_sync(
|
||||
NotificationService._get_recording_timestamps
|
||||
)(recording.worker_id)
|
||||
)(recording.worker_id, recording.get_livekit_cluster())
|
||||
|
||||
form_base_url = settings.TRANSCRIPTION_SATISFACTION_FORM_BASE_URL
|
||||
form_link = (
|
||||
|
||||
@@ -10,6 +10,7 @@ from livekit.protocol.agent_dispatch import (
|
||||
)
|
||||
|
||||
from core import utils
|
||||
from core.livekit.clusters import LiveKitClusterConfig
|
||||
from core.models import Recording
|
||||
|
||||
logger = getLogger(__name__)
|
||||
@@ -20,13 +21,22 @@ class MetadataCollectorException(Exception):
|
||||
|
||||
|
||||
class MetadataCollectorService:
|
||||
"""Service for dispatching and managing the metadata collector agent."""
|
||||
"""Service for dispatching and managing the metadata collector agent.
|
||||
|
||||
@async_to_sync
|
||||
async def start(self, recording: Recording):
|
||||
The agent is dispatched to the LiveKit room on the LiveKit cluster
|
||||
recording it, where its dispatch is deleted too.
|
||||
"""
|
||||
|
||||
def start(self, recording: Recording):
|
||||
"""Explicitly dispatch the metadata collector agent to a room."""
|
||||
|
||||
lkapi = utils.create_livekit_client()
|
||||
return self._start(recording, recording.get_livekit_cluster())
|
||||
|
||||
@async_to_sync
|
||||
async def _start(self, recording: Recording, cluster: LiveKitClusterConfig):
|
||||
"""Dispatch the metadata collector agent to a room on a LiveKit cluster."""
|
||||
|
||||
lkapi = utils.create_livekit_client(cluster=cluster)
|
||||
room_id = str(recording.room.id)
|
||||
|
||||
try:
|
||||
@@ -60,13 +70,18 @@ class MetadataCollectorService:
|
||||
|
||||
return dispatch_id
|
||||
|
||||
@async_to_sync
|
||||
async def stop(self, recording: Recording):
|
||||
def stop(self, recording: Recording):
|
||||
"""Stop and delete the agent dispatch associated to the room."""
|
||||
|
||||
return self._stop(recording, recording.get_livekit_cluster())
|
||||
|
||||
@async_to_sync
|
||||
async def _stop(self, recording: Recording, cluster: LiveKitClusterConfig):
|
||||
"""Delete the agent dispatch associated to the room on a LiveKit cluster."""
|
||||
|
||||
room_id = str(recording.room.id)
|
||||
dispatch_id = recording.options.get("metadata_collector_dispatch_id")
|
||||
lkapi = utils.create_livekit_client()
|
||||
lkapi = utils.create_livekit_client(cluster=cluster)
|
||||
|
||||
try:
|
||||
if not dispatch_id:
|
||||
|
||||
@@ -108,6 +108,7 @@ class RecordingEventsService:
|
||||
utils.notify_participants(
|
||||
room_name=str(recording.room.id),
|
||||
notification_data={"type": notification_type},
|
||||
cluster=recording.get_livekit_cluster(),
|
||||
)
|
||||
except utils.NotificationError as e:
|
||||
raise RecordingEventsError(
|
||||
@@ -139,7 +140,9 @@ class RecordingEventsService:
|
||||
if recording_status:
|
||||
try:
|
||||
RoomManagement.update_metadata(
|
||||
room_name, {"recording_status": recording_status}
|
||||
room_name,
|
||||
{"recording_status": recording_status},
|
||||
cluster=recording.get_livekit_cluster(),
|
||||
)
|
||||
except RoomNotFoundException:
|
||||
logger.info(
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# pylint: disable=no-member
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
from functools import lru_cache
|
||||
from typing import Any, ClassVar, Dict, Optional, Protocol, Type
|
||||
|
||||
@@ -12,6 +12,9 @@ from django.utils.module_loading import import_string
|
||||
|
||||
from livekit import api as livekit_api
|
||||
|
||||
from ... import utils
|
||||
from ...livekit.clusters import LiveKitClusterConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Codec / frequency constants matching LiveKit's H264_720P_30 preset.
|
||||
@@ -91,7 +94,7 @@ class WorkerServiceConfig:
|
||||
|
||||
return cls(
|
||||
output_folder=settings.RECORDING_OUTPUT_FOLDER,
|
||||
server_configurations=settings.LIVEKIT_CONFIGURATION,
|
||||
server_configurations=utils.livekit_api_configuration(),
|
||||
bucket_args={
|
||||
"endpoint": settings.AWS_S3_ENDPOINT_URL,
|
||||
"access_key": settings.AWS_S3_ACCESS_KEY_ID,
|
||||
@@ -124,8 +127,15 @@ class WorkerService(Protocol):
|
||||
"""Stop recording for a specified worker."""
|
||||
|
||||
|
||||
def get_worker_service(mode: str) -> WorkerService:
|
||||
"""Instantiate a worker service by its mode."""
|
||||
def get_worker_service(
|
||||
mode: str, *, cluster: Optional[LiveKitClusterConfig] = None
|
||||
) -> WorkerService:
|
||||
"""Instantiate a worker service by its mode, recording on a LiveKit cluster.
|
||||
|
||||
The worker service calls `cluster`, by default the default cluster as
|
||||
registered now, with the URL, API key and secret Django calls it with. The
|
||||
rest of its configuration comes from the settings, read once.
|
||||
"""
|
||||
|
||||
worker_registry: Dict[str, str] = settings.RECORDING_WORKER_CLASSES
|
||||
|
||||
@@ -139,5 +149,8 @@ def get_worker_service(mode: str) -> WorkerService:
|
||||
|
||||
worker_class: Type[WorkerService] = import_string(worker_class_path)
|
||||
|
||||
config = WorkerServiceConfig.from_settings()
|
||||
config = replace(
|
||||
WorkerServiceConfig.from_settings(),
|
||||
server_configurations=utils.livekit_api_configuration(cluster),
|
||||
)
|
||||
return worker_class(config=config)
|
||||
|
||||
@@ -72,7 +72,9 @@ class WorkerServiceMediator:
|
||||
|
||||
try:
|
||||
RoomManagement.update_metadata(
|
||||
room_name, {"recording_mode": mode, "recording_status": "starting"}
|
||||
room_name,
|
||||
{"recording_mode": mode, "recording_status": "starting"},
|
||||
cluster=recording.get_livekit_cluster(),
|
||||
)
|
||||
except RoomNotFoundException:
|
||||
logger.info(
|
||||
|
||||
@@ -16,6 +16,7 @@ from rest_framework import (
|
||||
from core import analytics, models
|
||||
from core.api import permissions, throttling
|
||||
from core.api.feature_flag import FeatureFlag
|
||||
from core.livekit.placements import PlacementService
|
||||
from core.services.sip_management import SIPException, SIPManagement
|
||||
|
||||
from . import authentication, serializers
|
||||
@@ -51,6 +52,10 @@ class RoomKitViewSet(viewsets.ViewSet):
|
||||
|
||||
The webhook-based creation path is kept: both converge on the same rule
|
||||
through the shared SIPManagement.
|
||||
|
||||
The device may be the room's first participant, so the LiveKit room is
|
||||
placed and created first, since LiveKit clusters need not create rooms on
|
||||
their own. When it cannot be, the response is a 503.
|
||||
"""
|
||||
|
||||
serializer = serializers.RoomKitJoinSerializer(data=request.data)
|
||||
@@ -63,6 +68,8 @@ class RoomKitViewSet(viewsets.ViewSet):
|
||||
except models.Room.DoesNotExist as e:
|
||||
raise drf_exceptions.NotFound("No room found for this PIN code.") from e
|
||||
|
||||
PlacementService.place(str(room.id), room)
|
||||
|
||||
try:
|
||||
created = SIPManagement().ensure_dispatch_rule(room)
|
||||
except SIPException as e:
|
||||
|
||||
@@ -13,6 +13,11 @@ from django.utils import timezone
|
||||
from livekit import api
|
||||
|
||||
from core import audit, auditing, models
|
||||
from core.livekit.clusters import (
|
||||
LiveKitClusterConfig,
|
||||
LiveKitClusterTokenVerifier,
|
||||
get_default_livekit_cluster,
|
||||
)
|
||||
from core.recording.enums import UNSUCCESSFUL_EVENTS, RecordingWorkerEvent
|
||||
from core.recording.event.authentication import MachineUser
|
||||
from core.recording.services.metadata_collector import (
|
||||
@@ -154,11 +159,7 @@ class LiveKitEventsService:
|
||||
"participant_left": self._handle_participant_left,
|
||||
}
|
||||
|
||||
token_verifier = api.TokenVerifier(
|
||||
settings.LIVEKIT_CONFIGURATION["api_key"],
|
||||
settings.LIVEKIT_CONFIGURATION["api_secret"],
|
||||
)
|
||||
self.webhook_receiver = api.WebhookReceiver(token_verifier)
|
||||
self.token_verifier = LiveKitClusterTokenVerifier()
|
||||
self.lobby_service = LobbyService()
|
||||
self.presence_cache = PresenceCache()
|
||||
self.sip_management = SIPManagement()
|
||||
@@ -175,15 +176,28 @@ class LiveKitEventsService:
|
||||
"Invalid LIVEKIT_WEBHOOK_EVENTS_FILTER_REGEX. Webhook filtering disabled."
|
||||
)
|
||||
|
||||
@property
|
||||
def cluster(self) -> LiveKitClusterConfig:
|
||||
"""Return the LiveKit cluster that signed the webhook received.
|
||||
|
||||
It is the default cluster until a webhook is received, as when a
|
||||
handler is called directly.
|
||||
"""
|
||||
return self.token_verifier.cluster or get_default_livekit_cluster()
|
||||
|
||||
def receive(self, request):
|
||||
"""Process webhook and route to appropriate handler."""
|
||||
"""Process webhook and route to appropriate handler.
|
||||
|
||||
The placement of a LiveKit room that finished is deleted first, for
|
||||
every room but connection test rooms, filtered out or not.
|
||||
"""
|
||||
|
||||
auth_token = request.headers.get("Authorization")
|
||||
if not auth_token:
|
||||
raise AuthenticationError("Authorization header missing")
|
||||
|
||||
try:
|
||||
data = self.webhook_receiver.receive(
|
||||
data = api.WebhookReceiver(self.token_verifier).receive(
|
||||
request.body.decode("utf-8"), auth_token
|
||||
)
|
||||
except Exception as e:
|
||||
@@ -198,7 +212,10 @@ class LiveKitEventsService:
|
||||
)
|
||||
return
|
||||
|
||||
if self._filter_regex and not self._filter_regex.search(room_name):
|
||||
if not self._follow_room_lifecycle(data, room_name):
|
||||
return
|
||||
|
||||
if self._is_filtered_out(room_name):
|
||||
logger.info("Filtered webhook event for room '%s'", room_name)
|
||||
return
|
||||
|
||||
@@ -218,6 +235,61 @@ class LiveKitEventsService:
|
||||
if handler is not None:
|
||||
handler(data)
|
||||
|
||||
def _is_filtered_out(self, room_name: str) -> bool:
|
||||
"""Tell whether LIVEKIT_WEBHOOK_EVENTS_FILTER_REGEX leaves a room out."""
|
||||
return self._filter_regex is not None and not self._filter_regex.search(
|
||||
room_name
|
||||
)
|
||||
|
||||
def _follow_room_lifecycle(self, data, room_name: str) -> bool:
|
||||
"""Keep the placement of a LiveKit room in step with its lifecycle.
|
||||
|
||||
Only the end of a LiveKit room whose SID the event carries is followed,
|
||||
on the LiveKit cluster that sent the webhook.
|
||||
|
||||
Return whether the event is to be handled further.
|
||||
"""
|
||||
sid = data.room.sid
|
||||
if not isinstance(sid, str) or not sid:
|
||||
return True
|
||||
|
||||
if data.event == LiveKitWebhookEventType.ROOM_FINISHED.value:
|
||||
return self._follow_finished_room(room_name, sid)
|
||||
return True
|
||||
|
||||
def _follow_finished_room(self, room_name: str, sid: str) -> bool:
|
||||
"""Delete the placement of a LiveKit room that finished.
|
||||
|
||||
Only its placement on the cluster that sent the webhook, with the SID
|
||||
of the finished LiveKit room, is deleted.
|
||||
|
||||
Return whether the room is to be cleaned up: when its placement was
|
||||
deleted, or when it has none. A placement on another cluster or with
|
||||
another SID points at the LiveKit room the room now uses, created since
|
||||
or the one it was relocated to, where participants admitted from the
|
||||
lobby must not be queued again: the event is then ignored.
|
||||
"""
|
||||
cluster_name = self.cluster.name
|
||||
placements = models.Placement.objects.filter(room_name=room_name)
|
||||
deleted, _ = placements.filter(cluster=cluster_name, sid=sid).delete()
|
||||
if deleted:
|
||||
return True
|
||||
|
||||
placement = placements.first()
|
||||
if placement is None:
|
||||
return True
|
||||
|
||||
logger.info(
|
||||
"LiveKit room %s finished on LiveKit cluster %s with SID %s, but is "
|
||||
"placed on LiveKit cluster %s with SID %r: ignoring the event",
|
||||
room_name,
|
||||
cluster_name,
|
||||
sid,
|
||||
placement.cluster,
|
||||
placement.sid,
|
||||
)
|
||||
return False
|
||||
|
||||
def _handle_egress_updated(self, data):
|
||||
"""Handle 'egress_updated' event."""
|
||||
|
||||
@@ -240,6 +312,10 @@ class LiveKitEventsService:
|
||||
|
||||
Egress ended is sent with one of these statuses:
|
||||
EGRESS_COMPLETE, EGRESS_FAILED, EGRESS_ABORTED, EGRESS_LIMIT_REACHED
|
||||
|
||||
The room's metadata is cleaned up, the metadata collector's dispatch
|
||||
deleted and participants notified on the LiveKit cluster recording the
|
||||
room, even once the room was placed elsewhere.
|
||||
"""
|
||||
|
||||
# Fetch recording
|
||||
@@ -268,7 +344,9 @@ class LiveKitEventsService:
|
||||
try:
|
||||
room_name = str(recording.room.id)
|
||||
RoomManagement.update_metadata(
|
||||
room_name, remove_keys=["recording_mode", "recording_status"]
|
||||
room_name,
|
||||
remove_keys=["recording_mode", "recording_status"],
|
||||
cluster=recording.get_livekit_cluster(),
|
||||
)
|
||||
except RoomNotFoundException:
|
||||
logger.info(
|
||||
@@ -354,12 +432,12 @@ class LiveKitEventsService:
|
||||
f"Failed to create sip dispatch rule for room {room_id}"
|
||||
) from e
|
||||
|
||||
@staticmethod
|
||||
def _close_deleted_room(room_id):
|
||||
def _close_deleted_room(self, room_id):
|
||||
"""Close a LiveKit room recreated after its room was soft deleted.
|
||||
|
||||
LiveKit auto-creates a room on join, so a participant still holding a
|
||||
valid token can bring a deleted room back to life until the token expires.
|
||||
The LiveKit room is closed on the LiveKit cluster it started on.
|
||||
"""
|
||||
|
||||
logger.warning(
|
||||
@@ -367,7 +445,7 @@ class LiveKitEventsService:
|
||||
)
|
||||
|
||||
try:
|
||||
RoomManagement.delete_room(str(room_id))
|
||||
RoomManagement.delete_room(str(room_id), cluster=self.cluster)
|
||||
except RoomNotFoundException:
|
||||
logger.info("LiveKit room %s is already closed", room_id)
|
||||
except RoomManagementException as e:
|
||||
|
||||
@@ -12,6 +12,7 @@ from django.core.cache import cache
|
||||
from django.utils import timezone
|
||||
|
||||
from core import models, utils
|
||||
from core.livekit.placements import PlacementService
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -176,6 +177,10 @@ class LobbyService:
|
||||
3. If unknown, add to waiting list
|
||||
4. If accepted, generate LiveKit config
|
||||
5. If denied, do nothing.
|
||||
|
||||
The LiveKit config is for the LiveKit cluster the LiveKit room is
|
||||
placed on, where it is created first. When it cannot be,
|
||||
LiveKitRoomUnavailableError is raised, and the lobby left as it was.
|
||||
"""
|
||||
|
||||
participant = None
|
||||
@@ -190,6 +195,7 @@ class LobbyService:
|
||||
user_role = room.get_role(user)
|
||||
|
||||
if self.can_bypass_lobby(room=room, user=user, role=user_role):
|
||||
cluster = PlacementService.place(room_id, room)
|
||||
if not is_new_participant:
|
||||
self.clear_participant_cache(room.id, participant.id)
|
||||
participant.status = LobbyParticipantStatus.ACCEPTED
|
||||
@@ -202,6 +208,7 @@ class LobbyService:
|
||||
configuration=room.configuration,
|
||||
participant_id=participant.id,
|
||||
role=user_role,
|
||||
cluster=cluster,
|
||||
)
|
||||
return participant, livekit_config
|
||||
|
||||
@@ -215,6 +222,7 @@ class LobbyService:
|
||||
self.refresh_waiting_status(room.id, participant.id)
|
||||
|
||||
elif participant.status == LobbyParticipantStatus.ACCEPTED:
|
||||
cluster = PlacementService.place(room_id, room)
|
||||
# wrongly named, contains access token to join a room
|
||||
livekit_config = utils.generate_livekit_config(
|
||||
room_id=room_id,
|
||||
@@ -224,6 +232,7 @@ class LobbyService:
|
||||
configuration=room.configuration,
|
||||
participant_id=participant.id,
|
||||
role=user_role,
|
||||
cluster=cluster,
|
||||
)
|
||||
|
||||
return participant, livekit_config
|
||||
|
||||
@@ -19,6 +19,8 @@ from livekit.api import (
|
||||
from livekit.protocol.models import ParticipantInfo
|
||||
|
||||
from core import utils
|
||||
from core.livekit.clusters import LiveKitClusterConfig
|
||||
from core.livekit.placements import PlacementService
|
||||
|
||||
from .lobby import LobbyService
|
||||
from .presence import PresenceCache
|
||||
@@ -35,13 +37,37 @@ class ParticipantNotFoundException(ParticipantsManagementException):
|
||||
|
||||
|
||||
class ParticipantsManagement:
|
||||
"""Service for managing participants."""
|
||||
"""Service for managing participants.
|
||||
|
||||
@async_to_sync
|
||||
async def mute(self, room_name: str, identity: str, track_sid: str):
|
||||
A room's LiveKit room is reached on the LiveKit cluster given as `cluster`,
|
||||
by default the one it is placed on.
|
||||
"""
|
||||
|
||||
def mute(
|
||||
self,
|
||||
room_name: str,
|
||||
identity: str,
|
||||
track_sid: str,
|
||||
*,
|
||||
cluster: Optional[LiveKitClusterConfig] = None,
|
||||
):
|
||||
"""Mute a specific audio or video track for a participant in a room."""
|
||||
|
||||
lkapi = utils.create_livekit_client()
|
||||
cluster = cluster or PlacementService.cluster_for(room_name)
|
||||
self._mute(room_name, identity, track_sid, cluster=cluster)
|
||||
|
||||
@async_to_sync
|
||||
async def _mute(
|
||||
self,
|
||||
room_name: str,
|
||||
identity: str,
|
||||
track_sid: str,
|
||||
*,
|
||||
cluster: LiveKitClusterConfig,
|
||||
):
|
||||
"""Mute a participant's track in a LiveKit room on a LiveKit cluster."""
|
||||
|
||||
lkapi = utils.create_livekit_client(cluster=cluster)
|
||||
|
||||
try:
|
||||
await lkapi.room.mute_published_track(
|
||||
@@ -72,10 +98,24 @@ class ParticipantsManagement:
|
||||
finally:
|
||||
await lkapi.aclose()
|
||||
|
||||
@async_to_sync
|
||||
async def remove(self, room_name: str, identity: str):
|
||||
def remove(
|
||||
self,
|
||||
room_name: str,
|
||||
identity: str,
|
||||
*,
|
||||
cluster: Optional[LiveKitClusterConfig] = None,
|
||||
):
|
||||
"""Remove a participant from a room and clear their lobby/presence cache."""
|
||||
|
||||
cluster = cluster or PlacementService.cluster_for(room_name)
|
||||
self._remove(room_name, identity, cluster=cluster)
|
||||
|
||||
@async_to_sync
|
||||
async def _remove(
|
||||
self, room_name: str, identity: str, *, cluster: LiveKitClusterConfig
|
||||
):
|
||||
"""Remove a participant from a LiveKit room on a LiveKit cluster."""
|
||||
|
||||
PresenceCache().clear(room_name, identity)
|
||||
|
||||
try:
|
||||
@@ -90,7 +130,7 @@ class ParticipantsManagement:
|
||||
exc_info=exc,
|
||||
)
|
||||
|
||||
lkapi = utils.create_livekit_client()
|
||||
lkapi = utils.create_livekit_client(cluster=cluster)
|
||||
|
||||
try:
|
||||
await lkapi.room.remove_participant(
|
||||
@@ -115,8 +155,7 @@ class ParticipantsManagement:
|
||||
finally:
|
||||
await lkapi.aclose()
|
||||
|
||||
@async_to_sync
|
||||
async def update( # noqa: PLR0917
|
||||
def update( # noqa: PLR0917
|
||||
self,
|
||||
room_name: str,
|
||||
identity: str,
|
||||
@@ -124,10 +163,37 @@ class ParticipantsManagement:
|
||||
attributes: Optional[Dict] = None,
|
||||
permission: Optional[Dict] = None,
|
||||
name: Optional[str] = None,
|
||||
*,
|
||||
cluster: Optional[LiveKitClusterConfig] = None,
|
||||
):
|
||||
"""Update participant properties such as metadata, attributes, permissions, or name."""
|
||||
|
||||
lkapi = utils.create_livekit_client()
|
||||
cluster = cluster or PlacementService.cluster_for(room_name)
|
||||
self._update(
|
||||
room_name,
|
||||
identity,
|
||||
cluster=cluster,
|
||||
metadata=metadata,
|
||||
attributes=attributes,
|
||||
permission=permission,
|
||||
name=name,
|
||||
)
|
||||
|
||||
@async_to_sync
|
||||
async def _update(
|
||||
self,
|
||||
room_name: str,
|
||||
identity: str,
|
||||
*,
|
||||
cluster: LiveKitClusterConfig,
|
||||
metadata: Optional[Dict],
|
||||
attributes: Optional[Dict],
|
||||
permission: Optional[Dict],
|
||||
name: Optional[str],
|
||||
):
|
||||
"""Update a participant of a LiveKit room on a LiveKit cluster."""
|
||||
|
||||
lkapi = utils.create_livekit_client(cluster=cluster)
|
||||
|
||||
try:
|
||||
await lkapi.room.update_participant(
|
||||
@@ -184,8 +250,13 @@ class ParticipantsManagement:
|
||||
|
||||
return present
|
||||
|
||||
@async_to_sync
|
||||
async def check_if_in_meeting(self, room_name: str, identity: str) -> bool:
|
||||
def check_if_in_meeting(
|
||||
self,
|
||||
room_name: str,
|
||||
identity: str,
|
||||
*,
|
||||
cluster: Optional[LiveKitClusterConfig] = None,
|
||||
) -> bool:
|
||||
"""Check whether `identity` is currently a participant in `room_name`.
|
||||
|
||||
Raises ParticipantsManagementException for unexpected LiveKit errors
|
||||
@@ -195,7 +266,16 @@ class ParticipantsManagement:
|
||||
if not room_name or not identity:
|
||||
return False
|
||||
|
||||
lkapi = utils.create_livekit_client()
|
||||
cluster = cluster or PlacementService.cluster_for(room_name)
|
||||
return self._check_if_in_meeting(room_name, identity, cluster=cluster)
|
||||
|
||||
@async_to_sync
|
||||
async def _check_if_in_meeting(
|
||||
self, room_name: str, identity: str, *, cluster: LiveKitClusterConfig
|
||||
) -> bool:
|
||||
"""Check whether `identity` is a participant of a LiveKit room on a cluster."""
|
||||
|
||||
lkapi = utils.create_livekit_client(cluster=cluster)
|
||||
|
||||
try:
|
||||
participant = await lkapi.room.get_participant(
|
||||
|
||||
@@ -19,6 +19,8 @@ from livekit.api import (
|
||||
)
|
||||
|
||||
from core import utils
|
||||
from core.livekit.clusters import LiveKitClusterConfig
|
||||
from core.livekit.placements import PlacementService
|
||||
from core.models import Room
|
||||
|
||||
from .sip_management import SIPException, SIPManagement
|
||||
@@ -38,24 +40,41 @@ class RoomManagement:
|
||||
"""Service for managing LiveKit rooms."""
|
||||
|
||||
@classmethod
|
||||
@async_to_sync
|
||||
async def update_metadata(
|
||||
def update_metadata(
|
||||
cls,
|
||||
room_name: str,
|
||||
metadata: Optional[Dict] = None,
|
||||
remove_keys: Optional[list[str]] = None,
|
||||
*,
|
||||
cluster: Optional[LiveKitClusterConfig] = None,
|
||||
):
|
||||
"""Merge values into a LiveKit room's metadata.
|
||||
|
||||
The `room_name` corresponds to the LiveKit room identifier
|
||||
(i.e. the Room model's UUID as a string).
|
||||
(i.e. the Room model's UUID as a string). The LiveKit room is
|
||||
updated on `cluster`, by default the LiveKit cluster it is placed on.
|
||||
|
||||
Raises:
|
||||
RoomNotFoundException: the room does not exist in LiveKit.
|
||||
RoomManagementException: the metadata update otherwise fails.
|
||||
"""
|
||||
|
||||
lkapi = utils.create_livekit_client()
|
||||
cluster = cluster or PlacementService.cluster_for(room_name)
|
||||
cls._update_metadata(room_name, metadata, remove_keys, cluster=cluster)
|
||||
|
||||
@classmethod
|
||||
@async_to_sync
|
||||
async def _update_metadata(
|
||||
cls,
|
||||
room_name: str,
|
||||
metadata: Optional[Dict],
|
||||
remove_keys: Optional[list[str]],
|
||||
*,
|
||||
cluster: LiveKitClusterConfig,
|
||||
):
|
||||
"""Merge values into a LiveKit room's metadata on a LiveKit cluster."""
|
||||
|
||||
lkapi = utils.create_livekit_client(cluster=cluster)
|
||||
|
||||
try:
|
||||
response = await lkapi.room.list_rooms(ListRoomsRequest(names=[room_name]))
|
||||
@@ -95,16 +114,28 @@ class RoomManagement:
|
||||
await lkapi.aclose()
|
||||
|
||||
@classmethod
|
||||
@async_to_sync
|
||||
async def delete_room(cls, room_name: str):
|
||||
def delete_room(
|
||||
cls, room_name: str, *, cluster: Optional[LiveKitClusterConfig] = None
|
||||
):
|
||||
"""Delete a LiveKit room and disconnect all participants.
|
||||
|
||||
The LiveKit room is deleted on `cluster`, by default the LiveKit
|
||||
cluster it is placed on.
|
||||
|
||||
Raises:
|
||||
RoomNotFoundException: the room does not exist in LiveKit.
|
||||
RoomManagementException: the deletion otherwise fails.
|
||||
"""
|
||||
|
||||
lkapi = utils.create_livekit_client()
|
||||
cluster = cluster or PlacementService.cluster_for(room_name)
|
||||
cls._delete_room(room_name, cluster=cluster)
|
||||
|
||||
@classmethod
|
||||
@async_to_sync
|
||||
async def _delete_room(cls, room_name: str, *, cluster: LiveKitClusterConfig):
|
||||
"""Delete a LiveKit room on a LiveKit cluster."""
|
||||
|
||||
lkapi = utils.create_livekit_client(cluster=cluster)
|
||||
|
||||
try:
|
||||
await lkapi.room.delete_room(DeleteRoomRequest(room=room_name))
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Service for managing subtitle agents in LiveKit rooms."""
|
||||
|
||||
from logging import getLogger
|
||||
from typing import Optional
|
||||
|
||||
from django.conf import settings
|
||||
|
||||
@@ -8,6 +9,8 @@ from asgiref.sync import async_to_sync
|
||||
from livekit.protocol.agent_dispatch import CreateAgentDispatchRequest
|
||||
|
||||
from core import utils
|
||||
from core.livekit.clusters import LiveKitClusterConfig
|
||||
from core.livekit.placements import PlacementService
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
@@ -19,11 +22,21 @@ class SubtitleException(Exception):
|
||||
class SubtitleService:
|
||||
"""Service for managing subtitle agents in LiveKit rooms."""
|
||||
|
||||
@async_to_sync
|
||||
async def start_subtitle(self, room):
|
||||
"""Start subtitle agent for the specified room."""
|
||||
def start_subtitle(self, room, *, cluster: Optional[LiveKitClusterConfig] = None):
|
||||
"""Start subtitle agent for the specified room.
|
||||
|
||||
lkapi = utils.create_livekit_client()
|
||||
The agent is dispatched on `cluster`, by default the LiveKit cluster the
|
||||
room's LiveKit room is placed on.
|
||||
"""
|
||||
|
||||
cluster = cluster or PlacementService.cluster_for(str(room.id))
|
||||
self._start_subtitle(room, cluster=cluster)
|
||||
|
||||
@async_to_sync
|
||||
async def _start_subtitle(self, room, *, cluster: LiveKitClusterConfig):
|
||||
"""Dispatch the subtitle agent to a room's LiveKit room on a cluster."""
|
||||
|
||||
lkapi = utils.create_livekit_client(cluster=cluster)
|
||||
|
||||
try:
|
||||
# Transcriber agent prevents duplicate subtitle agents per room
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
"""Tasks related to connection test rooms."""
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
from django.conf import settings
|
||||
|
||||
from core.livekit.clusters import get_default_livekit_cluster, get_livekit_cluster
|
||||
from core.livekit.exceptions import UnknownLiveKitClusterError
|
||||
from core.services.room_management import (
|
||||
RoomManagement,
|
||||
RoomManagementException,
|
||||
@@ -15,11 +18,16 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@task
|
||||
def delete_connection_test_room(room_name: str):
|
||||
def delete_connection_test_room(room_name: str, cluster_name: Optional[str] = None):
|
||||
"""Force-delete an ephemeral connection-test room.
|
||||
|
||||
Used as a hard cap so a participant cannot keep an auto-refreshed
|
||||
LiveKit session open indefinitely after requesting a test token.
|
||||
|
||||
The room is deleted on the LiveKit cluster named `cluster_name`, where it
|
||||
was created. Without a cluster name, as in tasks queued before connection
|
||||
test rooms were created on a chosen cluster, and when the cluster is no
|
||||
longer registered, it is deleted on the default cluster.
|
||||
"""
|
||||
prefix = settings.CONNECTION_TEST_ROOM_PREFIX
|
||||
if not room_name.startswith(prefix):
|
||||
@@ -30,8 +38,23 @@ def delete_connection_test_room(room_name: str):
|
||||
)
|
||||
return
|
||||
|
||||
if cluster_name is None:
|
||||
cluster = get_default_livekit_cluster()
|
||||
else:
|
||||
try:
|
||||
cluster = get_livekit_cluster(cluster_name)
|
||||
except UnknownLiveKitClusterError:
|
||||
cluster = get_default_livekit_cluster()
|
||||
logger.warning(
|
||||
"Connection test room '%s' was created on LiveKit cluster %s, "
|
||||
"which is not registered: deleting it on the default cluster %s.",
|
||||
room_name,
|
||||
cluster_name,
|
||||
cluster.name,
|
||||
)
|
||||
|
||||
try:
|
||||
RoomManagement.delete_room(room_name)
|
||||
RoomManagement.delete_room(room_name, cluster=cluster)
|
||||
except RoomNotFoundException:
|
||||
# Room may already be gone after empty/departure timeout.
|
||||
logger.info("Connection test room '%s' already gone.", room_name)
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Unit tests for the LiveKit token authentication."""
|
||||
|
||||
import pytest
|
||||
from livekit.api.access_token import Claims
|
||||
from rest_framework.test import APIRequestFactory
|
||||
|
||||
from core.authentication.livekit import LiveKitTokenAuthentication
|
||||
from core.factories import UserFactory
|
||||
from core.utils import generate_token
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
|
||||
|
||||
def test_authenticate_token_from_another_cluster(two_livekit_clusters):
|
||||
"""A token signed by any registered LiveKit cluster authenticates its user.
|
||||
|
||||
The request's auth stays the token's claims, which carry that cluster.
|
||||
"""
|
||||
_paris, lyon = two_livekit_clusters
|
||||
user = UserFactory()
|
||||
token = generate_token(room="my-room", user=user, cluster=lyon)
|
||||
request = APIRequestFactory().post(
|
||||
"/", HTTP_AUTHORIZATION=f"X-LiveKit-Token {token}"
|
||||
)
|
||||
|
||||
authenticated_user, claims = LiveKitTokenAuthentication().authenticate(request)
|
||||
|
||||
assert authenticated_user == user
|
||||
assert isinstance(claims, Claims)
|
||||
assert claims.identity == str(user.sub)
|
||||
assert claims.video.room == "my-room"
|
||||
assert claims.cluster == lyon
|
||||
@@ -1,17 +1,30 @@
|
||||
"""Fixtures for tests in the Meet core application"""
|
||||
|
||||
import json
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
from dockerflow.logging import request_id_context
|
||||
|
||||
from core.audit.testing import capture_audit
|
||||
from core.livekit.clusters import get_livekit_clusters
|
||||
from core.livekit.placements import PlacementService
|
||||
from core.tests.utils import CREATED_LIVEKIT_ROOM_SID, livekit_cluster_entry
|
||||
|
||||
USER = "user"
|
||||
TEAM = "team"
|
||||
VIA = [USER, TEAM]
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
"""Register the markers of the tests in the Meet core application."""
|
||||
config.addinivalue_line(
|
||||
"markers",
|
||||
"real_livekit_room_creation: create LiveKit rooms through the LiveKit "
|
||||
"API rather than the fake_livekit_room_creation fixture",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_user_get_teams():
|
||||
"""Mock for the "get_teams" method on the User model."""
|
||||
@@ -26,6 +39,27 @@ def audit_events():
|
||||
yield events
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def fake_livekit_room_creation(request):
|
||||
"""Create LiveKit rooms without calling LiveKit.
|
||||
|
||||
Credentials are handed out only once the LiveKit room is created on its
|
||||
cluster, so every join would otherwise call LiveKit. Each LiveKit room
|
||||
created gets CREATED_LIVEKIT_ROOM_SID as SID. Return the mock standing for
|
||||
PlacementService.create_livekit_room, or None in tests marked
|
||||
real_livekit_room_creation, which go through the real one.
|
||||
"""
|
||||
if request.node.get_closest_marker("real_livekit_room_creation"):
|
||||
yield None
|
||||
return
|
||||
with mock.patch.object(
|
||||
PlacementService,
|
||||
"create_livekit_room",
|
||||
return_value=CREATED_LIVEKIT_ROOM_SID,
|
||||
) as create_livekit_room:
|
||||
yield create_livekit_room
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def isolated_request_id():
|
||||
"""Keep dockerflow's request id from leaking from one test to the next.
|
||||
@@ -37,3 +71,15 @@ def isolated_request_id():
|
||||
token = request_id_context.set(None)
|
||||
yield
|
||||
request_id_context.reset(token)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def two_livekit_clusters(settings):
|
||||
"""Register the Paris and Lyon LiveKit clusters, Paris being the default.
|
||||
|
||||
Each has its own API key and secret. Return both clusters, Paris first.
|
||||
"""
|
||||
settings.LIVEKIT_CLUSTERS = json.dumps(
|
||||
[livekit_cluster_entry("paris"), livekit_cluster_entry("lyon")]
|
||||
)
|
||||
return get_livekit_clusters()
|
||||
|
||||
@@ -0,0 +1,260 @@
|
||||
"""Unit tests for the registry of LiveKit clusters."""
|
||||
|
||||
import dataclasses
|
||||
import json
|
||||
import time
|
||||
from datetime import timedelta
|
||||
|
||||
import jwt
|
||||
import pytest
|
||||
from livekit.api import AccessToken, VideoGrants
|
||||
from livekit.api.access_token import Claims
|
||||
|
||||
from core.livekit.clusters import (
|
||||
LiveKitClusterConfig,
|
||||
LiveKitClusterTokenVerifier,
|
||||
get_default_livekit_cluster,
|
||||
get_livekit_cluster,
|
||||
get_livekit_cluster_by_api_key,
|
||||
get_livekit_clusters,
|
||||
)
|
||||
from core.livekit.exceptions import UnknownLiveKitClusterError
|
||||
from core.tests.utils import livekit_cluster_entry
|
||||
|
||||
PARIS = LiveKitClusterConfig(
|
||||
name="paris",
|
||||
url="wss://paris.livekit.test",
|
||||
api_url="wss://paris.livekit.test",
|
||||
api_key="key-paris",
|
||||
api_secret="secret-paris-padded-for-minimum-len!-livekit",
|
||||
)
|
||||
LYON = LiveKitClusterConfig(
|
||||
name="lyon",
|
||||
url="wss://lyon.livekit.test",
|
||||
api_url="http://livekit.lyon.svc:7880",
|
||||
api_key="key-lyon",
|
||||
api_secret="secret-lyon-padded-for-minimum-len!-livekit",
|
||||
weight=0,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def two_clusters(settings):
|
||||
"""Register the Paris and Lyon LiveKit clusters, Paris being the default."""
|
||||
settings.LIVEKIT_CLUSTERS = json.dumps(
|
||||
[
|
||||
livekit_cluster_entry("paris"),
|
||||
livekit_cluster_entry(
|
||||
"lyon",
|
||||
api_url="http://livekit.lyon.svc:7880",
|
||||
weight=0,
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_get_livekit_clusters_from_livekit_configuration(settings):
|
||||
"""Without LIVEKIT_CLUSTERS, LIVEKIT_CONFIGURATION describes a single cluster.
|
||||
|
||||
It is named "default" and its URL serves both browsers and Django.
|
||||
"""
|
||||
configuration = settings.LIVEKIT_CONFIGURATION
|
||||
|
||||
assert settings.LIVEKIT_CLUSTERS is None
|
||||
assert get_livekit_clusters() == (
|
||||
LiveKitClusterConfig(
|
||||
name="default",
|
||||
url=configuration["url"],
|
||||
api_url=configuration["url"],
|
||||
api_key=configuration["api_key"],
|
||||
api_secret=configuration["api_secret"],
|
||||
weight=1,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("two_clusters")
|
||||
def test_get_livekit_clusters_from_livekit_clusters():
|
||||
"""LIVEKIT_CLUSTERS lists the clusters in order, api_url defaulting to url."""
|
||||
assert get_livekit_clusters() == (PARIS, LYON)
|
||||
|
||||
|
||||
def test_get_livekit_clusters_follows_settings_changes(settings):
|
||||
"""The clusters are cached on the raw settings, so a change applies at once."""
|
||||
clusters = get_livekit_clusters()
|
||||
assert get_livekit_clusters() is clusters
|
||||
|
||||
settings.LIVEKIT_CONFIGURATION = {
|
||||
**settings.LIVEKIT_CONFIGURATION,
|
||||
"url": "wss://elsewhere.livekit.test",
|
||||
}
|
||||
assert get_livekit_clusters()[0].url == "wss://elsewhere.livekit.test"
|
||||
|
||||
settings.LIVEKIT_CLUSTERS = json.dumps([livekit_cluster_entry("paris")])
|
||||
assert get_livekit_clusters() == (PARIS,)
|
||||
|
||||
settings.LIVEKIT_CLUSTERS = json.dumps([livekit_cluster_entry("lille")])
|
||||
assert [cluster.name for cluster in get_livekit_clusters()] == ["lille"]
|
||||
|
||||
|
||||
def test_get_livekit_clusters_invalid_livekit_clusters(settings):
|
||||
"""An invalid LIVEKIT_CLUSTERS is refused at runtime as it is at startup."""
|
||||
settings.LIVEKIT_CLUSTERS = "[]"
|
||||
|
||||
with pytest.raises(ValueError, match="LIVEKIT_CLUSTERS"):
|
||||
get_livekit_clusters()
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("two_clusters")
|
||||
def test_get_default_livekit_cluster():
|
||||
"""The default cluster is the first one listed."""
|
||||
assert get_default_livekit_cluster() == PARIS
|
||||
|
||||
|
||||
def test_get_default_livekit_cluster_from_livekit_configuration():
|
||||
"""Without LIVEKIT_CLUSTERS, the default cluster is the one named "default"."""
|
||||
assert get_default_livekit_cluster() == get_livekit_clusters()[0]
|
||||
assert get_default_livekit_cluster().name == "default"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("two_clusters")
|
||||
def test_get_livekit_cluster():
|
||||
"""A cluster is found by its name."""
|
||||
assert get_livekit_cluster("lyon") == LYON
|
||||
assert get_livekit_cluster("paris") == PARIS
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("two_clusters")
|
||||
def test_get_livekit_cluster_unknown():
|
||||
"""An unknown name raises an error naming it."""
|
||||
with pytest.raises(
|
||||
UnknownLiveKitClusterError, match="No LiveKit cluster is named 'default'."
|
||||
):
|
||||
get_livekit_cluster("default")
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("two_clusters")
|
||||
def test_get_livekit_cluster_by_api_key():
|
||||
"""A cluster is found by its API key."""
|
||||
assert get_livekit_cluster_by_api_key("key-lyon") == LYON
|
||||
assert get_livekit_cluster_by_api_key("key-paris") == PARIS
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("two_clusters")
|
||||
@pytest.mark.parametrize("api_key", ["key-lille", "KEY-LYON", "", None])
|
||||
def test_get_livekit_cluster_by_api_key_unknown(api_key):
|
||||
"""An API key no cluster uses finds none."""
|
||||
assert get_livekit_cluster_by_api_key(api_key) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("api_key", ["", None])
|
||||
def test_get_livekit_cluster_by_api_key_unset(settings, api_key):
|
||||
"""A missing API key finds no cluster, not even one whose key is unset."""
|
||||
settings.LIVEKIT_CONFIGURATION = {
|
||||
"url": None,
|
||||
"api_key": api_key,
|
||||
"api_secret": None,
|
||||
}
|
||||
|
||||
assert get_livekit_clusters()[0].api_key == api_key
|
||||
assert get_livekit_cluster_by_api_key(api_key) is None
|
||||
|
||||
|
||||
def test_livekit_cluster_config_hides_the_api_secret():
|
||||
"""The API secret stays out of the representation, which may end up in logs."""
|
||||
assert PARIS.api_secret not in repr(PARIS)
|
||||
assert PARIS.api_secret not in str(PARIS)
|
||||
assert "key-paris" in repr(PARIS)
|
||||
|
||||
|
||||
def test_livekit_cluster_config_is_frozen():
|
||||
"""A cluster is shared between callers, so it cannot be changed."""
|
||||
with pytest.raises(dataclasses.FrozenInstanceError):
|
||||
PARIS.url = "wss://elsewhere.livekit.test"
|
||||
|
||||
|
||||
def livekit_token(api_key, api_secret, ttl_minutes=60):
|
||||
"""Return a participant token signed with this API key and secret."""
|
||||
return (
|
||||
AccessToken(api_key=api_key, api_secret=api_secret)
|
||||
.with_identity("participant-1")
|
||||
.with_grants(VideoGrants(room="my-room", room_join=True))
|
||||
.with_ttl(timedelta(minutes=ttl_minutes))
|
||||
.to_jwt()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("two_clusters")
|
||||
@pytest.mark.parametrize("cluster", [PARIS, LYON], ids=["paris", "lyon"])
|
||||
def test_livekit_cluster_token_verifier(cluster):
|
||||
"""A token signed by any registered cluster is verified with its keys.
|
||||
|
||||
The claims carry that cluster, and so does the verifier.
|
||||
"""
|
||||
verifier = LiveKitClusterTokenVerifier()
|
||||
|
||||
claims = verifier.verify(livekit_token(cluster.api_key, cluster.api_secret))
|
||||
|
||||
assert isinstance(claims, Claims)
|
||||
assert claims.identity == "participant-1"
|
||||
assert claims.video.room == "my-room"
|
||||
assert claims.cluster == cluster
|
||||
assert verifier.cluster == cluster
|
||||
|
||||
|
||||
def test_livekit_cluster_token_verifier_from_livekit_configuration(settings):
|
||||
"""Without LIVEKIT_CLUSTERS, tokens are verified with LIVEKIT_CONFIGURATION."""
|
||||
configuration = settings.LIVEKIT_CONFIGURATION
|
||||
|
||||
claims = LiveKitClusterTokenVerifier().verify(
|
||||
livekit_token(configuration["api_key"], configuration["api_secret"])
|
||||
)
|
||||
|
||||
assert claims.cluster == get_default_livekit_cluster()
|
||||
assert claims.cluster.name == "default"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("two_clusters")
|
||||
def test_livekit_cluster_token_verifier_unknown_api_key():
|
||||
"""A token whose API key no cluster uses is refused as having a wrong issuer."""
|
||||
verifier = LiveKitClusterTokenVerifier()
|
||||
token = livekit_token("key-lille", "secret-lille-padded-for-minimum-len!-livekit")
|
||||
|
||||
with pytest.raises(jwt.InvalidIssuerError, match="Invalid issuer"):
|
||||
verifier.verify(token)
|
||||
|
||||
assert verifier.cluster is None
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("two_clusters")
|
||||
def test_livekit_cluster_token_verifier_missing_issuer():
|
||||
"""A token without issuer names no cluster and is refused."""
|
||||
token = jwt.encode(
|
||||
{"sub": "participant-1", "exp": int(time.time()) + 3600},
|
||||
PARIS.api_secret,
|
||||
algorithm="HS256",
|
||||
)
|
||||
|
||||
with pytest.raises(jwt.MissingRequiredClaimError, match='"iss"'):
|
||||
LiveKitClusterTokenVerifier().verify(token)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("two_clusters")
|
||||
def test_livekit_cluster_token_verifier_signed_by_another_cluster():
|
||||
"""A token issued with the API key of a cluster must be signed with its secret.
|
||||
|
||||
A cluster cannot sign tokens in the name of another one.
|
||||
"""
|
||||
token = livekit_token(LYON.api_key, PARIS.api_secret)
|
||||
|
||||
with pytest.raises(jwt.InvalidSignatureError):
|
||||
LiveKitClusterTokenVerifier().verify(token)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("two_clusters")
|
||||
def test_livekit_cluster_token_verifier_expired():
|
||||
"""A token from a registered cluster is still checked as any token is."""
|
||||
token = livekit_token(LYON.api_key, LYON.api_secret, ttl_minutes=-5)
|
||||
|
||||
with pytest.raises(jwt.ExpiredSignatureError):
|
||||
LiveKitClusterTokenVerifier().verify(token)
|
||||
@@ -0,0 +1,889 @@
|
||||
"""Unit tests for the placements of LiveKit rooms on the LiveKit clusters."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from unittest import mock
|
||||
|
||||
from django.db.models import F
|
||||
from django.utils import timezone
|
||||
|
||||
import aiohttp
|
||||
import pytest
|
||||
from livekit.api import TwirpError
|
||||
from livekit.protocol.models import Room as LiveKitRoom # pylint: disable=E0611
|
||||
from livekit.protocol.room import CreateRoomRequest # pylint: disable=E0611
|
||||
|
||||
from core import utils
|
||||
from core.factories import RoomFactory
|
||||
from core.livekit.clusters import get_default_livekit_cluster
|
||||
from core.livekit.exceptions import (
|
||||
LiveKitRoomCreationError,
|
||||
NoCandidateLiveKitClusterError,
|
||||
PlacementChangedError,
|
||||
)
|
||||
from core.livekit.placements import (
|
||||
PLACEMENT_CREATION_ATTEMPTS,
|
||||
PLACEMENT_UPDATE_ATTEMPTS,
|
||||
PlacementService,
|
||||
)
|
||||
from core.livekit.strategies import PlacementContext, WeightedRandomStrategy
|
||||
from core.models import Placement
|
||||
from core.tests.utils import CREATED_LIVEKIT_ROOM_SID, livekit_cluster_entry
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
|
||||
|
||||
def test_cluster_for_single_cluster(django_assert_num_queries):
|
||||
"""With a single cluster, every room lives there and placements are not read."""
|
||||
Placement.objects.create(room_name="my-room", cluster="lyon")
|
||||
|
||||
with django_assert_num_queries(0):
|
||||
cluster = PlacementService.cluster_for("my-room")
|
||||
|
||||
assert cluster == get_default_livekit_cluster()
|
||||
assert cluster.name == "default"
|
||||
|
||||
|
||||
def test_cluster_for_single_listed_cluster(settings, django_assert_num_queries):
|
||||
"""A single cluster listed in LIVEKIT_CLUSTERS is the only one too."""
|
||||
settings.LIVEKIT_CLUSTERS = json.dumps([livekit_cluster_entry("paris")])
|
||||
Placement.objects.create(room_name="my-room", cluster="lyon")
|
||||
|
||||
with django_assert_num_queries(0):
|
||||
cluster = PlacementService.cluster_for("my-room")
|
||||
|
||||
assert cluster.name == "paris"
|
||||
|
||||
|
||||
def test_cluster_for_placed_room(two_livekit_clusters, django_assert_num_queries):
|
||||
"""A placed room lives on the cluster of its placement."""
|
||||
_paris, lyon = two_livekit_clusters
|
||||
Placement.objects.create(room_name="my-room", cluster="lyon")
|
||||
Placement.objects.create(room_name="other-room", cluster="paris")
|
||||
|
||||
with django_assert_num_queries(1):
|
||||
cluster = PlacementService.cluster_for("my-room")
|
||||
|
||||
assert cluster == lyon
|
||||
|
||||
|
||||
def test_cluster_for_room_without_placement(two_livekit_clusters):
|
||||
"""A room without placement is managed on the default cluster.
|
||||
|
||||
It may have been live since before placements existed, or not be placed yet.
|
||||
"""
|
||||
paris, _lyon = two_livekit_clusters
|
||||
Placement.objects.create(room_name="other-room", cluster="lyon")
|
||||
|
||||
assert PlacementService.cluster_for("my-room") == paris
|
||||
|
||||
|
||||
def test_cluster_for_unregistered_cluster(two_livekit_clusters, caplog):
|
||||
"""A room placed on a cluster since removed from the registry is managed on
|
||||
the default cluster, with a warning."""
|
||||
paris, _lyon = two_livekit_clusters
|
||||
Placement.objects.create(room_name="my-room", cluster="lille")
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="core.livekit.placements"):
|
||||
cluster = PlacementService.cluster_for("my-room")
|
||||
|
||||
assert cluster == paris
|
||||
assert (
|
||||
"LiveKit room my-room is placed on LiveKit cluster lille, which is not "
|
||||
"registered: managing it on the default cluster paris"
|
||||
) in caplog.text
|
||||
|
||||
|
||||
def test_candidates(two_livekit_clusters):
|
||||
"""Every registered cluster may receive new placements, in registry order."""
|
||||
assert PlacementService.candidates() == two_livekit_clusters
|
||||
|
||||
|
||||
def test_candidates_weighing_zero(settings):
|
||||
"""Clusters weighing 0 receive no new placements."""
|
||||
settings.LIVEKIT_CLUSTERS = json.dumps(
|
||||
[
|
||||
livekit_cluster_entry("paris", weight=0),
|
||||
livekit_cluster_entry("lyon", weight=2),
|
||||
livekit_cluster_entry("marseille", weight=0),
|
||||
livekit_cluster_entry("lille"),
|
||||
]
|
||||
)
|
||||
|
||||
assert [cluster.name for cluster in PlacementService.candidates()] == [
|
||||
"lyon",
|
||||
"lille",
|
||||
]
|
||||
|
||||
|
||||
def test_choose_cluster_single_cluster():
|
||||
"""With a single cluster, every LiveKit room is placed there."""
|
||||
assert PlacementService.choose_cluster("my-room") == get_default_livekit_cluster()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("registered", (True, False))
|
||||
def test_choose_cluster(registered, two_livekit_clusters):
|
||||
"""The placement strategy picks among the candidates, knowing the LiveKit
|
||||
room's name and the room it is named after, if any."""
|
||||
paris, lyon = two_livekit_clusters
|
||||
room = RoomFactory() if registered else None
|
||||
room_name = str(room.id) if registered else "my-room"
|
||||
|
||||
with mock.patch.object(
|
||||
WeightedRandomStrategy, "choose", autospec=True, return_value=lyon
|
||||
) as choose:
|
||||
cluster = PlacementService.choose_cluster(room_name, room)
|
||||
|
||||
assert cluster == lyon
|
||||
choose.assert_called_once_with(
|
||||
mock.ANY, (paris, lyon), PlacementContext(room_name=room_name, room=room)
|
||||
)
|
||||
|
||||
|
||||
def test_choose_cluster_never_weighing_zero(settings):
|
||||
"""A cluster weighing 0 is never picked. By default, each candidate is picked
|
||||
for as many of the equally likely draws as it weighs."""
|
||||
settings.LIVEKIT_CLUSTERS = json.dumps(
|
||||
[
|
||||
livekit_cluster_entry("paris", weight=0),
|
||||
livekit_cluster_entry("lyon"),
|
||||
livekit_cluster_entry("lille", weight=3),
|
||||
]
|
||||
)
|
||||
|
||||
with mock.patch(
|
||||
"core.livekit.strategies.randbelow", side_effect=range(4)
|
||||
) as randbelow:
|
||||
picked = [PlacementService.choose_cluster("my-room").name for _draw in range(4)]
|
||||
|
||||
assert randbelow.call_args_list == [mock.call(4)] * 4
|
||||
assert picked == ["lyon", "lille", "lille", "lille"]
|
||||
|
||||
|
||||
def test_choose_cluster_with_the_strategy_set(two_livekit_clusters, settings):
|
||||
"""LIVEKIT_PLACEMENT_STRATEGY names the placement strategy, which picks the
|
||||
cluster where the default one would draw another."""
|
||||
_paris, lyon = two_livekit_clusters
|
||||
settings.LIVEKIT_PLACEMENT_STRATEGY = (
|
||||
"core.livekit.strategies.LeastPlacementsStrategy"
|
||||
)
|
||||
Placement.objects.create(room_name="other-room", cluster="paris")
|
||||
|
||||
with mock.patch("core.livekit.strategies.randbelow", return_value=0):
|
||||
assert PlacementService.choose_cluster("my-room") == lyon
|
||||
|
||||
|
||||
def test_choose_cluster_without_candidate(settings, caplog):
|
||||
"""With every cluster weighing 0, no LiveKit room can be placed: this is
|
||||
logged as critical."""
|
||||
settings.LIVEKIT_CLUSTERS = json.dumps(
|
||||
[
|
||||
livekit_cluster_entry("paris", weight=0),
|
||||
livekit_cluster_entry("lyon", weight=0),
|
||||
]
|
||||
)
|
||||
|
||||
with (
|
||||
caplog.at_level(logging.CRITICAL, logger="core.livekit.placements"),
|
||||
pytest.raises(
|
||||
NoCandidateLiveKitClusterError,
|
||||
match="No LiveKit cluster may receive new placements",
|
||||
),
|
||||
):
|
||||
PlacementService.choose_cluster("my-room")
|
||||
|
||||
assert [record.levelname for record in caplog.records] == ["CRITICAL"]
|
||||
assert (
|
||||
"No LiveKit cluster may receive new placements: LiveKit room my-room "
|
||||
"cannot be placed"
|
||||
) in caplog.text
|
||||
|
||||
|
||||
def test_adopt_registered_room(caplog):
|
||||
"""A LiveKit room named after a room's id is placed where it lives, for the room."""
|
||||
cluster = get_default_livekit_cluster()
|
||||
room = RoomFactory()
|
||||
|
||||
with caplog.at_level(logging.INFO, logger="core.livekit.placements"):
|
||||
placement = PlacementService.adopt(str(room.id), cluster, "RM_abc")
|
||||
|
||||
assert placement == Placement.objects.get()
|
||||
assert placement.room_name == str(room.id)
|
||||
assert placement.room == room
|
||||
assert placement.cluster == "default"
|
||||
assert placement.sid == "RM_abc"
|
||||
assert (
|
||||
f"Adopted LiveKit room {room.id}, live on LiveKit cluster default with SID "
|
||||
"RM_abc"
|
||||
) in caplog.text
|
||||
|
||||
|
||||
def test_adopt_unregistered_room(settings):
|
||||
"""A LiveKit room named after a slug is placed where it lives, for no room,
|
||||
while unregistered rooms are allowed."""
|
||||
settings.ALLOW_UNREGISTERED_ROOMS = True
|
||||
|
||||
placement = PlacementService.adopt(
|
||||
"my-room", get_default_livekit_cluster(), "RM_abc"
|
||||
)
|
||||
|
||||
assert placement == Placement.objects.get()
|
||||
assert placement.room_name == "my-room"
|
||||
assert placement.room is None
|
||||
assert placement.cluster == "default"
|
||||
assert placement.sid == "RM_abc"
|
||||
|
||||
|
||||
def test_adopt_unregistered_room_not_allowed(settings):
|
||||
"""A LiveKit room named after a slug is left without placement while
|
||||
unregistered rooms are not allowed."""
|
||||
settings.ALLOW_UNREGISTERED_ROOMS = False
|
||||
|
||||
assert (
|
||||
PlacementService.adopt("my-room", get_default_livekit_cluster(), "RM_abc")
|
||||
is None
|
||||
)
|
||||
assert not Placement.objects.exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"room_name",
|
||||
(
|
||||
"!JIfCxVLcKKkWrmVBOb:your-domain.com",
|
||||
"My room",
|
||||
"",
|
||||
"a" * 101,
|
||||
"connection-test-0b6f8a4e-2c1d-4e8f-9a3b-5d7c6e1f2a4b",
|
||||
),
|
||||
)
|
||||
def test_adopt_not_a_room_name(room_name, settings, caplog):
|
||||
"""A LiveKit room named neither after a room's id nor after a slug a room
|
||||
may have, or a connection test room, is silently left without placement."""
|
||||
settings.ALLOW_UNREGISTERED_ROOMS = True
|
||||
settings.CONNECTION_TEST_ROOM_PREFIX = "connection-test"
|
||||
|
||||
with caplog.at_level(logging.INFO, logger="core.livekit.placements"):
|
||||
placement = PlacementService.adopt(
|
||||
room_name, get_default_livekit_cluster(), "RM_abc"
|
||||
)
|
||||
|
||||
assert placement is None
|
||||
assert not Placement.objects.exists()
|
||||
assert not caplog.records
|
||||
|
||||
|
||||
@pytest.mark.parametrize("deleted", (False, True))
|
||||
def test_adopt_no_active_room(deleted, caplog):
|
||||
"""A LiveKit room named after a UUID that is no active room's id is left
|
||||
without placement, with a warning."""
|
||||
room = RoomFactory()
|
||||
room_name = str(room.id)
|
||||
if deleted:
|
||||
room.soft_delete()
|
||||
else:
|
||||
room.delete()
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="core.livekit.placements"):
|
||||
placement = PlacementService.adopt(
|
||||
room_name, get_default_livekit_cluster(), "RM_abc"
|
||||
)
|
||||
|
||||
assert placement is None
|
||||
assert not Placement.objects.exists()
|
||||
assert (
|
||||
f"LiveKit room {room_name}, live on LiveKit cluster default, is named after "
|
||||
"no active room: leaving it without placement"
|
||||
) in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("too_long", ("name", "sid"))
|
||||
def test_adopt_too_long(too_long, settings, caplog):
|
||||
"""A LiveKit room whose name or SID does not fit in a placement is left
|
||||
without one, with a warning.
|
||||
|
||||
Slugs a room may have always fit, but not every spelling of a room's id."""
|
||||
settings.ALLOW_UNREGISTERED_ROOMS = True
|
||||
room_name, sid = "my-room", "RM_abc"
|
||||
if too_long == "name":
|
||||
room_name = "-".join(RoomFactory().id.hex) + "-" * 200
|
||||
else:
|
||||
sid = "R" * 64
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="core.livekit.placements"):
|
||||
placement = PlacementService.adopt(
|
||||
room_name, get_default_livekit_cluster(), sid
|
||||
)
|
||||
|
||||
assert placement is None
|
||||
assert not Placement.objects.exists()
|
||||
assert (
|
||||
f"LiveKit room {room_name}, live on LiveKit cluster default with SID {sid}, "
|
||||
"has a name or SID too long for a placement"
|
||||
) in caplog.text
|
||||
|
||||
|
||||
def test_adopt_longest_name_and_sid(settings):
|
||||
"""A LiveKit room named after the longest slug a room may have, and whose SID
|
||||
just fits in a placement, is adopted."""
|
||||
settings.ALLOW_UNREGISTERED_ROOMS = True
|
||||
|
||||
placement = PlacementService.adopt(
|
||||
"a" * 100, get_default_livekit_cluster(), "R" * 63
|
||||
)
|
||||
|
||||
assert placement == Placement.objects.get()
|
||||
|
||||
|
||||
def test_adopt_placed_room(two_livekit_clusters, caplog):
|
||||
"""A LiveKit room placed meanwhile keeps its placement, which is returned."""
|
||||
_paris, lyon = two_livekit_clusters
|
||||
room = RoomFactory()
|
||||
placement = Placement.objects.create(
|
||||
room_name=str(room.id), room=room, cluster="paris", sid="RM_old"
|
||||
)
|
||||
|
||||
with caplog.at_level(logging.INFO, logger="core.livekit.placements"):
|
||||
adopted = PlacementService.adopt(str(room.id), lyon, "RM_new")
|
||||
|
||||
assert adopted == placement
|
||||
kept_placement = Placement.objects.get()
|
||||
assert kept_placement.cluster == "paris"
|
||||
assert kept_placement.sid == "RM_old"
|
||||
assert kept_placement.updated_at == placement.updated_at
|
||||
assert "Adopted" not in caplog.text
|
||||
|
||||
|
||||
def placed_and_deleted_meanwhile(times):
|
||||
"""Return a bulk_create for placements during which, the first times it is
|
||||
called, another request places the same LiveKit room and deletes that
|
||||
placement before it is read back."""
|
||||
bulk_create = Placement.objects.bulk_create
|
||||
calls = []
|
||||
|
||||
def side_effect(objs, **kwargs):
|
||||
calls.append(objs)
|
||||
if len(calls) > times:
|
||||
return bulk_create(objs, **kwargs)
|
||||
concurrent = Placement.objects.create(
|
||||
room_name=objs[0].room_name, cluster="lyon"
|
||||
)
|
||||
created = bulk_create(objs, **kwargs)
|
||||
concurrent.delete()
|
||||
return created
|
||||
|
||||
return side_effect
|
||||
|
||||
|
||||
@pytest.mark.parametrize("times", (1, PLACEMENT_CREATION_ATTEMPTS - 1))
|
||||
def test_adopt_placed_and_deleted_meanwhile(times, settings):
|
||||
"""A LiveKit room whose placement by another request is gone by the time it
|
||||
is read back is adopted on another attempt."""
|
||||
settings.ALLOW_UNREGISTERED_ROOMS = True
|
||||
|
||||
with mock.patch.object(
|
||||
Placement.objects,
|
||||
"bulk_create",
|
||||
side_effect=placed_and_deleted_meanwhile(times),
|
||||
) as bulk_create:
|
||||
placement = PlacementService.adopt(
|
||||
"my-room", get_default_livekit_cluster(), "RM_abc"
|
||||
)
|
||||
|
||||
assert placement == Placement.objects.get()
|
||||
assert placement.cluster == "default"
|
||||
assert placement.sid == "RM_abc"
|
||||
assert bulk_create.call_count == times + 1
|
||||
|
||||
|
||||
def test_adopt_attempts_are_bounded(settings):
|
||||
"""A LiveKit room whose placements by other requests keep vanishing before
|
||||
being read back is left without placement after a few attempts."""
|
||||
settings.ALLOW_UNREGISTERED_ROOMS = True
|
||||
|
||||
with mock.patch.object(
|
||||
Placement.objects,
|
||||
"bulk_create",
|
||||
side_effect=placed_and_deleted_meanwhile(PLACEMENT_CREATION_ATTEMPTS),
|
||||
) as bulk_create:
|
||||
placement = PlacementService.adopt(
|
||||
"my-room", get_default_livekit_cluster(), "RM_abc"
|
||||
)
|
||||
|
||||
assert placement is None
|
||||
assert not Placement.objects.exists()
|
||||
assert bulk_create.call_count == PLACEMENT_CREATION_ATTEMPTS
|
||||
|
||||
|
||||
def picking(cluster):
|
||||
"""Patch the default placement strategy to pick a cluster."""
|
||||
return mock.patch.object(
|
||||
WeightedRandomStrategy, "choose", autospec=True, return_value=cluster
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("registered", (True, False))
|
||||
def test_place_new_room(
|
||||
registered, two_livekit_clusters, fake_livekit_room_creation, caplog
|
||||
):
|
||||
"""A LiveKit room without placement is placed on the cluster the placement
|
||||
strategy picks, knowing the room it is named after, if any, and created
|
||||
there: its placement records its SID."""
|
||||
paris, lyon = two_livekit_clusters
|
||||
room = RoomFactory() if registered else None
|
||||
room_name = str(room.id) if registered else "my-room"
|
||||
fake_livekit_room_creation.return_value = "RM_new"
|
||||
|
||||
with (
|
||||
picking(lyon) as choose,
|
||||
caplog.at_level(logging.INFO, logger="core.livekit.placements"),
|
||||
):
|
||||
cluster = PlacementService.place(room_name, room)
|
||||
|
||||
assert cluster == lyon
|
||||
choose.assert_called_once_with(
|
||||
mock.ANY, (paris, lyon), PlacementContext(room_name=room_name, room=room)
|
||||
)
|
||||
fake_livekit_room_creation.assert_called_once_with(room_name, lyon)
|
||||
placement = Placement.objects.get()
|
||||
assert placement.room_name == room_name
|
||||
assert placement.room == room
|
||||
assert placement.cluster == "lyon"
|
||||
assert placement.sid == "RM_new"
|
||||
assert placement.epoch == 0
|
||||
assert placement.absent_since is None
|
||||
assert f"Placed LiveKit room {room_name} on LiveKit cluster lyon" in caplog.text
|
||||
|
||||
|
||||
def test_place_placed_room(two_livekit_clusters, fake_livekit_room_creation):
|
||||
"""A placed LiveKit room is created again on the cluster of its placement,
|
||||
without asking the placement strategy: its placement records the SID
|
||||
returned, and that its LiveKit room is no longer absent."""
|
||||
_paris, lyon = two_livekit_clusters
|
||||
room = RoomFactory()
|
||||
placement = Placement.objects.create(
|
||||
room_name=str(room.id),
|
||||
room=room,
|
||||
cluster="lyon",
|
||||
sid="RM_old",
|
||||
epoch=2,
|
||||
absent_since=timezone.now(),
|
||||
)
|
||||
fake_livekit_room_creation.return_value = "RM_new"
|
||||
|
||||
with mock.patch.object(WeightedRandomStrategy, "choose") as choose:
|
||||
cluster = PlacementService.place(str(room.id), room)
|
||||
|
||||
assert cluster == lyon
|
||||
choose.assert_not_called()
|
||||
fake_livekit_room_creation.assert_called_once_with(str(room.id), lyon)
|
||||
placed = Placement.objects.get()
|
||||
assert placed.pk == placement.pk
|
||||
assert placed.sid == "RM_new"
|
||||
assert placed.absent_since is None
|
||||
assert placed.epoch == 2
|
||||
assert placed.updated_at > placement.updated_at
|
||||
|
||||
|
||||
def test_place_twice(two_livekit_clusters, fake_livekit_room_creation):
|
||||
"""A LiveKit room placed by a first call keeps its placement on the next
|
||||
ones, which do not ask the placement strategy."""
|
||||
_paris, lyon = two_livekit_clusters
|
||||
|
||||
with picking(lyon) as choose:
|
||||
clusters = [PlacementService.place("my-room") for _call in range(2)]
|
||||
|
||||
assert clusters == [lyon, lyon]
|
||||
choose.assert_called_once()
|
||||
assert fake_livekit_room_creation.call_args_list == [mock.call("my-room", lyon)] * 2
|
||||
assert Placement.objects.get().cluster == "lyon"
|
||||
|
||||
|
||||
def test_place_on_cluster_weighing_zero(settings, fake_livekit_room_creation):
|
||||
"""A LiveKit room placed on a cluster that receives no new placements stays
|
||||
there."""
|
||||
settings.LIVEKIT_CLUSTERS = json.dumps(
|
||||
[livekit_cluster_entry("paris"), livekit_cluster_entry("lyon", weight=0)]
|
||||
)
|
||||
Placement.objects.create(room_name="my-room", cluster="lyon")
|
||||
|
||||
assert PlacementService.place("my-room").name == "lyon"
|
||||
assert Placement.objects.get().cluster == "lyon"
|
||||
fake_livekit_room_creation.assert_called_once()
|
||||
|
||||
|
||||
def test_place_concurrently(two_livekit_clusters, fake_livekit_room_creation):
|
||||
"""Requests placing a LiveKit room at the same time all end on the one
|
||||
placement created first, whatever the placement strategy picked for each.
|
||||
|
||||
Here, while the strategy picks Paris for a first request, a second one
|
||||
places the LiveKit room on Lyon."""
|
||||
paris, lyon = two_livekit_clusters
|
||||
picks = iter((paris, lyon))
|
||||
concurrent_clusters = []
|
||||
|
||||
def choose(_strategy, _candidates, context):
|
||||
pick = next(picks)
|
||||
if pick == paris:
|
||||
concurrent_clusters.append(PlacementService.place(context.room_name))
|
||||
return pick
|
||||
|
||||
with mock.patch.object(
|
||||
WeightedRandomStrategy, "choose", autospec=True, side_effect=choose
|
||||
):
|
||||
cluster = PlacementService.place("my-room")
|
||||
|
||||
assert cluster == lyon
|
||||
assert concurrent_clusters == [lyon]
|
||||
placement = Placement.objects.get()
|
||||
assert placement.cluster == "lyon"
|
||||
assert placement.sid == CREATED_LIVEKIT_ROOM_SID
|
||||
assert fake_livekit_room_creation.call_args_list == [mock.call("my-room", lyon)] * 2
|
||||
|
||||
|
||||
def test_place_placed_meanwhile_creation_failure(
|
||||
two_livekit_clusters, fake_livekit_room_creation
|
||||
):
|
||||
"""A placement another request created while this one placed the LiveKit
|
||||
room stays when its cluster fails to create the LiveKit room: credentials
|
||||
may have been handed out for it."""
|
||||
paris, lyon = two_livekit_clusters
|
||||
fake_livekit_room_creation.side_effect = LiveKitRoomCreationError("Timed out")
|
||||
bulk_create = Placement.objects.bulk_create
|
||||
|
||||
def placed_meanwhile(objs, **kwargs):
|
||||
Placement.objects.create(room_name=objs[0].room_name, cluster="paris")
|
||||
return bulk_create(objs, **kwargs)
|
||||
|
||||
with (
|
||||
picking(lyon),
|
||||
mock.patch.object(
|
||||
Placement.objects, "bulk_create", side_effect=placed_meanwhile
|
||||
),
|
||||
pytest.raises(LiveKitRoomCreationError),
|
||||
):
|
||||
PlacementService.place("my-room")
|
||||
|
||||
fake_livekit_room_creation.assert_called_once_with("my-room", paris)
|
||||
assert Placement.objects.get().cluster == "paris"
|
||||
|
||||
|
||||
def test_place_new_room_creation_failure(
|
||||
two_livekit_clusters, fake_livekit_room_creation, caplog
|
||||
):
|
||||
"""A placement created by a call whose cluster fails to create the LiveKit
|
||||
room is deleted, so that the next call may place the LiveKit room on
|
||||
another cluster."""
|
||||
paris, lyon = two_livekit_clusters
|
||||
fake_livekit_room_creation.side_effect = LiveKitRoomCreationError("Timed out")
|
||||
|
||||
with (
|
||||
picking(lyon),
|
||||
caplog.at_level(logging.INFO, logger="core.livekit.placements"),
|
||||
pytest.raises(LiveKitRoomCreationError, match="Timed out"),
|
||||
):
|
||||
PlacementService.place("my-room")
|
||||
|
||||
fake_livekit_room_creation.assert_called_once_with("my-room", lyon)
|
||||
assert not Placement.objects.exists()
|
||||
assert (
|
||||
"Deleted the placement of LiveKit room my-room on LiveKit cluster lyon, "
|
||||
"which failed to create it"
|
||||
) in caplog.text
|
||||
|
||||
fake_livekit_room_creation.side_effect = None
|
||||
with picking(paris):
|
||||
assert PlacementService.place("my-room") == paris
|
||||
assert Placement.objects.get().cluster == "paris"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("two_livekit_clusters")
|
||||
@pytest.mark.parametrize("sid", ("", "RM_old"))
|
||||
def test_place_placed_room_creation_failure(sid, fake_livekit_room_creation):
|
||||
"""The placement of a LiveKit room placed earlier stays when its cluster
|
||||
fails to create the LiveKit room: credentials may have been handed out for
|
||||
it, or be about to."""
|
||||
placement = Placement.objects.create(room_name="my-room", cluster="lyon", sid=sid)
|
||||
fake_livekit_room_creation.side_effect = LiveKitRoomCreationError("Timed out")
|
||||
|
||||
with pytest.raises(LiveKitRoomCreationError):
|
||||
PlacementService.place("my-room")
|
||||
|
||||
kept_placement = Placement.objects.get()
|
||||
assert kept_placement.cluster == "lyon"
|
||||
assert kept_placement.sid == sid
|
||||
assert kept_placement.updated_at == placement.updated_at
|
||||
|
||||
|
||||
def change_during_first_creation(**changes):
|
||||
"""Return a fake LiveKit room creation during the first of which the
|
||||
placement of the LiveKit room is updated with these changes, or deleted
|
||||
without any."""
|
||||
sids = iter(("RM_first", "RM_second"))
|
||||
|
||||
def create_livekit_room(room_name, _cluster):
|
||||
sid = next(sids)
|
||||
if sid == "RM_first":
|
||||
placements = Placement.objects.filter(room_name=room_name)
|
||||
if changes:
|
||||
placements.update(**changes)
|
||||
else:
|
||||
placements.delete()
|
||||
return sid
|
||||
|
||||
return create_livekit_room
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("changes", "expected_cluster", "expected_epoch"),
|
||||
(
|
||||
pytest.param({"sid": "RM_other"}, "lyon", 0, id="sid"),
|
||||
pytest.param({"epoch": 1}, "lyon", 1, id="epoch"),
|
||||
pytest.param(
|
||||
{"cluster": "paris", "previous_cluster": "lyon", "epoch": 1},
|
||||
"paris",
|
||||
1,
|
||||
id="relocated",
|
||||
),
|
||||
pytest.param({}, "paris", 0, id="deleted"),
|
||||
),
|
||||
)
|
||||
def test_place_changed_meanwhile(
|
||||
changes,
|
||||
expected_cluster,
|
||||
expected_epoch,
|
||||
two_livekit_clusters,
|
||||
fake_livekit_room_creation,
|
||||
):
|
||||
"""A placement that changed while its cluster created the LiveKit room is
|
||||
read again, and the LiveKit room created once more where it now says, or
|
||||
placed again when it is gone."""
|
||||
paris, lyon = two_livekit_clusters
|
||||
Placement.objects.create(room_name="my-room", cluster="lyon")
|
||||
fake_livekit_room_creation.side_effect = change_during_first_creation(**changes)
|
||||
|
||||
with picking(paris):
|
||||
cluster = PlacementService.place("my-room")
|
||||
|
||||
assert cluster.name == expected_cluster
|
||||
assert fake_livekit_room_creation.call_args_list == [
|
||||
mock.call("my-room", lyon),
|
||||
mock.call("my-room", cluster),
|
||||
]
|
||||
placement = Placement.objects.get()
|
||||
assert placement.cluster == expected_cluster
|
||||
assert placement.epoch == expected_epoch
|
||||
assert placement.sid == "RM_second"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sid", ("", "RM_earlier"))
|
||||
@pytest.mark.parametrize("absent", (False, True))
|
||||
def test_place_given_the_created_sid_meanwhile(
|
||||
sid,
|
||||
absent,
|
||||
two_livekit_clusters,
|
||||
fake_livekit_room_creation,
|
||||
django_assert_num_queries,
|
||||
):
|
||||
"""A placement given, while its cluster created the LiveKit room, the very
|
||||
SID the cluster returns, as by another request joining at the same time,
|
||||
is the one credentials are handed out for: it is not read again, nor the
|
||||
LiveKit room created once more, and it is no longer absent."""
|
||||
_paris, lyon = two_livekit_clusters
|
||||
Placement.objects.create(room_name="my-room", cluster="lyon", sid=sid)
|
||||
fake_livekit_room_creation.side_effect = change_during_first_creation(
|
||||
sid="RM_first", absent_since=timezone.now() if absent else None
|
||||
)
|
||||
|
||||
placement_read = 1
|
||||
change_meanwhile = 1
|
||||
sid_update = 1
|
||||
with django_assert_num_queries(placement_read + change_meanwhile + sid_update):
|
||||
cluster = PlacementService.place("my-room")
|
||||
|
||||
assert cluster == lyon
|
||||
fake_livekit_room_creation.assert_called_once_with("my-room", lyon)
|
||||
placement = Placement.objects.get()
|
||||
assert placement.cluster == "lyon"
|
||||
assert placement.epoch == 0
|
||||
assert placement.sid == "RM_first"
|
||||
assert placement.absent_since is None
|
||||
|
||||
|
||||
def test_place_keeps_changing(two_livekit_clusters, fake_livekit_room_creation, caplog):
|
||||
"""A placement that changes again while its cluster creates the LiveKit room
|
||||
once more is left as it is: no credentials can be handed out for now."""
|
||||
_paris, lyon = two_livekit_clusters
|
||||
Placement.objects.create(room_name="my-room", cluster="lyon")
|
||||
|
||||
def bump_epoch_each_time(room_name, _cluster):
|
||||
Placement.objects.filter(room_name=room_name).update(epoch=F("epoch") + 1)
|
||||
return "RM_new"
|
||||
|
||||
fake_livekit_room_creation.side_effect = bump_epoch_each_time
|
||||
|
||||
with (
|
||||
caplog.at_level(logging.WARNING, logger="core.livekit.placements"),
|
||||
pytest.raises(PlacementChangedError, match="kept changing"),
|
||||
):
|
||||
PlacementService.place("my-room")
|
||||
|
||||
assert (
|
||||
fake_livekit_room_creation.call_args_list
|
||||
== [mock.call("my-room", lyon)] * PLACEMENT_UPDATE_ATTEMPTS
|
||||
)
|
||||
placement = Placement.objects.get()
|
||||
assert placement.epoch == PLACEMENT_UPDATE_ATTEMPTS
|
||||
assert placement.sid == ""
|
||||
assert (
|
||||
"Placement of LiveKit room my-room kept changing while its LiveKit room "
|
||||
"was created"
|
||||
) in caplog.text
|
||||
|
||||
|
||||
def test_place_on_unregistered_cluster(
|
||||
two_livekit_clusters, fake_livekit_room_creation, caplog
|
||||
):
|
||||
"""A LiveKit room placed on a cluster since removed from the registry is
|
||||
placed again, with a warning."""
|
||||
_paris, lyon = two_livekit_clusters
|
||||
unregistered = Placement.objects.create(
|
||||
room_name="my-room", cluster="lille", sid="RM_old"
|
||||
)
|
||||
|
||||
with (
|
||||
picking(lyon),
|
||||
caplog.at_level(logging.WARNING, logger="core.livekit.placements"),
|
||||
):
|
||||
assert PlacementService.place("my-room") == lyon
|
||||
|
||||
placement = Placement.objects.get()
|
||||
assert placement.pk != unregistered.pk
|
||||
assert placement.cluster == "lyon"
|
||||
assert placement.sid == CREATED_LIVEKIT_ROOM_SID
|
||||
fake_livekit_room_creation.assert_called_once_with("my-room", lyon)
|
||||
assert (
|
||||
"LiveKit room my-room is placed on LiveKit cluster lille, which is not "
|
||||
"registered: placing it again"
|
||||
) in caplog.text
|
||||
|
||||
|
||||
def test_place_on_unregistered_cluster_attempts_are_bounded(
|
||||
two_livekit_clusters, fake_livekit_room_creation
|
||||
):
|
||||
"""A LiveKit room that other requests keep placing on clusters missing from
|
||||
the registry is not placed after a few attempts."""
|
||||
_paris, lyon = two_livekit_clusters
|
||||
bulk_create = Placement.objects.bulk_create
|
||||
|
||||
def placed_on_unregistered_cluster_meanwhile(objs, **kwargs):
|
||||
Placement.objects.create(room_name=objs[0].room_name, cluster="lille")
|
||||
return bulk_create(objs, **kwargs)
|
||||
|
||||
with (
|
||||
picking(lyon),
|
||||
mock.patch.object(
|
||||
Placement.objects,
|
||||
"bulk_create",
|
||||
side_effect=placed_on_unregistered_cluster_meanwhile,
|
||||
) as patched_bulk_create,
|
||||
pytest.raises(PlacementChangedError),
|
||||
):
|
||||
PlacementService.place("my-room")
|
||||
|
||||
assert patched_bulk_create.call_count == PLACEMENT_CREATION_ATTEMPTS
|
||||
assert not Placement.objects.exists()
|
||||
fake_livekit_room_creation.assert_not_called()
|
||||
|
||||
|
||||
def test_place_placed_and_deleted_meanwhile_attempts_are_bounded(
|
||||
fake_livekit_room_creation,
|
||||
):
|
||||
"""A LiveKit room whose placements by other requests keep vanishing before
|
||||
being read back is not placed after a few attempts."""
|
||||
with (
|
||||
mock.patch.object(
|
||||
Placement.objects,
|
||||
"bulk_create",
|
||||
side_effect=placed_and_deleted_meanwhile(PLACEMENT_CREATION_ATTEMPTS),
|
||||
) as bulk_create,
|
||||
pytest.raises(PlacementChangedError),
|
||||
):
|
||||
PlacementService.place("my-room")
|
||||
|
||||
assert bulk_create.call_count == PLACEMENT_CREATION_ATTEMPTS
|
||||
assert not Placement.objects.exists()
|
||||
fake_livekit_room_creation.assert_not_called()
|
||||
|
||||
|
||||
def test_place_without_candidate(settings, fake_livekit_room_creation):
|
||||
"""A LiveKit room cannot be placed while no cluster may receive new
|
||||
placements."""
|
||||
settings.LIVEKIT_CLUSTERS = json.dumps([livekit_cluster_entry("paris", weight=0)])
|
||||
|
||||
with pytest.raises(NoCandidateLiveKitClusterError):
|
||||
PlacementService.place("my-room")
|
||||
|
||||
assert not Placement.objects.exists()
|
||||
fake_livekit_room_creation.assert_not_called()
|
||||
|
||||
|
||||
def livekit_client(create_room):
|
||||
"""Return a fake LiveKit API client whose room service creates rooms with
|
||||
this mock."""
|
||||
client = mock.MagicMock()
|
||||
client.room.create_room = create_room
|
||||
client.aclose = mock.AsyncMock()
|
||||
return client
|
||||
|
||||
|
||||
@pytest.mark.real_livekit_room_creation
|
||||
@mock.patch.object(utils, "create_livekit_client")
|
||||
def test_create_livekit_room(create_livekit_client, two_livekit_clusters, settings):
|
||||
"""A LiveKit room is created on its cluster, without metadata, which would
|
||||
replace a live LiveKit room's, and its SID is returned. The cluster is
|
||||
given LIVEKIT_CREATE_ROOM_TIMEOUT_SECONDS to answer."""
|
||||
settings.LIVEKIT_CREATE_ROOM_TIMEOUT_SECONDS = 5
|
||||
_paris, lyon = two_livekit_clusters
|
||||
create_room = mock.AsyncMock(return_value=LiveKitRoom(sid="RM_new", name="my-room"))
|
||||
client = livekit_client(create_room)
|
||||
create_livekit_client.return_value = client
|
||||
|
||||
assert PlacementService.create_livekit_room("my-room", lyon) == "RM_new"
|
||||
|
||||
create_livekit_client.assert_called_once_with(cluster=lyon, timeout=5)
|
||||
create_room.assert_awaited_once_with(CreateRoomRequest(name="my-room"))
|
||||
assert create_room.await_args.args[0].metadata == ""
|
||||
client.aclose.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.real_livekit_room_creation
|
||||
@pytest.mark.parametrize(
|
||||
"error",
|
||||
(
|
||||
TwirpError("unavailable", "no node available", status=503),
|
||||
TimeoutError(),
|
||||
aiohttp.ClientConnectionError("Connection refused"),
|
||||
),
|
||||
)
|
||||
@mock.patch.object(utils, "create_livekit_client")
|
||||
def test_create_livekit_room_failure(
|
||||
create_livekit_client, error, two_livekit_clusters, caplog
|
||||
):
|
||||
"""A cluster that cannot be reached, does not answer in time or answers with
|
||||
an error fails to create the LiveKit room, which is logged."""
|
||||
_paris, lyon = two_livekit_clusters
|
||||
client = livekit_client(mock.AsyncMock(side_effect=error))
|
||||
create_livekit_client.return_value = client
|
||||
|
||||
with (
|
||||
caplog.at_level(logging.ERROR, logger="core.livekit.placements"),
|
||||
pytest.raises(
|
||||
LiveKitRoomCreationError,
|
||||
match="LiveKit cluster lyon failed to create LiveKit room my-room",
|
||||
) as raised,
|
||||
):
|
||||
PlacementService.create_livekit_room("my-room", lyon)
|
||||
|
||||
assert raised.value.__cause__ is error
|
||||
client.aclose.assert_awaited_once()
|
||||
assert "LiveKit cluster lyon failed to create LiveKit room my-room" in caplog.text
|
||||
@@ -0,0 +1,29 @@
|
||||
"""Tests that the conditional deletes of placements stay a single statement."""
|
||||
|
||||
from django.db import router
|
||||
from django.db.models import signals
|
||||
from django.db.models.deletion import Collector
|
||||
|
||||
from core.models import Placement
|
||||
|
||||
|
||||
def test_placement_deletes_are_a_single_statement():
|
||||
"""Deleting placements filtered on their epoch and SID must issue one DELETE
|
||||
statement, which Django does only while nothing listens for the deletion
|
||||
of a placement and no relation points at it."""
|
||||
collector = Collector(using=router.db_for_write(Placement))
|
||||
|
||||
assert collector.can_fast_delete(Placement.objects.all()), (
|
||||
"The conditional deletes of placements (place(), the operations, the "
|
||||
"reconciliation, the room_finished webhook) must stay a single "
|
||||
"DELETE ... WHERE on epoch and SID, or a join can delete a placement "
|
||||
"whose SID another join just recorded: no relation may point at "
|
||||
"Placement"
|
||||
)
|
||||
for signal in (signals.pre_delete, signals.post_delete):
|
||||
assert not signal.has_listeners(Placement), (
|
||||
"No receiver may listen for the deletion of a placement: the "
|
||||
"conditional deletes of placements must stay a single "
|
||||
"DELETE ... WHERE on epoch and SID, or a join can delete a "
|
||||
"placement whose SID another join just recorded"
|
||||
)
|
||||
@@ -0,0 +1,313 @@
|
||||
"""Unit tests for the parsing of the LIVEKIT_CLUSTERS setting."""
|
||||
|
||||
import json
|
||||
import re
|
||||
|
||||
import pytest
|
||||
|
||||
from core.livekit.specs import parse_livekit_clusters
|
||||
from core.tests.utils import livekit_cluster_entry
|
||||
|
||||
SECRET = "s3cr3t-never-shown"
|
||||
|
||||
|
||||
def test_parse_livekit_clusters_reads_every_field():
|
||||
"""Every field of an entry is kept, the API secret hidden from the repr."""
|
||||
[spec] = parse_livekit_clusters(
|
||||
json.dumps(
|
||||
[
|
||||
{
|
||||
"name": "paris-1",
|
||||
"url": "wss://paris-1.livekit.test",
|
||||
"api_url": "http://livekit.paris-1.svc:7880",
|
||||
"api_key": "key-paris-1",
|
||||
"api_secret": SECRET,
|
||||
"weight": 0,
|
||||
}
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
assert spec.name == "paris-1"
|
||||
assert spec.url == "wss://paris-1.livekit.test"
|
||||
assert spec.api_url == "http://livekit.paris-1.svc:7880"
|
||||
assert spec.api_key == "key-paris-1"
|
||||
assert spec.api_secret.get_secret_value() == SECRET
|
||||
assert spec.weight == 0
|
||||
assert SECRET not in repr(spec)
|
||||
|
||||
|
||||
def test_parse_livekit_clusters_optional_fields():
|
||||
"""The API URL and the weight may be left out."""
|
||||
[spec] = parse_livekit_clusters(json.dumps([livekit_cluster_entry("paris")]))
|
||||
|
||||
assert spec.api_url is None
|
||||
assert spec.weight == 1
|
||||
|
||||
|
||||
def test_parse_livekit_clusters_keeps_their_order():
|
||||
"""The clusters keep the order they are listed in, the default one first."""
|
||||
specs = parse_livekit_clusters(
|
||||
json.dumps([livekit_cluster_entry(name) for name in ("lyon", "paris", "lille")])
|
||||
)
|
||||
|
||||
assert [spec.name for spec in specs] == ["lyon", "paris", "lille"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scheme", ["http", "https", "ws", "wss"])
|
||||
def test_parse_livekit_clusters_keeps_urls_as_written(scheme):
|
||||
"""
|
||||
URLs of the four schemes LiveKit serves are accepted and left untouched.
|
||||
"""
|
||||
[spec] = parse_livekit_clusters(
|
||||
json.dumps(
|
||||
[
|
||||
livekit_cluster_entry(
|
||||
"paris",
|
||||
url=f"{scheme}://paris.livekit.test",
|
||||
api_url=f"{scheme}://10.0.0.1:7880",
|
||||
)
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
assert spec.url == f"{scheme}://paris.livekit.test"
|
||||
assert spec.api_url == f"{scheme}://10.0.0.1:7880"
|
||||
|
||||
|
||||
def test_parse_livekit_clusters_accepts_upper_case_schemes():
|
||||
"""The scheme is not case sensitive, and the URL is kept as written."""
|
||||
[spec] = parse_livekit_clusters(
|
||||
json.dumps([livekit_cluster_entry("paris", url="HTTPS://paris.livekit.test")])
|
||||
)
|
||||
|
||||
assert spec.url == "HTTPS://paris.livekit.test"
|
||||
|
||||
|
||||
def test_parse_livekit_clusters_accepts_international_urls():
|
||||
"""URLs may hold characters beyond ASCII, such as international domain names."""
|
||||
[spec] = parse_livekit_clusters(
|
||||
json.dumps([livekit_cluster_entry("paris", url="wss://visio.exämple.fr/é")])
|
||||
)
|
||||
|
||||
assert spec.url == "wss://visio.exämple.fr/é"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw,problem",
|
||||
[
|
||||
("", "LIVEKIT_CLUSTERS: Invalid JSON"),
|
||||
('{"name": "paris"}', "LIVEKIT_CLUSTERS: Input should be a valid array"),
|
||||
("[]", "LIVEKIT_CLUSTERS: List should have at least 1 item"),
|
||||
('["paris"]', "LIVEKIT_CLUSTERS[0]: Input should be an object"),
|
||||
],
|
||||
)
|
||||
def test_parse_livekit_clusters_rejects_invalid_lists(raw, problem):
|
||||
"""The value must be a non-empty JSON list of objects."""
|
||||
with pytest.raises(ValueError, match=re.escape(problem)):
|
||||
parse_livekit_clusters(raw)
|
||||
|
||||
|
||||
NAME_PATTERN_PROBLEM = (
|
||||
"LIVEKIT_CLUSTERS[1].name: String should match pattern '^[a-z0-9-]{1,63}$'"
|
||||
)
|
||||
URL_SCHEME_PROBLEM = "URL scheme should be 'http', 'https', 'ws' or 'wss'"
|
||||
URL_AUTHORITY_PROBLEM = "URL should be written scheme://host"
|
||||
URL_CHARACTERS_PROBLEM = "URL should not contain whitespace or non-printable characters"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"changes,problem",
|
||||
[
|
||||
({"name": "Paris"}, NAME_PATTERN_PROBLEM),
|
||||
({"name": "paris_1"}, NAME_PATTERN_PROBLEM),
|
||||
({"name": ""}, NAME_PATTERN_PROBLEM),
|
||||
({"name": "p" * 64}, NAME_PATTERN_PROBLEM),
|
||||
({"name": "paris\n"}, NAME_PATTERN_PROBLEM),
|
||||
(
|
||||
{"url": "ftp://paris.livekit.test"},
|
||||
f"LIVEKIT_CLUSTERS[1].url: {URL_SCHEME_PROBLEM}",
|
||||
),
|
||||
(
|
||||
{"url": "paris.livekit.test"},
|
||||
"LIVEKIT_CLUSTERS[1].url: Input should be a valid URL",
|
||||
),
|
||||
(
|
||||
{"url": "https:/paris.livekit.test"},
|
||||
f"LIVEKIT_CLUSTERS[1].url: {URL_AUTHORITY_PROBLEM}",
|
||||
),
|
||||
(
|
||||
{"url": "wss:\\\\paris.livekit.test"},
|
||||
f"LIVEKIT_CLUSTERS[1].url: {URL_AUTHORITY_PROBLEM}",
|
||||
),
|
||||
(
|
||||
{"url": "https:paris.livekit.test"},
|
||||
f"LIVEKIT_CLUSTERS[1].url: {URL_AUTHORITY_PROBLEM}",
|
||||
),
|
||||
(
|
||||
{"api_url": "https:///paris.livekit.test"},
|
||||
f"LIVEKIT_CLUSTERS[1].api_url: {URL_AUTHORITY_PROBLEM}",
|
||||
),
|
||||
(
|
||||
{"url": "wss://paris.livekit.test "},
|
||||
f"LIVEKIT_CLUSTERS[1].url: {URL_CHARACTERS_PROBLEM}",
|
||||
),
|
||||
(
|
||||
{"url": "wss://paris.live\tkit.test"},
|
||||
f"LIVEKIT_CLUSTERS[1].url: {URL_CHARACTERS_PROBLEM}",
|
||||
),
|
||||
(
|
||||
{"url": "wss://paris.\u200blivekit.test"},
|
||||
f"LIVEKIT_CLUSTERS[1].url: {URL_CHARACTERS_PROBLEM}",
|
||||
),
|
||||
(
|
||||
{"api_url": "redis://paris.livekit.test"},
|
||||
f"LIVEKIT_CLUSTERS[1].api_url: {URL_SCHEME_PROBLEM}",
|
||||
),
|
||||
(
|
||||
{"api_url": " http://10.0.0.1:7880"},
|
||||
f"LIVEKIT_CLUSTERS[1].api_url: {URL_CHARACTERS_PROBLEM}",
|
||||
),
|
||||
(
|
||||
{"api_key": ""},
|
||||
"LIVEKIT_CLUSTERS[1].api_key: String should have at least 1 character",
|
||||
),
|
||||
(
|
||||
{"api_secret": ""},
|
||||
"LIVEKIT_CLUSTERS[1].api_secret: Value should have at least 1 item",
|
||||
),
|
||||
(
|
||||
{"region": "eu-west-3"},
|
||||
"LIVEKIT_CLUSTERS[1].region: Extra inputs are not permitted",
|
||||
),
|
||||
(
|
||||
{"weight": -1},
|
||||
"LIVEKIT_CLUSTERS[1].weight: Input should be greater than or equal to 0",
|
||||
),
|
||||
(
|
||||
{"weight": "2"},
|
||||
"LIVEKIT_CLUSTERS[1].weight: Input should be a valid integer",
|
||||
),
|
||||
(
|
||||
{"weight": True},
|
||||
"LIVEKIT_CLUSTERS[1].weight: Input should be a valid integer",
|
||||
),
|
||||
(
|
||||
{"weight": 1.5},
|
||||
"LIVEKIT_CLUSTERS[1].weight: Input should be a valid integer",
|
||||
),
|
||||
(
|
||||
{"redis_url": "redis://paris.livekit.test"},
|
||||
"LIVEKIT_CLUSTERS[1].redis_url: Extra inputs are not permitted",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_parse_livekit_clusters_rejects_invalid_entries(changes, problem):
|
||||
"""Each field is checked, values are not coerced and unknown fields are refused."""
|
||||
raw = json.dumps(
|
||||
[livekit_cluster_entry("lyon"), {**livekit_cluster_entry("paris"), **changes}]
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match=re.escape(problem)):
|
||||
parse_livekit_clusters(raw)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ["name", "url", "api_key", "api_secret"])
|
||||
def test_parse_livekit_clusters_requires_fields(field):
|
||||
"""The name, the URL and the API key pair are required."""
|
||||
entry = livekit_cluster_entry("paris")
|
||||
del entry[field]
|
||||
|
||||
with pytest.raises(
|
||||
ValueError, match=re.escape(f"LIVEKIT_CLUSTERS[0].{field}: Field required")
|
||||
):
|
||||
parse_livekit_clusters(json.dumps([entry]))
|
||||
|
||||
|
||||
def test_parse_livekit_clusters_reports_every_problem():
|
||||
"""All the problems found are reported at once."""
|
||||
raw = json.dumps(
|
||||
[
|
||||
{**livekit_cluster_entry("paris"), "name": "Paris"},
|
||||
livekit_cluster_entry("lyon", weight=-1),
|
||||
]
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError) as excinfo:
|
||||
parse_livekit_clusters(raw)
|
||||
|
||||
assert str(excinfo.value) == (
|
||||
f"{NAME_PATTERN_PROBLEM.replace('[1]', '[0]')}; "
|
||||
"LIVEKIT_CLUSTERS[1].weight: Input should be greater than or equal to 0"
|
||||
)
|
||||
|
||||
|
||||
def test_parse_livekit_clusters_rejects_duplicate_names():
|
||||
"""Two clusters cannot have the same name."""
|
||||
raw = json.dumps(
|
||||
[
|
||||
livekit_cluster_entry("paris"),
|
||||
livekit_cluster_entry("lyon"),
|
||||
{**livekit_cluster_entry("lille"), "name": "paris"},
|
||||
]
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError) as excinfo:
|
||||
parse_livekit_clusters(raw)
|
||||
|
||||
assert str(excinfo.value) == (
|
||||
"LIVEKIT_CLUSTERS[2].name: 'paris' is already the name of LIVEKIT_CLUSTERS[0]"
|
||||
)
|
||||
|
||||
|
||||
def test_parse_livekit_clusters_rejects_shared_api_keys():
|
||||
"""Two clusters cannot share the API key that tells them apart, left unshown."""
|
||||
raw = json.dumps(
|
||||
[
|
||||
livekit_cluster_entry("paris", api_key=SECRET),
|
||||
livekit_cluster_entry("lyon"),
|
||||
livekit_cluster_entry("lille", api_key=SECRET),
|
||||
]
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError) as excinfo:
|
||||
parse_livekit_clusters(raw)
|
||||
|
||||
assert str(excinfo.value) == (
|
||||
"LIVEKIT_CLUSTERS[2].api_key: LIVEKIT_CLUSTERS[0] already uses this API key, "
|
||||
"each LiveKit cluster needs its own"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw",
|
||||
[
|
||||
f'[{{"api_secret": "{SECRET}"',
|
||||
json.dumps({"api_secret": SECRET}),
|
||||
json.dumps([SECRET]),
|
||||
json.dumps([livekit_cluster_entry("paris", api_secret=[SECRET])]),
|
||||
json.dumps([livekit_cluster_entry("paris", weight=SECRET)]),
|
||||
json.dumps([livekit_cluster_entry("paris", api_secret_file=SECRET)]),
|
||||
json.dumps([livekit_cluster_entry("paris", url=f"wss://{SECRET} ")]),
|
||||
],
|
||||
ids=[
|
||||
"invalid JSON",
|
||||
"not a list",
|
||||
"not an object",
|
||||
"invalid secret",
|
||||
"secret in another field",
|
||||
"unknown field",
|
||||
"secret in a URL with whitespace",
|
||||
],
|
||||
)
|
||||
def test_parse_livekit_clusters_never_shows_the_values_read(raw):
|
||||
"""Problems name places only, and pydantic's error is not chained.
|
||||
|
||||
Each of these values makes pydantic's own message show the secret.
|
||||
"""
|
||||
with pytest.raises(ValueError) as excinfo:
|
||||
parse_livekit_clusters(raw)
|
||||
|
||||
assert SECRET not in str(excinfo.value)
|
||||
assert excinfo.value.__cause__ is None
|
||||
assert excinfo.value.__context__ is None
|
||||
@@ -0,0 +1,199 @@
|
||||
"""Unit tests for the placement strategies picking the LiveKit cluster of new
|
||||
placements."""
|
||||
|
||||
import secrets
|
||||
from unittest import mock
|
||||
|
||||
from django.core.cache import cache
|
||||
|
||||
import pytest
|
||||
|
||||
from core.livekit.clusters import LiveKitClusterConfig
|
||||
from core.livekit.strategies import (
|
||||
LeastPlacementsStrategy,
|
||||
PlacementContext,
|
||||
RoundRobinStrategy,
|
||||
WeightedRandomStrategy,
|
||||
get_placement_strategy,
|
||||
)
|
||||
from core.models import Placement
|
||||
|
||||
from meet import settings as meet_settings
|
||||
|
||||
CONTEXT = PlacementContext(room_name="my-room")
|
||||
|
||||
|
||||
def livekit_cluster(name, weight=1):
|
||||
"""Return a LiveKit cluster with this name and weight."""
|
||||
return LiveKitClusterConfig(
|
||||
name=name,
|
||||
url=f"wss://{name}.livekit.test",
|
||||
api_url=f"wss://{name}.livekit.test",
|
||||
api_key=f"key-{name}",
|
||||
api_secret=f"secret-{name}-padded-for-minimum-len!-livekit",
|
||||
weight=weight,
|
||||
)
|
||||
|
||||
|
||||
def test_get_placement_strategy_default():
|
||||
"""By default, LiveKit clusters are picked at random, in proportion to their
|
||||
weights."""
|
||||
assert isinstance(get_placement_strategy(), WeightedRandomStrategy)
|
||||
|
||||
|
||||
def test_get_placement_strategy_set(settings):
|
||||
"""The strategy named by LIVEKIT_PLACEMENT_STRATEGY is looked up on every call."""
|
||||
settings.LIVEKIT_PLACEMENT_STRATEGY = "core.livekit.strategies.RoundRobinStrategy"
|
||||
|
||||
assert isinstance(get_placement_strategy(), RoundRobinStrategy)
|
||||
|
||||
settings.LIVEKIT_PLACEMENT_STRATEGY = (
|
||||
"core.livekit.strategies.LeastPlacementsStrategy"
|
||||
)
|
||||
|
||||
assert isinstance(get_placement_strategy(), LeastPlacementsStrategy)
|
||||
|
||||
|
||||
def test_get_placement_strategy_not_imported_while_settings_load():
|
||||
"""The settings load without importing the placement strategy.
|
||||
|
||||
A strategy may need applications that are not ready while the settings load.
|
||||
"""
|
||||
with mock.patch.object(
|
||||
meet_settings.Test,
|
||||
"LIVEKIT_PLACEMENT_STRATEGY",
|
||||
"core.livekit.missing.Strategy",
|
||||
):
|
||||
meet_settings.Test.post_setup()
|
||||
|
||||
|
||||
def test_weighted_random_strategy():
|
||||
"""Each candidate is picked for as many of the equally likely draws as it
|
||||
weighs."""
|
||||
candidates = (
|
||||
livekit_cluster("paris", weight=1),
|
||||
livekit_cluster("lyon", weight=3),
|
||||
livekit_cluster("lille", weight=2),
|
||||
)
|
||||
|
||||
with mock.patch(
|
||||
"core.livekit.strategies.randbelow", side_effect=range(6)
|
||||
) as randbelow:
|
||||
picked = [
|
||||
WeightedRandomStrategy().choose(candidates, CONTEXT).name
|
||||
for _draw in range(6)
|
||||
]
|
||||
|
||||
assert randbelow.call_args_list == [mock.call(6)] * 6
|
||||
assert picked == ["paris", "lyon", "lyon", "lyon", "lille", "lille"]
|
||||
|
||||
|
||||
def test_weighted_random_strategy_single_candidate():
|
||||
"""A single candidate is always picked, whatever its weight."""
|
||||
lyon = livekit_cluster("lyon", weight=5)
|
||||
|
||||
assert WeightedRandomStrategy().choose((lyon,), CONTEXT) == lyon
|
||||
|
||||
|
||||
@pytest.fixture(name="round_robin_clusters")
|
||||
def round_robin_clusters_fixture():
|
||||
"""Return three LiveKit clusters of different weights, named for this test
|
||||
alone.
|
||||
|
||||
Turns are counted in the cache, which tests running in parallel share and
|
||||
which outlives them, so the counters of the first one, two and three of
|
||||
these clusters are deleted afterwards.
|
||||
"""
|
||||
suffix = secrets.token_hex(4)
|
||||
clusters = tuple(
|
||||
livekit_cluster(f"{name}-{suffix}", weight=weight)
|
||||
for name, weight in (("paris", 1), ("lyon", 3), ("lille", 2))
|
||||
)
|
||||
yield clusters
|
||||
cache.delete_many(
|
||||
[RoundRobinStrategy.counter_key(clusters[:size]) for size in (1, 2, 3)]
|
||||
)
|
||||
|
||||
|
||||
def test_round_robin_strategy(round_robin_clusters):
|
||||
"""Candidates are picked in turn, whatever their weights, the turns being
|
||||
shared by every instance of the strategy."""
|
||||
paris, lyon, lille = round_robin_clusters
|
||||
|
||||
picked = [
|
||||
RoundRobinStrategy().choose(round_robin_clusters, CONTEXT) for _turn in range(7)
|
||||
]
|
||||
|
||||
assert picked == [paris, lyon, lille, paris, lyon, lille, paris]
|
||||
|
||||
|
||||
def test_round_robin_strategy_turns_per_candidates(round_robin_clusters):
|
||||
"""Each set of candidates has turns of its own."""
|
||||
paris, lyon, lille = round_robin_clusters
|
||||
strategy = RoundRobinStrategy()
|
||||
|
||||
assert strategy.choose((paris, lyon, lille), CONTEXT) == paris
|
||||
assert strategy.choose((paris, lyon), CONTEXT) == paris
|
||||
assert strategy.choose((paris, lyon, lille), CONTEXT) == lyon
|
||||
assert strategy.choose((paris, lyon), CONTEXT) == lyon
|
||||
assert strategy.choose((paris, lyon), CONTEXT) == paris
|
||||
assert strategy.choose((paris, lyon, lille), CONTEXT) == lille
|
||||
|
||||
|
||||
def test_round_robin_strategy_counter_lost(round_robin_clusters):
|
||||
"""Should the counter vanish right after its creation, the turn goes to the
|
||||
first candidate, and turns start over."""
|
||||
paris, lyon, _lille = round_robin_clusters
|
||||
strategy = RoundRobinStrategy()
|
||||
assert strategy.choose(round_robin_clusters, CONTEXT) == paris
|
||||
assert strategy.choose(round_robin_clusters, CONTEXT) == lyon
|
||||
key = RoundRobinStrategy.counter_key(round_robin_clusters)
|
||||
|
||||
with mock.patch.object(
|
||||
cache, "add", side_effect=lambda *args, **kwargs: cache.delete(key)
|
||||
):
|
||||
assert strategy.choose(round_robin_clusters, CONTEXT) == paris
|
||||
|
||||
assert strategy.choose(round_robin_clusters, CONTEXT) == paris
|
||||
assert strategy.choose(round_robin_clusters, CONTEXT) == lyon
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@pytest.mark.parametrize(
|
||||
("placements", "picked"),
|
||||
(
|
||||
({}, "paris"),
|
||||
({"paris": 1}, "lyon"),
|
||||
({"paris": 1, "lyon": 2}, "lyon"),
|
||||
({"paris": 1, "lyon": 3}, "paris"),
|
||||
({"paris": 1, "lyon": 4}, "paris"),
|
||||
),
|
||||
)
|
||||
def test_least_placements_strategy(placements, picked, django_assert_num_queries):
|
||||
"""The candidate holding the fewest placements for its weight is picked, the
|
||||
first one on a tie, with a single query."""
|
||||
candidates = (
|
||||
livekit_cluster("paris", weight=1),
|
||||
livekit_cluster("lyon", weight=3),
|
||||
)
|
||||
for cluster, count in placements.items():
|
||||
for index in range(count):
|
||||
Placement.objects.create(room_name=f"{cluster}-{index}", cluster=cluster)
|
||||
|
||||
with django_assert_num_queries(1):
|
||||
chosen = LeastPlacementsStrategy().choose(candidates, CONTEXT)
|
||||
|
||||
assert chosen.name == picked
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_least_placements_strategy_tie():
|
||||
"""On a tie, the candidate registered first is picked."""
|
||||
paris = livekit_cluster("paris", weight=1)
|
||||
lyon = livekit_cluster("lyon", weight=2)
|
||||
Placement.objects.create(room_name="paris-0", cluster="paris")
|
||||
Placement.objects.create(room_name="lyon-0", cluster="lyon")
|
||||
Placement.objects.create(room_name="lyon-1", cluster="lyon")
|
||||
|
||||
assert LeastPlacementsStrategy().choose((paris, lyon), CONTEXT) == paris
|
||||
assert LeastPlacementsStrategy().choose((lyon, paris), CONTEXT) == lyon
|
||||
@@ -12,11 +12,13 @@ from unittest import mock
|
||||
from django.contrib.sites.models import Site
|
||||
|
||||
import pytest
|
||||
from asgiref.sync import async_to_sync
|
||||
from celery.exceptions import OperationalError as CeleryOperationalError
|
||||
|
||||
from core import factories, models
|
||||
from core.analytics import UserFeatureFlag
|
||||
from core.audit.testing import find_events
|
||||
from core.livekit.clusters import get_default_livekit_cluster
|
||||
from core.recording.event.notification import NotificationService, notification_service
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
@@ -394,7 +396,9 @@ def test_notify_summary_service_post_args_with_metadata(
|
||||
mock.call(metadata_filename, expires_in=60 * 60 * 24, override_domain=False),
|
||||
mock.call(recording.key, expires_in=60 * 60 * 24, override_domain=False),
|
||||
]
|
||||
mock_get_recording_timestamps.assert_awaited_once_with("egress-1")
|
||||
mock_get_recording_timestamps.assert_awaited_once_with(
|
||||
"egress-1", get_default_livekit_cluster()
|
||||
)
|
||||
|
||||
|
||||
@mock.patch("core.recording.event.notification.requests.post")
|
||||
@@ -472,7 +476,9 @@ def test_notify_summary_service_post_args_without_metadata(
|
||||
mock_generate_download_s3_url.assert_called_once_with(
|
||||
recording.key, expires_in=60 * 60 * 24, override_domain=False
|
||||
)
|
||||
mock_get_recording_timestamps.assert_awaited_once_with(recording.worker_id)
|
||||
mock_get_recording_timestamps.assert_awaited_once_with(
|
||||
recording.worker_id, get_default_livekit_cluster()
|
||||
)
|
||||
mock_is_feature_flag_enabled.assert_called_once_with(
|
||||
owner, UserFeatureFlag.TRANSCRIPT_SUMMARY_ENABLED
|
||||
)
|
||||
@@ -538,6 +544,70 @@ def test_notify_summary_service_v2_payload_json_serializable_without_timestamps(
|
||||
json.dumps(payload)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version", [1, 2])
|
||||
@mock.patch.object(
|
||||
NotificationService, "_get_recording_timestamps", new_callable=mock.AsyncMock
|
||||
)
|
||||
def test_notify_summary_service_timestamps_on_the_recording_cluster(
|
||||
mock_get_recording_timestamps, version, settings, two_livekit_clusters
|
||||
):
|
||||
"""The recording's egress is looked up on the LiveKit cluster that recorded
|
||||
the room, even once the room was placed elsewhere."""
|
||||
settings.SUMMARY_SERVICE_VERSION = version
|
||||
settings.SUMMARY_SERVICE_ENDPOINT = "https://summary.test/api/v2/tasks"
|
||||
settings.SUMMARY_SERVICE_API_TOKEN = "summary-token"
|
||||
_paris, lyon = two_livekit_clusters
|
||||
recording = factories.RecordingFactory(
|
||||
mode="transcript", worker_id="egress-1", livekit_cluster="lyon"
|
||||
)
|
||||
models.Placement.objects.create(room_name=str(recording.room.id), cluster="paris")
|
||||
factories.UserRecordingAccessFactory(
|
||||
recording=recording, role=models.RoleChoices.OWNER
|
||||
)
|
||||
mock_get_recording_timestamps.return_value = (None, None)
|
||||
|
||||
with (
|
||||
mock.patch("core.recording.event.notification.requests.post") as mock_post,
|
||||
mock.patch(
|
||||
"core.recording.event.notification.generate_download_s3_url",
|
||||
return_value="https://storage.test/recording.ogg",
|
||||
),
|
||||
):
|
||||
mock_post.return_value.json.return_value = {"job_id": "job-8"}
|
||||
assert NotificationService._notify_summary_service(recording) is True
|
||||
|
||||
mock_get_recording_timestamps.assert_awaited_once_with("egress-1", lyon)
|
||||
|
||||
|
||||
@mock.patch("core.utils.create_livekit_client")
|
||||
def test_get_recording_timestamps_on_the_given_cluster(
|
||||
mock_create_livekit_client, two_livekit_clusters
|
||||
):
|
||||
"""The egress is looked up on the LiveKit cluster given."""
|
||||
_paris, lyon = two_livekit_clusters
|
||||
started_at = datetime.datetime(2026, 1, 2, 10, 30, tzinfo=datetime.timezone.utc)
|
||||
ended_at = datetime.datetime(2026, 1, 2, 11, 45, tzinfo=datetime.timezone.utc)
|
||||
file_result = mock.Mock(
|
||||
started_at=int(started_at.timestamp()) * 10**9,
|
||||
ended_at=int(ended_at.timestamp()) * 10**9,
|
||||
)
|
||||
client = mock.AsyncMock()
|
||||
client.egress.list_egress.return_value = mock.Mock(
|
||||
items=[mock.Mock(file_results=[file_result])]
|
||||
)
|
||||
mock_create_livekit_client.return_value = client
|
||||
|
||||
timestamps = async_to_sync(NotificationService._get_recording_timestamps)(
|
||||
"egress-1", lyon
|
||||
)
|
||||
|
||||
assert timestamps == (started_at, ended_at)
|
||||
mock_create_livekit_client.assert_called_once_with(cluster=lyon)
|
||||
client.egress.list_egress.assert_awaited_once()
|
||||
[request] = client.egress.list_egress.await_args.args
|
||||
assert request.egress_id == "egress-1"
|
||||
|
||||
|
||||
def test_summary_recipient_is_oldest_access():
|
||||
"""With several accesses, the summary goes to the oldest one."""
|
||||
recording = factories.RecordingFactory()
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
"""
|
||||
Test MetadataCollectorService service.
|
||||
"""
|
||||
|
||||
# pylint: disable=redefined-outer-name
|
||||
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
from livekit.protocol.agent_dispatch import CreateAgentDispatchRequest
|
||||
|
||||
from core.factories import RecordingFactory
|
||||
from core.models import Placement
|
||||
from core.recording.services.metadata_collector import MetadataCollectorService
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_create_livekit_client():
|
||||
"""Mock the creation of LiveKit clients: every client created is one mock."""
|
||||
with mock.patch("core.utils.create_livekit_client") as create_livekit_client:
|
||||
create_livekit_client.return_value = mock.AsyncMock()
|
||||
yield create_livekit_client
|
||||
|
||||
|
||||
def test_start_on_the_recording_cluster(
|
||||
mock_create_livekit_client, two_livekit_clusters, settings
|
||||
):
|
||||
"""The metadata collector agent is dispatched to the room on the LiveKit
|
||||
cluster recording it, even once the room was placed elsewhere."""
|
||||
_paris, lyon = two_livekit_clusters
|
||||
recording = RecordingFactory(status="active", livekit_cluster="lyon")
|
||||
Placement.objects.create(room_name=str(recording.room.id), cluster="paris")
|
||||
client = mock_create_livekit_client.return_value
|
||||
client.agent_dispatch.create_dispatch.return_value = mock.Mock(id="AD_lyon")
|
||||
|
||||
assert MetadataCollectorService().start(recording) == "AD_lyon"
|
||||
|
||||
mock_create_livekit_client.assert_called_once_with(cluster=lyon)
|
||||
client.agent_dispatch.create_dispatch.assert_awaited_once_with(
|
||||
CreateAgentDispatchRequest(
|
||||
agent_name=settings.METADATA_COLLECTOR_AGENT_NAME,
|
||||
room=str(recording.room.id),
|
||||
metadata=str(recording.id),
|
||||
)
|
||||
)
|
||||
recording.refresh_from_db()
|
||||
assert recording.options == {"metadata_collector_dispatch_id": "AD_lyon"}
|
||||
|
||||
|
||||
def test_stop_on_the_recording_cluster(
|
||||
mock_create_livekit_client, two_livekit_clusters
|
||||
):
|
||||
"""The metadata collector agent's dispatch is deleted on the LiveKit cluster
|
||||
recording the room, even once the room was placed elsewhere."""
|
||||
_paris, lyon = two_livekit_clusters
|
||||
recording = RecordingFactory(
|
||||
status="active",
|
||||
livekit_cluster="lyon",
|
||||
options={"metadata_collector_dispatch_id": "AD_lyon"},
|
||||
)
|
||||
Placement.objects.create(room_name=str(recording.room.id), cluster="paris")
|
||||
|
||||
MetadataCollectorService().stop(recording)
|
||||
|
||||
mock_create_livekit_client.assert_called_once_with(cluster=lyon)
|
||||
client = mock_create_livekit_client.return_value
|
||||
client.agent_dispatch.delete_dispatch.assert_awaited_once_with(
|
||||
dispatch_id="AD_lyon", room_name=str(recording.room.id)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("livekit_cluster", ["", "lille"])
|
||||
def test_start_and_stop_on_the_default_cluster(
|
||||
livekit_cluster, mock_create_livekit_client, two_livekit_clusters
|
||||
):
|
||||
"""A recording naming no LiveKit cluster, or one no longer registered, has
|
||||
the metadata collector agent dispatched and deleted on the default cluster,
|
||||
wherever the room is placed."""
|
||||
paris, _lyon = two_livekit_clusters
|
||||
recording = RecordingFactory(status="active", livekit_cluster=livekit_cluster)
|
||||
Placement.objects.create(room_name=str(recording.room.id), cluster="lyon")
|
||||
client = mock_create_livekit_client.return_value
|
||||
client.agent_dispatch.create_dispatch.return_value = mock.Mock(id="AD_paris")
|
||||
|
||||
service = MetadataCollectorService()
|
||||
service.start(recording)
|
||||
service.stop(recording)
|
||||
|
||||
assert mock_create_livekit_client.call_args_list == [mock.call(cluster=paris)] * 2
|
||||
client.agent_dispatch.delete_dispatch.assert_awaited_once_with(
|
||||
dispatch_id="AD_paris", room_name=str(recording.room.id)
|
||||
)
|
||||
@@ -10,6 +10,8 @@ from unittest import mock
|
||||
import pytest
|
||||
|
||||
from core.factories import RecordingFactory
|
||||
from core.livekit.clusters import get_default_livekit_cluster
|
||||
from core.models import Placement
|
||||
from core.recording.enums import RecordingWorkerEvent
|
||||
from core.recording.services.recording_events import (
|
||||
RecordingEventsError,
|
||||
@@ -46,7 +48,9 @@ def test_handle_limit_reached_success(mock_notify, mode, notification_type, serv
|
||||
|
||||
assert recording.status == "stopped"
|
||||
mock_notify.assert_called_once_with(
|
||||
room_name=str(recording.room.id), notification_data={"type": notification_type}
|
||||
room_name=str(recording.room.id),
|
||||
notification_data={"type": notification_type},
|
||||
cluster=get_default_livekit_cluster(),
|
||||
)
|
||||
|
||||
|
||||
@@ -66,7 +70,9 @@ def test_handle_interrupted_success(mock_notify, mode, notification_type, servic
|
||||
|
||||
assert recording.status == "stopped"
|
||||
mock_notify.assert_called_once_with(
|
||||
room_name=str(recording.room.id), notification_data={"type": notification_type}
|
||||
room_name=str(recording.room.id),
|
||||
notification_data={"type": notification_type},
|
||||
cluster=get_default_livekit_cluster(),
|
||||
)
|
||||
|
||||
|
||||
@@ -86,7 +92,9 @@ def test_handle_failed_success(mock_notify, mode, notification_type, service):
|
||||
|
||||
assert recording.status == "failed"
|
||||
mock_notify.assert_called_once_with(
|
||||
room_name=str(recording.room.id), notification_data={"type": notification_type}
|
||||
room_name=str(recording.room.id),
|
||||
notification_data={"type": notification_type},
|
||||
cluster=get_default_livekit_cluster(),
|
||||
)
|
||||
|
||||
|
||||
@@ -106,7 +114,9 @@ def test_handle_aborted_success(mock_notify, mode, notification_type, service):
|
||||
|
||||
assert recording.status == "aborted"
|
||||
mock_notify.assert_called_once_with(
|
||||
room_name=str(recording.room.id), notification_data={"type": notification_type}
|
||||
room_name=str(recording.room.id),
|
||||
notification_data={"type": notification_type},
|
||||
cluster=get_default_livekit_cluster(),
|
||||
)
|
||||
|
||||
|
||||
@@ -152,6 +162,7 @@ def test_handle_event_notification_error( # noqa: PLR0913, PLR0917
|
||||
mock_notify.assert_called_once_with(
|
||||
room_name=str(recording.room.id),
|
||||
notification_data={"type": f"{notification_prefix}{notification_suffix}"},
|
||||
cluster=get_default_livekit_cluster(),
|
||||
)
|
||||
|
||||
|
||||
@@ -232,7 +243,9 @@ def test_handle_update_syncs_room_metadata(
|
||||
service.handle_update(recording, event)
|
||||
|
||||
mock_update_metadata.assert_called_once_with(
|
||||
str(recording.room.id), {"recording_status": recording_status}
|
||||
str(recording.room.id),
|
||||
{"recording_status": recording_status},
|
||||
cluster=get_default_livekit_cluster(),
|
||||
)
|
||||
|
||||
|
||||
@@ -308,6 +321,7 @@ def test_handle_terminal_event_dispatches_on_event_and_status( # noqa: PLR0913,
|
||||
mock_notify.assert_called_once_with(
|
||||
room_name=str(recording.room.id),
|
||||
notification_data={"type": f"screenRecording{notification_type}"},
|
||||
cluster=get_default_livekit_cluster(),
|
||||
)
|
||||
|
||||
|
||||
@@ -488,3 +502,46 @@ def test_log_worker_error_stays_quiet_on_anything_else(event, service, caplog):
|
||||
service.log_worker_error(recording, event, error="some error", error_code=500)
|
||||
|
||||
assert "Recording worker reported" not in caplog.text
|
||||
|
||||
|
||||
def live_room_client():
|
||||
"""Return a LiveKit client mock on which the room is live, without metadata."""
|
||||
client = mock.AsyncMock()
|
||||
client.room.list_rooms.return_value = mock.Mock(rooms=[mock.Mock(metadata="")])
|
||||
return client
|
||||
|
||||
|
||||
@mock.patch("core.utils.create_livekit_client")
|
||||
def test_handle_update_on_the_recording_cluster(
|
||||
mock_create_livekit_client, two_livekit_clusters, service
|
||||
):
|
||||
"""The room's metadata is updated on the LiveKit cluster recording it, even
|
||||
once the room was placed elsewhere."""
|
||||
_paris, lyon = two_livekit_clusters
|
||||
recording = RecordingFactory(status="active", livekit_cluster="lyon")
|
||||
Placement.objects.create(room_name=str(recording.room.id), cluster="paris")
|
||||
client = live_room_client()
|
||||
mock_create_livekit_client.return_value = client
|
||||
|
||||
service.handle_update(recording, RecordingWorkerEvent.STARTED)
|
||||
|
||||
mock_create_livekit_client.assert_called_once_with(cluster=lyon)
|
||||
client.room.update_room_metadata.assert_awaited_once()
|
||||
|
||||
|
||||
@mock.patch("core.utils.create_livekit_client")
|
||||
def test_notify_participants_on_the_recording_cluster(
|
||||
mock_create_livekit_client, two_livekit_clusters, service
|
||||
):
|
||||
"""Participants are notified on the LiveKit cluster recording the room, even
|
||||
once the room was placed elsewhere."""
|
||||
_paris, lyon = two_livekit_clusters
|
||||
recording = RecordingFactory(status="active", livekit_cluster="lyon")
|
||||
Placement.objects.create(room_name=str(recording.room.id), cluster="paris")
|
||||
client = live_room_client()
|
||||
mock_create_livekit_client.return_value = client
|
||||
|
||||
service._handle_limit_reached(recording)
|
||||
|
||||
mock_create_livekit_client.assert_called_once_with(cluster=lyon)
|
||||
client.room.send_data.assert_awaited_once()
|
||||
|
||||
@@ -4,7 +4,8 @@ Test worker service factories.
|
||||
|
||||
# pylint: disable=protected-access,redefined-outer-name,unused-argument,no-member
|
||||
|
||||
from dataclasses import FrozenInstanceError
|
||||
import json
|
||||
from dataclasses import FrozenInstanceError, replace
|
||||
from unittest.mock import Mock
|
||||
|
||||
from django.test import override_settings
|
||||
@@ -14,11 +15,13 @@ from livekit import (
|
||||
api as livekit_api_codec,
|
||||
)
|
||||
|
||||
from core.livekit.clusters import get_livekit_cluster
|
||||
from core.recording.worker.factories import (
|
||||
WorkerService,
|
||||
WorkerServiceConfig,
|
||||
get_worker_service,
|
||||
)
|
||||
from core.tests.utils import livekit_cluster_entry
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
@@ -30,11 +33,16 @@ def clear_lru_cache():
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def test_settings():
|
||||
def test_settings(settings):
|
||||
"""Fixture to provide test Django settings"""
|
||||
mocked_settings = {
|
||||
"RECORDING_OUTPUT_FOLDER": "/test/output",
|
||||
"LIVEKIT_CONFIGURATION": {"server": "test.example.com"},
|
||||
"LIVEKIT_CLUSTERS": json.dumps(
|
||||
[
|
||||
livekit_cluster_entry("paris", api_url="http://livekit.paris.svc:7880"),
|
||||
livekit_cluster_entry("lyon"),
|
||||
]
|
||||
),
|
||||
"AWS_S3_ENDPOINT_URL": "https://s3.test.com",
|
||||
"AWS_S3_ACCESS_KEY_ID": "test_key",
|
||||
"AWS_S3_SECRET_ACCESS_KEY": "test_secret",
|
||||
@@ -52,9 +60,8 @@ def test_settings():
|
||||
"RECORDING_ENCODING_KEY_FRAME_INTERVAL_S": 4.0,
|
||||
}
|
||||
|
||||
# Use override_settings to properly patch Django settings
|
||||
with override_settings(**mocked_settings):
|
||||
yield test_settings
|
||||
for name, value in mocked_settings.items():
|
||||
setattr(settings, name, value)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -67,7 +74,11 @@ def default_config(test_settings):
|
||||
def test_config_initialization(default_config):
|
||||
"""Test that WorkerServiceConfig is properly initialized from settings"""
|
||||
assert default_config.output_folder == "/test/output"
|
||||
assert default_config.server_configurations == {"server": "test.example.com"}
|
||||
assert default_config.server_configurations == {
|
||||
"url": "http://livekit.paris.svc:7880",
|
||||
"api_key": "key-paris",
|
||||
"api_secret": "secret-paris-padded-for-minimum-len!-livekit",
|
||||
}
|
||||
assert default_config.bucket_args == {
|
||||
"endpoint": "https://s3.test.com",
|
||||
"access_key": "test_key",
|
||||
@@ -90,6 +101,14 @@ def test_config_initialization(default_config):
|
||||
}
|
||||
|
||||
|
||||
def test_config_server_configurations_from_livekit_configuration(settings):
|
||||
"""Without LIVEKIT_CLUSTERS, LiveKit is called as LIVEKIT_CONFIGURATION says."""
|
||||
config = WorkerServiceConfig.from_settings()
|
||||
|
||||
assert settings.LIVEKIT_CLUSTERS is None
|
||||
assert config.server_configurations == settings.LIVEKIT_CONFIGURATION
|
||||
|
||||
|
||||
def test_config_immutability(default_config):
|
||||
"""Test that config instances are immutable after creation"""
|
||||
with pytest.raises(FrozenInstanceError):
|
||||
@@ -99,7 +118,6 @@ def test_config_immutability(default_config):
|
||||
@pytest.mark.parametrize("custom_encoding_enabled", [True, False])
|
||||
@override_settings(
|
||||
RECORDING_OUTPUT_FOLDER="/test/output",
|
||||
LIVEKIT_CONFIGURATION={"server": "test.example.com"},
|
||||
AWS_S3_ENDPOINT_URL="https://s3.test.com",
|
||||
AWS_S3_ACCESS_KEY_ID="test_key",
|
||||
AWS_S3_SECRET_ACCESS_KEY="test_secret",
|
||||
@@ -161,7 +179,6 @@ def test_config_encoding_options_none_when_default_missing(
|
||||
|
||||
@override_settings(
|
||||
RECORDING_OUTPUT_FOLDER="/test/output",
|
||||
LIVEKIT_CONFIGURATION={"server": "test.example.com"},
|
||||
AWS_S3_ENDPOINT_URL="https://s3.test.com",
|
||||
AWS_S3_ACCESS_KEY_ID="test_key",
|
||||
AWS_S3_SECRET_ACCESS_KEY="test_secret",
|
||||
@@ -208,6 +225,43 @@ def test_factory_valid_mode(mock_import_string, settings, default_config):
|
||||
assert worker.config == default_config
|
||||
|
||||
|
||||
def test_factory_given_cluster(mock_import_string, settings, default_config):
|
||||
"""The worker service calls the LiveKit cluster it is given, with the URL,
|
||||
API key and secret Django calls that cluster with."""
|
||||
settings.RECORDING_WORKER_CLASSES = {"test_mode": "path.to.MockWorkerService"}
|
||||
|
||||
worker = get_worker_service("test_mode", cluster=get_livekit_cluster("lyon"))
|
||||
|
||||
assert worker.config == replace(
|
||||
default_config,
|
||||
server_configurations={
|
||||
"url": "wss://lyon.livekit.test",
|
||||
"api_key": "key-lyon",
|
||||
"api_secret": "secret-lyon-padded-for-minimum-len!-livekit",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_factory_default_cluster_registered_now(
|
||||
mock_import_string, settings, default_config
|
||||
):
|
||||
"""Without a cluster, the worker service calls the default LiveKit cluster
|
||||
registered now, though the rest of its configuration is cached."""
|
||||
settings.RECORDING_WORKER_CLASSES = {"test_mode": "path.to.MockWorkerService"}
|
||||
settings.LIVEKIT_CLUSTERS = json.dumps([livekit_cluster_entry("marseille")])
|
||||
|
||||
worker = get_worker_service("test_mode")
|
||||
|
||||
assert worker.config == replace(
|
||||
default_config,
|
||||
server_configurations={
|
||||
"url": "wss://marseille.livekit.test",
|
||||
"api_key": "key-marseille",
|
||||
"api_secret": "secret-marseille-padded-for-minimum-len!-livekit",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_factory_invalid_mode(settings, mock_import_string, default_config):
|
||||
"""Test getting worker service with invalid mode raises ValueError."""
|
||||
|
||||
|
||||
@@ -8,7 +8,8 @@ from unittest.mock import Mock
|
||||
import pytest
|
||||
|
||||
from core.factories import RecordingFactory
|
||||
from core.models import RecordingStatusChoices
|
||||
from core.livekit.clusters import get_default_livekit_cluster
|
||||
from core.models import Placement, RecordingStatusChoices
|
||||
from core.recording.worker.exceptions import (
|
||||
RecordingStartError,
|
||||
RecordingStopError,
|
||||
@@ -61,6 +62,29 @@ def test_start_recording_success(mock_update_metadata, mediator, mock_worker_ser
|
||||
mock_update_metadata.assert_called_once_with(
|
||||
str(mock_recording.room.id),
|
||||
{"recording_mode": mock_recording.mode, "recording_status": "starting"},
|
||||
cluster=get_default_livekit_cluster(),
|
||||
)
|
||||
|
||||
|
||||
@mock.patch("core.services.room_management.RoomManagement.update_metadata")
|
||||
def test_start_recording_on_the_recording_cluster(
|
||||
mock_update_metadata, mediator, mock_worker_service, two_livekit_clusters
|
||||
):
|
||||
"""The room's metadata is updated on the LiveKit cluster recording it, even
|
||||
once the room was placed elsewhere."""
|
||||
_paris, lyon = two_livekit_clusters
|
||||
mock_worker_service.start.return_value = "test-worker-123"
|
||||
recording = RecordingFactory(
|
||||
status=RecordingStatusChoices.INITIATED, worker_id=None, livekit_cluster="lyon"
|
||||
)
|
||||
Placement.objects.create(room_name=str(recording.room.id), cluster="paris")
|
||||
|
||||
mediator.start(recording)
|
||||
|
||||
mock_update_metadata.assert_called_once_with(
|
||||
str(recording.room.id),
|
||||
{"recording_mode": recording.mode, "recording_status": "starting"},
|
||||
cluster=lyon,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -9,7 +9,11 @@ from unittest import mock
|
||||
import pytest
|
||||
|
||||
from ...factories import RoomFactory
|
||||
from ...livekit.clusters import get_default_livekit_cluster
|
||||
from ...livekit.exceptions import LiveKitRoomCreationError
|
||||
from ...models import Placement
|
||||
from ...services.sip_management import SIPException
|
||||
from ..utils import CREATED_LIVEKIT_ROOM_SID
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
|
||||
@@ -303,3 +307,71 @@ def test_join_sip_failure(mock_sip_management, mock_capture, settings, client):
|
||||
assert response.status_code == 500
|
||||
mock_sip_instance.ensure_dispatch_rule.assert_called_once_with(room)
|
||||
mock_capture.assert_not_called()
|
||||
|
||||
|
||||
@mock.patch("core.roomkit.viewsets.SIPManagement")
|
||||
def test_join_places_the_room_first(
|
||||
mock_sip_management, settings, client, fake_livekit_room_creation
|
||||
):
|
||||
"""The LiveKit room should be placed and created before the dispatch rule, as
|
||||
the device may be the room's first participant."""
|
||||
|
||||
mock_sip_instance = mock_sip_management.return_value
|
||||
|
||||
settings.ROOMKIT_ENABLED = True
|
||||
settings.ROOMKIT_SERVER_TO_SERVER_API_TOKEN = "testAuthToken"
|
||||
|
||||
room = RoomFactory(pin_code="1234567890")
|
||||
placements_at_rule_creation = []
|
||||
|
||||
def ensure_dispatch_rule(rule_room):
|
||||
placements_at_rule_creation.extend(
|
||||
Placement.objects.filter(room=rule_room).values_list("cluster", "sid")
|
||||
)
|
||||
return True
|
||||
|
||||
mock_sip_instance.ensure_dispatch_rule.side_effect = ensure_dispatch_rule
|
||||
|
||||
response = client.post(
|
||||
"/api/v1.0/roomkit/join/",
|
||||
{"pin_code": room.pin_code},
|
||||
HTTP_AUTHORIZATION="Bearer testAuthToken",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
fake_livekit_room_creation.assert_called_once_with(
|
||||
str(room.id), get_default_livekit_cluster()
|
||||
)
|
||||
assert placements_at_rule_creation == [("default", CREATED_LIVEKIT_ROOM_SID)]
|
||||
mock_sip_instance.ensure_dispatch_rule.assert_called_once_with(room)
|
||||
|
||||
|
||||
@mock.patch("core.roomkit.viewsets.analytics.capture")
|
||||
@mock.patch("core.roomkit.viewsets.SIPManagement")
|
||||
def test_join_room_creation_failure(
|
||||
mock_sip_management, mock_capture, settings, client, fake_livekit_room_creation
|
||||
):
|
||||
"""Requests should fail with a 503, without any dispatch rule, when the
|
||||
LiveKit room cannot be created."""
|
||||
|
||||
mock_sip_instance = mock_sip_management.return_value
|
||||
|
||||
settings.ROOMKIT_ENABLED = True
|
||||
settings.ROOMKIT_SERVER_TO_SERVER_API_TOKEN = "testAuthToken"
|
||||
|
||||
room = RoomFactory(pin_code="1234567890")
|
||||
fake_livekit_room_creation.side_effect = LiveKitRoomCreationError("Timed out")
|
||||
|
||||
response = client.post(
|
||||
"/api/v1.0/roomkit/join/",
|
||||
{"pin_code": room.pin_code},
|
||||
HTTP_AUTHORIZATION="Bearer testAuthToken",
|
||||
)
|
||||
|
||||
assert response.status_code == 503
|
||||
assert response.json() == {
|
||||
"detail": "This room cannot be joined for now, please try again."
|
||||
}
|
||||
mock_sip_instance.ensure_dispatch_rule.assert_not_called()
|
||||
mock_capture.assert_not_called()
|
||||
assert not Placement.objects.exists()
|
||||
|
||||
@@ -4,10 +4,13 @@ Test rooms API endpoints in the Meet core app: join.
|
||||
|
||||
# pylint: disable=redefined-outer-name
|
||||
|
||||
import json
|
||||
import logging
|
||||
from unittest import mock
|
||||
|
||||
from django.conf import settings
|
||||
from django.contrib.auth.models import AnonymousUser
|
||||
from django.db import connection
|
||||
from django.utils.crypto import get_random_string
|
||||
|
||||
import jwt
|
||||
@@ -20,8 +23,14 @@ from ...factories import (
|
||||
UserFactory,
|
||||
UserResourceAccessFactory,
|
||||
)
|
||||
from ...models import RoleChoices, RoomAccessLevel
|
||||
from ..utils import generate_user_access_token
|
||||
from ...livekit.clusters import get_default_livekit_cluster
|
||||
from ...livekit.exceptions import LiveKitRoomCreationError
|
||||
from ...models import Placement, RoleChoices, RoomAccessLevel
|
||||
from ..utils import (
|
||||
CREATED_LIVEKIT_ROOM_SID,
|
||||
generate_user_access_token,
|
||||
livekit_cluster_entry,
|
||||
)
|
||||
|
||||
fake = Faker()
|
||||
pytestmark = pytest.mark.django_db
|
||||
@@ -67,11 +76,14 @@ def join_url(room_id):
|
||||
("access_level", "requester", "can_join_directly"), ACCESS_MATRIX
|
||||
)
|
||||
@mock.patch("core.utils.generate_token", return_value=livekit_token)
|
||||
def test_api_rooms_join_access(mock_token, access_level, requester, can_join_directly):
|
||||
def test_api_rooms_join_access(
|
||||
mock_token, access_level, requester, can_join_directly, fake_livekit_room_creation
|
||||
):
|
||||
"""
|
||||
Joining a room should return LiveKit credentials and the pin code only to
|
||||
users who may enter without going through the lobby, and the accesses
|
||||
only to the room's administrators and owners.
|
||||
only to the room's administrators and owners. Its LiveKit room should only
|
||||
be placed and created for those handed out credentials.
|
||||
"""
|
||||
room = RoomFactory(
|
||||
access_level=access_level,
|
||||
@@ -128,9 +140,18 @@ def test_api_rooms_join_access(mock_token, access_level, requester, can_join_dir
|
||||
sources=["camera"],
|
||||
role=role,
|
||||
participant_id=None,
|
||||
cluster=get_default_livekit_cluster(),
|
||||
)
|
||||
fake_livekit_room_creation.assert_called_once_with(
|
||||
str(room.id), get_default_livekit_cluster()
|
||||
)
|
||||
else:
|
||||
mock_token.assert_not_called()
|
||||
fake_livekit_room_creation.assert_not_called()
|
||||
assert (
|
||||
Placement.objects.filter(room_name=str(room.id), room=room).exists()
|
||||
is can_join_directly
|
||||
)
|
||||
|
||||
|
||||
@mock.patch("core.utils.generate_token", return_value=livekit_token)
|
||||
@@ -148,14 +169,16 @@ def test_api_rooms_join_by_slug(mock_token):
|
||||
|
||||
@mock.patch("core.utils.generate_token", return_value=livekit_token)
|
||||
def test_api_rooms_join_members_num_queries(mock_token, django_assert_num_queries):
|
||||
"""The role of the requester should only be looked up once."""
|
||||
"""The role of the requester should only be looked up once. Placing the
|
||||
LiveKit room takes four more queries: its placement is read, created, read
|
||||
back and given the SID of the LiveKit room created."""
|
||||
user = UserFactory()
|
||||
room = RoomFactory(access_level=RoomAccessLevel.RESTRICTED)
|
||||
UserResourceAccessFactory(resource=room, user=user, role=RoleChoices.MEMBER)
|
||||
client = APIClient()
|
||||
client.force_login(user)
|
||||
|
||||
with django_assert_num_queries(3):
|
||||
with django_assert_num_queries(7):
|
||||
response = client.post(join_url(room.id), {}, format="json")
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -163,19 +186,39 @@ def test_api_rooms_join_members_num_queries(mock_token, django_assert_num_querie
|
||||
assert mock_token.call_args.kwargs["role"] == RoleChoices.MEMBER
|
||||
|
||||
|
||||
@mock.patch("core.utils.generate_token", return_value=livekit_token)
|
||||
def test_api_rooms_join_placed_room_num_queries(mock_token, django_assert_num_queries):
|
||||
"""Once the LiveKit room is placed, its placement is only read and given the
|
||||
SID of the LiveKit room created."""
|
||||
user = UserFactory()
|
||||
room = RoomFactory(access_level=RoomAccessLevel.RESTRICTED)
|
||||
UserResourceAccessFactory(resource=room, user=user, role=RoleChoices.MEMBER)
|
||||
Placement.objects.create(room_name=str(room.id), room=room, cluster="default")
|
||||
client = APIClient()
|
||||
client.force_login(user)
|
||||
|
||||
with django_assert_num_queries(5):
|
||||
response = client.post(join_url(room.id), {}, format="json")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert "livekit" in response.json()
|
||||
mock_token.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("role", [RoleChoices.ADMIN, RoleChoices.OWNER])
|
||||
@mock.patch("core.utils.generate_token", return_value=livekit_token)
|
||||
def test_api_rooms_join_administrators_num_queries(
|
||||
mock_token, role, django_assert_num_queries
|
||||
):
|
||||
"""Only the accesses should be queried on top of the role for administrators."""
|
||||
"""Only the accesses should be queried on top of the role and the placement
|
||||
of the LiveKit room for administrators."""
|
||||
user = UserFactory()
|
||||
room = RoomFactory(access_level=RoomAccessLevel.RESTRICTED)
|
||||
UserResourceAccessFactory(resource=room, user=user, role=role)
|
||||
client = APIClient()
|
||||
client.force_login(user)
|
||||
|
||||
with django_assert_num_queries(4):
|
||||
with django_assert_num_queries(8):
|
||||
response = client.post(join_url(room.id), {}, format="json")
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -184,6 +227,141 @@ def test_api_rooms_join_administrators_num_queries(
|
||||
assert mock_token.call_args.kwargs["role"] == role
|
||||
|
||||
|
||||
@pytest.mark.parametrize("registered", [True, False])
|
||||
def test_api_rooms_join_placed_cluster(
|
||||
registered, settings, two_livekit_clusters, fake_livekit_room_creation
|
||||
):
|
||||
"""
|
||||
Joining a room should hand out credentials for the LiveKit cluster its
|
||||
LiveKit room is placed on, where the LiveKit room is created: its URL, and
|
||||
a token it signed.
|
||||
"""
|
||||
settings.ALLOW_UNREGISTERED_ROOMS = True
|
||||
_paris, lyon = two_livekit_clusters
|
||||
room = RoomFactory(access_level=RoomAccessLevel.PUBLIC) if registered else None
|
||||
room_name = str(room.id) if registered else "unregistered-room"
|
||||
|
||||
with mock.patch("core.livekit.strategies.randbelow", return_value=1):
|
||||
response = APIClient().post(join_url(room_name), {}, format="json")
|
||||
|
||||
assert response.status_code == 200
|
||||
config = response.json()["livekit"]
|
||||
assert config["url"] == lyon.url
|
||||
assert config["room"] == room_name
|
||||
claims = jwt.decode(config["token"], lyon.api_secret, algorithms=["HS256"])
|
||||
assert claims["iss"] == lyon.api_key
|
||||
assert claims["video"]["room"] == room_name
|
||||
fake_livekit_room_creation.assert_called_once_with(room_name, lyon)
|
||||
placement = Placement.objects.get()
|
||||
assert placement.room_name == room_name
|
||||
assert placement.room == room
|
||||
assert placement.cluster == "lyon"
|
||||
assert placement.sid == CREATED_LIVEKIT_ROOM_SID
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_api_rooms_join_creates_livekit_room_outside_transaction(
|
||||
fake_livekit_room_creation,
|
||||
):
|
||||
"""
|
||||
The LiveKit room should be created outside any transaction, once its
|
||||
placement is committed: other requests see it while LiveKit answers.
|
||||
"""
|
||||
room = RoomFactory(access_level=RoomAccessLevel.PUBLIC)
|
||||
in_atomic_block = []
|
||||
|
||||
def create_livekit_room(_room_name, _cluster):
|
||||
in_atomic_block.append(connection.in_atomic_block)
|
||||
return CREATED_LIVEKIT_ROOM_SID
|
||||
|
||||
fake_livekit_room_creation.side_effect = create_livekit_room
|
||||
|
||||
response = APIClient().post(join_url(room.id), {}, format="json")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert in_atomic_block == [False]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("registered", [True, False])
|
||||
@mock.patch("core.utils.generate_token", return_value=livekit_token)
|
||||
def test_api_rooms_join_room_creation_failure(
|
||||
mock_token, registered, settings, fake_livekit_room_creation
|
||||
):
|
||||
"""
|
||||
Joining a room whose LiveKit room cannot be created should return a 503
|
||||
without credentials, and leave the room without placement, so that the
|
||||
next join may place it on another cluster.
|
||||
"""
|
||||
settings.ALLOW_UNREGISTERED_ROOMS = True
|
||||
room = RoomFactory(access_level=RoomAccessLevel.PUBLIC) if registered else None
|
||||
fake_livekit_room_creation.side_effect = LiveKitRoomCreationError("Timed out")
|
||||
|
||||
response = APIClient().post(
|
||||
join_url(room.id if registered else "unregistered-room"), {}, format="json"
|
||||
)
|
||||
|
||||
assert response.status_code == 503
|
||||
assert response.json() == {
|
||||
"detail": "This room cannot be joined for now, please try again."
|
||||
}
|
||||
mock_token.assert_not_called()
|
||||
assert not Placement.objects.exists()
|
||||
|
||||
|
||||
@mock.patch("core.utils.generate_token", return_value=livekit_token)
|
||||
def test_api_rooms_join_placed_room_creation_failure(
|
||||
mock_token, two_livekit_clusters, fake_livekit_room_creation
|
||||
):
|
||||
"""
|
||||
Joining a room placed earlier, whose LiveKit cluster cannot create its
|
||||
LiveKit room, should return a 503 without credentials, and keep its
|
||||
placement: participants may be connected there.
|
||||
"""
|
||||
_paris, lyon = two_livekit_clusters
|
||||
room = RoomFactory(access_level=RoomAccessLevel.PUBLIC)
|
||||
placement = Placement.objects.create(
|
||||
room_name=str(room.id), room=room, cluster="lyon", sid="RM_old"
|
||||
)
|
||||
fake_livekit_room_creation.side_effect = LiveKitRoomCreationError("Timed out")
|
||||
|
||||
response = APIClient().post(join_url(room.id), {}, format="json")
|
||||
|
||||
assert response.status_code == 503
|
||||
assert response.json() == {
|
||||
"detail": "This room cannot be joined for now, please try again."
|
||||
}
|
||||
mock_token.assert_not_called()
|
||||
fake_livekit_room_creation.assert_called_once_with(str(room.id), lyon)
|
||||
kept_placement = Placement.objects.get()
|
||||
assert kept_placement.pk == placement.pk
|
||||
assert kept_placement.cluster == "lyon"
|
||||
assert kept_placement.sid == "RM_old"
|
||||
assert kept_placement.updated_at == placement.updated_at
|
||||
|
||||
|
||||
@mock.patch("core.utils.generate_token", return_value=livekit_token)
|
||||
def test_api_rooms_join_without_candidate(
|
||||
mock_token, settings, fake_livekit_room_creation, caplog
|
||||
):
|
||||
"""
|
||||
Joining a room should return a 503 without credentials while no LiveKit
|
||||
cluster may receive new placements, which is logged as critical.
|
||||
"""
|
||||
settings.LIVEKIT_CLUSTERS = json.dumps([livekit_cluster_entry("paris", weight=0)])
|
||||
room = RoomFactory(access_level=RoomAccessLevel.PUBLIC)
|
||||
|
||||
with caplog.at_level(logging.CRITICAL, logger="core.livekit.placements"):
|
||||
response = APIClient().post(join_url(room.id), {}, format="json")
|
||||
|
||||
assert response.status_code == 503
|
||||
assert response.json() == {
|
||||
"detail": "This room cannot be joined for now, please try again."
|
||||
}
|
||||
mock_token.assert_not_called()
|
||||
fake_livekit_room_creation.assert_not_called()
|
||||
assert "No LiveKit cluster may receive new placements" in caplog.text
|
||||
|
||||
|
||||
def test_api_rooms_join_get_not_allowed():
|
||||
"""Joining a room should only be possible with a POST request."""
|
||||
room = RoomFactory(access_level=RoomAccessLevel.PUBLIC)
|
||||
@@ -231,6 +409,7 @@ def test_api_rooms_join_unregistered_allowed(mock_token, authenticated, settings
|
||||
sources=None,
|
||||
role=None,
|
||||
participant_id=None,
|
||||
cluster=get_default_livekit_cluster(),
|
||||
)
|
||||
|
||||
|
||||
@@ -263,6 +442,7 @@ def test_api_rooms_join_unregistered_allowed_not_normalized(mock_token, settings
|
||||
sources=None,
|
||||
role=None,
|
||||
participant_id=None,
|
||||
cluster=get_default_livekit_cluster(),
|
||||
)
|
||||
|
||||
|
||||
@@ -296,6 +476,65 @@ def test_api_rooms_join_unregistered_not_allowed(mock_token, settings):
|
||||
mock_token.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("length", "status_code"), [(100, 200), (101, 404)])
|
||||
@mock.patch("core.utils.generate_token", return_value=livekit_token)
|
||||
def test_api_rooms_join_unregistered_slug_length(
|
||||
mock_token, length, status_code, settings
|
||||
):
|
||||
"""
|
||||
Joining an unregistered room whose slug is longer than a room's slug may be
|
||||
should return a 404 rather than credentials: no room could ever have it.
|
||||
"""
|
||||
settings.ALLOW_UNREGISTERED_ROOMS = True
|
||||
|
||||
response = APIClient().post(join_url("a" * length), {}, format="json")
|
||||
|
||||
assert response.status_code == status_code
|
||||
assert mock_token.called is (status_code == 200)
|
||||
assert Placement.objects.filter(room_name="a" * length).exists() is (
|
||||
status_code == 200
|
||||
)
|
||||
|
||||
|
||||
@mock.patch("core.utils.generate_token", return_value=livekit_token)
|
||||
def test_api_rooms_join_unregistered_connection_test_slug(
|
||||
mock_token, settings, fake_livekit_room_creation
|
||||
):
|
||||
"""
|
||||
Joining an unregistered room named like a connection test room should
|
||||
return a 404 rather than credentials: LiveKit webhooks ignore such rooms,
|
||||
so their placement would never be deleted.
|
||||
"""
|
||||
settings.ALLOW_UNREGISTERED_ROOMS = True
|
||||
settings.CONNECTION_TEST_ROOM_PREFIX = "connection-test"
|
||||
|
||||
response = APIClient().post(join_url("Connection-Test-Room"), {}, format="json")
|
||||
|
||||
assert response.status_code == 404
|
||||
assert response.json() == {"detail": "No Room matches the given query."}
|
||||
mock_token.assert_not_called()
|
||||
fake_livekit_room_creation.assert_not_called()
|
||||
assert not Placement.objects.exists()
|
||||
|
||||
|
||||
@mock.patch("core.utils.generate_token", return_value=livekit_token)
|
||||
def test_api_rooms_join_registered_connection_test_slug(mock_token, settings):
|
||||
"""
|
||||
A registered room keeps being joined by a slug starting like the names of
|
||||
connection test rooms: its LiveKit room is named after its id.
|
||||
"""
|
||||
settings.ALLOW_UNREGISTERED_ROOMS = True
|
||||
settings.CONNECTION_TEST_ROOM_PREFIX = "connection-test"
|
||||
room = RoomFactory(name="Connection test room", access_level=RoomAccessLevel.PUBLIC)
|
||||
|
||||
response = APIClient().post(join_url(room.slug), {}, format="json")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["livekit"]["room"] == str(room.id)
|
||||
mock_token.assert_called_once()
|
||||
assert Placement.objects.get().room_name == str(room.id)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("lookup", ["---", "!!!", "_"])
|
||||
@mock.patch("core.utils.generate_token", return_value=livekit_token)
|
||||
def test_api_rooms_join_unregistered_empty_slug(mock_token, lookup, settings):
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
"""
|
||||
Test rooms API endpoints in the Meet core app: throttling of the joins placing
|
||||
unregistered rooms.
|
||||
"""
|
||||
|
||||
from django.utils.crypto import get_random_string
|
||||
|
||||
import pytest
|
||||
from rest_framework.test import APIClient
|
||||
|
||||
from ...factories import RoomFactory, UserFactory
|
||||
from ...models import Placement, RoomAccessLevel
|
||||
from ..utils import random_remote_addr
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
|
||||
|
||||
def join_url(room_id):
|
||||
"""Return the url to join a room."""
|
||||
return f"/api/v1.0/rooms/{room_id!s}/join/"
|
||||
|
||||
|
||||
@pytest.fixture(name="unregistered_room_placement_throttle")
|
||||
def unregistered_room_placement_throttle_fixture(monkeypatch, settings):
|
||||
"""Allow unregistered rooms, two of which may be placed per minute."""
|
||||
settings.ALLOW_UNREGISTERED_ROOMS = True
|
||||
monkeypatch.setitem(
|
||||
settings.REST_FRAMEWORK["DEFAULT_THROTTLE_RATES"],
|
||||
"unregistered_room_placement",
|
||||
"2/minute",
|
||||
)
|
||||
|
||||
|
||||
def place_unregistered_rooms(client, count=2):
|
||||
"""Join as many unregistered rooms that are not placed yet."""
|
||||
for _ in range(count):
|
||||
response = client.post(
|
||||
join_url(f"room-{get_random_string(12)}"), {}, format="json"
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("unregistered_room_placement_throttle")
|
||||
def test_api_rooms_join_unregistered_placement_throttled(fake_livekit_room_creation):
|
||||
"""Joins placing unregistered rooms should be refused over the rate, before
|
||||
anything is placed or created, while another address may still place one."""
|
||||
client = APIClient(REMOTE_ADDR=random_remote_addr())
|
||||
place_unregistered_rooms(client)
|
||||
fake_livekit_room_creation.reset_mock()
|
||||
|
||||
response = client.post(join_url("one-room-too-many"), {}, format="json")
|
||||
|
||||
assert response.status_code == 429
|
||||
assert "livekit" not in response.json()
|
||||
assert int(response.headers["Retry-After"]) > 0
|
||||
assert not Placement.objects.filter(room_name="one-room-too-many").exists()
|
||||
fake_livekit_room_creation.assert_not_called()
|
||||
|
||||
response = APIClient(REMOTE_ADDR=random_remote_addr()).post(
|
||||
join_url("one-room-too-many"), {}, format="json"
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert Placement.objects.filter(room_name="one-room-too-many").exists()
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("unregistered_room_placement_throttle")
|
||||
def test_api_rooms_join_unregistered_placement_throttled_per_user():
|
||||
"""Authenticated callers should be counted each on their own, whatever the
|
||||
address they share or change."""
|
||||
address = random_remote_addr()
|
||||
user = UserFactory()
|
||||
client = APIClient(REMOTE_ADDR=address)
|
||||
client.force_login(user)
|
||||
place_unregistered_rooms(client)
|
||||
|
||||
other_client = APIClient(REMOTE_ADDR=address)
|
||||
other_client.force_login(UserFactory())
|
||||
place_unregistered_rooms(other_client)
|
||||
place_unregistered_rooms(APIClient(REMOTE_ADDR=address))
|
||||
|
||||
elsewhere = APIClient(REMOTE_ADDR=random_remote_addr())
|
||||
elsewhere.force_login(user)
|
||||
response = elsewhere.post(join_url("one-room-too-many"), {}, format="json")
|
||||
|
||||
assert response.status_code == 429
|
||||
assert not Placement.objects.filter(room_name="one-room-too-many").exists()
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("unregistered_room_placement_throttle")
|
||||
def test_api_rooms_join_placed_unregistered_room_not_throttled():
|
||||
"""Joining an unregistered room that is placed already should neither be
|
||||
counted nor refused: its participants ask again while they reconnect."""
|
||||
Placement.objects.create(room_name="placed-room", cluster="default")
|
||||
client = APIClient(REMOTE_ADDR=random_remote_addr())
|
||||
|
||||
for _ in range(3):
|
||||
response = client.post(join_url("placed-room"), {}, format="json")
|
||||
assert response.status_code == 200
|
||||
|
||||
place_unregistered_rooms(client)
|
||||
|
||||
for slug in ["placed-room", "Placed Room"]:
|
||||
response = client.post(join_url(slug), {}, format="json")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["livekit"]["room"] == "placed-room"
|
||||
|
||||
response = client.post(join_url("one-room-too-many"), {}, format="json")
|
||||
assert response.status_code == 429
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("unregistered_room_placement_throttle")
|
||||
def test_api_rooms_join_registered_room_not_throttled():
|
||||
"""Joining registered rooms, placed or not, should neither be counted as
|
||||
placing unregistered rooms nor refused once that rate is spent."""
|
||||
client = APIClient(REMOTE_ADDR=random_remote_addr())
|
||||
|
||||
for _ in range(3):
|
||||
room = RoomFactory(access_level=RoomAccessLevel.PUBLIC)
|
||||
response = client.post(join_url(room.slug), {}, format="json")
|
||||
assert response.status_code == 200
|
||||
|
||||
place_unregistered_rooms(client)
|
||||
|
||||
room = RoomFactory(access_level=RoomAccessLevel.PUBLIC)
|
||||
response = client.post(join_url(room.id), {}, format="json")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["livekit"]["room"] == str(room.id)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("unregistered_room_placement_throttle")
|
||||
def test_api_rooms_join_refused_slugs_not_throttled():
|
||||
"""Slugs that are no unregistered room should answer a 404 without being
|
||||
counted as placing one."""
|
||||
client = APIClient(REMOTE_ADDR=random_remote_addr())
|
||||
|
||||
for _ in range(3):
|
||||
response = client.post(join_url("a" * 101), {}, format="json")
|
||||
assert response.status_code == 404
|
||||
|
||||
place_unregistered_rooms(client)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("placed", [False, True])
|
||||
def test_api_rooms_join_unregistered_num_queries(
|
||||
placed, settings, django_assert_num_queries
|
||||
):
|
||||
"""Looking for the placement of an unregistered room before throttling its
|
||||
join should take a single query more."""
|
||||
settings.ALLOW_UNREGISTERED_ROOMS = True
|
||||
if placed:
|
||||
Placement.objects.create(room_name="unregistered-room", cluster="default")
|
||||
client = APIClient(REMOTE_ADDR=random_remote_addr())
|
||||
|
||||
with django_assert_num_queries(4 if placed else 6):
|
||||
response = client.post(join_url("unregistered-room"), {}, format="json")
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -9,14 +9,17 @@ from unittest import mock
|
||||
from django.core import signing
|
||||
from django.core.cache import cache
|
||||
|
||||
import jwt
|
||||
import pytest
|
||||
from freezegun import freeze_time
|
||||
from rest_framework.test import APIClient
|
||||
|
||||
from ... import utils
|
||||
from ...factories import RoomFactory, UserFactory
|
||||
from ...models import RoomAccessLevel
|
||||
from ...livekit.exceptions import LiveKitRoomCreationError
|
||||
from ...models import Placement, RoomAccessLevel
|
||||
from ...services.lobby import LobbyService
|
||||
from ..utils import random_remote_addr
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
|
||||
@@ -347,6 +350,108 @@ def test_request_entry_waiting_participant_public_room(settings):
|
||||
assert not LobbyService()._index_members(room.id)
|
||||
|
||||
|
||||
def test_request_entry_public_room_placed_cluster(
|
||||
two_livekit_clusters, fake_livekit_room_creation
|
||||
):
|
||||
"""Entry granted at once should come with credentials for the LiveKit cluster
|
||||
the LiveKit room is placed on, where it is created: its URL, and a token it
|
||||
signed."""
|
||||
_paris, lyon = two_livekit_clusters
|
||||
room = RoomFactory(access_level=RoomAccessLevel.PUBLIC)
|
||||
|
||||
with mock.patch("core.livekit.strategies.randbelow", return_value=1):
|
||||
response = APIClient(REMOTE_ADDR=random_remote_addr()).post(
|
||||
f"/api/v1.0/rooms/{room.id}/request-entry/", {"username": "Guest"}
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["status"] == "accepted"
|
||||
config = response.json()["livekit"]
|
||||
assert config["url"] == lyon.url
|
||||
claims = jwt.decode(config["token"], lyon.api_secret, algorithms=["HS256"])
|
||||
assert claims["iss"] == lyon.api_key
|
||||
assert claims["video"]["room"] == str(room.id)
|
||||
fake_livekit_room_creation.assert_called_once_with(str(room.id), lyon)
|
||||
assert Placement.objects.get(room=room).cluster == "lyon"
|
||||
|
||||
|
||||
def test_request_entry_accepted_participant_placed_cluster(
|
||||
two_livekit_clusters, fake_livekit_room_creation
|
||||
):
|
||||
"""A participant admitted from the lobby should be handed out credentials for
|
||||
the LiveKit cluster the LiveKit room is placed on, where it is created: its
|
||||
URL, and a token it signed."""
|
||||
_paris, lyon = two_livekit_clusters
|
||||
room = RoomFactory(access_level=RoomAccessLevel.RESTRICTED)
|
||||
Placement.objects.create(room_name=str(room.id), room=room, cluster="lyon")
|
||||
client = APIClient(REMOTE_ADDR=random_remote_addr())
|
||||
|
||||
with mock.patch.object(utils, "notify_participants", return_value=None):
|
||||
response = client.post(
|
||||
f"/api/v1.0/rooms/{room.id}/request-entry/", {"username": "Guest"}
|
||||
)
|
||||
assert response.json()["livekit"] is None
|
||||
fake_livekit_room_creation.assert_not_called()
|
||||
participant_id = response.json()["id"]
|
||||
LobbyService().handle_participant_entry(
|
||||
room.id, _lobby_signer().unsign(participant_id), True
|
||||
)
|
||||
|
||||
response = client.post(
|
||||
f"/api/v1.0/rooms/{room.id}/request-entry/",
|
||||
{"username": "Guest", "participant_id": participant_id},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["status"] == "accepted"
|
||||
config = response.json()["livekit"]
|
||||
assert config["url"] == lyon.url
|
||||
claims = jwt.decode(config["token"], lyon.api_secret, algorithms=["HS256"])
|
||||
assert claims["iss"] == lyon.api_key
|
||||
assert claims["video"]["room"] == str(room.id)
|
||||
fake_livekit_room_creation.assert_called_once_with(str(room.id), lyon)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("access_level", ["public", "restricted"])
|
||||
def test_request_entry_room_creation_failure(
|
||||
access_level, settings, fake_livekit_room_creation
|
||||
):
|
||||
"""Entry granted to a room whose LiveKit room cannot be created should be
|
||||
answered with a 503, leaving the participant where they were in the lobby."""
|
||||
settings.LOBBY_KEY_PREFIX = "mocked-cache-prefix"
|
||||
room = RoomFactory(access_level=access_level)
|
||||
participant_id = str(uuid.uuid4())
|
||||
status = "waiting" if access_level == "public" else "accepted"
|
||||
cache.set(
|
||||
f"mocked-cache-prefix_{room.id}_{participant_id}",
|
||||
{
|
||||
"id": participant_id,
|
||||
"username": "Guest",
|
||||
"status": status,
|
||||
"color": "#123456",
|
||||
"entered_at": "2025-01-01T10:00:00+00:00",
|
||||
},
|
||||
)
|
||||
fake_livekit_room_creation.side_effect = LiveKitRoomCreationError("Timed out")
|
||||
|
||||
response = APIClient(REMOTE_ADDR=random_remote_addr()).post(
|
||||
f"/api/v1.0/rooms/{room.id}/request-entry/",
|
||||
{
|
||||
"username": "Guest",
|
||||
"participant_id": _lobby_signer().sign(participant_id),
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 503
|
||||
assert response.json() == {
|
||||
"detail": "This room cannot be joined for now, please try again."
|
||||
}
|
||||
assert (
|
||||
cache.get(f"mocked-cache-prefix_{room.id}_{participant_id}")["status"] == status
|
||||
)
|
||||
assert not Placement.objects.exists()
|
||||
|
||||
|
||||
def test_request_entry_invalid_data():
|
||||
"""Should return 400 for invalid request data."""
|
||||
room = RoomFactory()
|
||||
|
||||
@@ -174,6 +174,34 @@ def test_api_rooms_retrieve_anonymous_unregistered_empty_slug(lookup):
|
||||
assert response.json() == {"detail": "No Room matches the given query."}
|
||||
|
||||
|
||||
@override_settings(ALLOW_UNREGISTERED_ROOMS=True)
|
||||
def test_api_rooms_retrieve_anonymous_unregistered_slug_too_long():
|
||||
"""
|
||||
Retrieving an unregistered room whose slug is longer than a room's slug may
|
||||
be should return a 404: no room could ever have it.
|
||||
"""
|
||||
client = APIClient()
|
||||
response = client.get(f"/api/v1.0/rooms/{'a' * 101}/")
|
||||
|
||||
assert response.status_code == 404
|
||||
assert response.json() == {"detail": "No Room matches the given query."}
|
||||
|
||||
|
||||
@override_settings(
|
||||
ALLOW_UNREGISTERED_ROOMS=True, CONNECTION_TEST_ROOM_PREFIX="connection-test"
|
||||
)
|
||||
def test_api_rooms_retrieve_anonymous_unregistered_connection_test_slug():
|
||||
"""
|
||||
Retrieving an unregistered room named like a connection test room should
|
||||
return a 404: it could never be joined.
|
||||
"""
|
||||
client = APIClient()
|
||||
response = client.get("/api/v1.0/rooms/connection-test-room/")
|
||||
|
||||
assert response.status_code == 404
|
||||
assert response.json() == {"detail": "No Room matches the given query."}
|
||||
|
||||
|
||||
@override_settings(ALLOW_UNREGISTERED_ROOMS=True)
|
||||
@pytest.mark.parametrize("template", ["{}!", "_{}", "{}~"])
|
||||
def test_api_rooms_retrieve_anonymous_unregistered_uuid_slug(template):
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
Test rooms API endpoints in the Meet core app: start recording.
|
||||
"""
|
||||
|
||||
# pylint: disable=redefined-outer-name,unused-argument,no-member,too-many-lines
|
||||
# pylint: disable=redefined-outer-name,unused-argument,no-member,too-many-lines,protected-access
|
||||
|
||||
from unittest import mock
|
||||
|
||||
@@ -10,9 +10,11 @@ import pytest
|
||||
from livekit import api as livekit_api
|
||||
from rest_framework.test import APIClient
|
||||
|
||||
from ... import utils
|
||||
from ...audit.testing import capture_audit, find_events
|
||||
from ...factories import RoomFactory, UserFactory
|
||||
from ...models import Recording, RoomAccessLevel
|
||||
from ...livekit.clusters import get_default_livekit_cluster
|
||||
from ...models import Placement, Recording, RoomAccessLevel
|
||||
from ...recording.worker.exceptions import RecordingStartError
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
@@ -155,7 +157,9 @@ def test_start_recording_worker_error(
|
||||
{"mode": "screen_recording"},
|
||||
)
|
||||
|
||||
mock_worker_service_factory.assert_called_once_with(mode="screen_recording")
|
||||
mock_worker_service_factory.assert_called_once_with(
|
||||
mode="screen_recording", cluster=get_default_livekit_cluster()
|
||||
)
|
||||
|
||||
assert response.status_code == 502
|
||||
assert response.json() == {
|
||||
@@ -265,7 +269,9 @@ def test_start_recording_success(
|
||||
{"mode": "screen_recording"},
|
||||
)
|
||||
|
||||
mock_worker_service_factory.assert_called_once_with(mode="screen_recording")
|
||||
mock_worker_service_factory.assert_called_once_with(
|
||||
mode="screen_recording", cluster=get_default_livekit_cluster()
|
||||
)
|
||||
|
||||
assert response.status_code == 201
|
||||
assert response.json() == {
|
||||
@@ -278,6 +284,7 @@ def test_start_recording_success(
|
||||
|
||||
assert recording.room == room
|
||||
assert recording.mode == "screen_recording"
|
||||
assert recording.livekit_cluster == get_default_livekit_cluster().name
|
||||
|
||||
# Verify recording access details
|
||||
assert recording.accesses.count() == 1
|
||||
@@ -286,6 +293,54 @@ def test_start_recording_success(
|
||||
assert access.role == "owner"
|
||||
|
||||
|
||||
@mock.patch("core.utils.create_livekit_client")
|
||||
def test_start_recording_on_the_placed_cluster(
|
||||
mock_create_livekit_client, settings, two_livekit_clusters
|
||||
):
|
||||
"""A room is recorded on the LiveKit cluster it is placed on, which the
|
||||
recording keeps: the egress, the room's metadata and the metadata collector
|
||||
agent are all on that cluster."""
|
||||
settings.RECORDING_ENABLE = True
|
||||
settings.METADATA_COLLECTOR_ENABLED = True
|
||||
_paris, lyon = two_livekit_clusters
|
||||
room = RoomFactory()
|
||||
user = UserFactory()
|
||||
room.accesses.create(user=user, role="owner")
|
||||
Placement.objects.create(room_name=str(room.id), room=room, cluster="lyon")
|
||||
client = mock.AsyncMock()
|
||||
client._egress.start_room_composite_egress.return_value = mock.Mock(
|
||||
egress_id="EG_lyon"
|
||||
)
|
||||
client.room.list_rooms.return_value = mock.Mock(rooms=[mock.Mock(metadata="")])
|
||||
client.agent_dispatch.create_dispatch.return_value = mock.Mock(id="AD_lyon")
|
||||
mock_create_livekit_client.return_value = client
|
||||
api_client = APIClient()
|
||||
api_client.force_login(user)
|
||||
|
||||
response = api_client.post(
|
||||
f"/api/v1.0/rooms/{room.id}/start-recording/",
|
||||
{
|
||||
"mode": "screen_recording",
|
||||
"options": {"transcribe": True, "collect_metadata": True},
|
||||
},
|
||||
format="json",
|
||||
)
|
||||
|
||||
assert response.status_code == 201
|
||||
recording = Recording.objects.get(room=room)
|
||||
assert recording.livekit_cluster == "lyon"
|
||||
assert recording.worker_id == "EG_lyon"
|
||||
assert recording.options["metadata_collector_dispatch_id"] == "AD_lyon"
|
||||
assert mock_create_livekit_client.call_args_list == [
|
||||
mock.call(utils.livekit_api_configuration(lyon)),
|
||||
mock.call(cluster=lyon),
|
||||
mock.call(cluster=lyon),
|
||||
]
|
||||
client._egress.start_room_composite_egress.assert_awaited_once()
|
||||
client.room.update_room_metadata.assert_awaited_once()
|
||||
client.agent_dispatch.create_dispatch.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["fr", "en", "nl", "de"])
|
||||
def test_start_recording_options_language_valid(
|
||||
settings, mock_worker_service_factory, mock_worker_manager, value
|
||||
|
||||
@@ -2,16 +2,19 @@
|
||||
Test rooms API endpoints in the Meet core app: stop recording.
|
||||
"""
|
||||
|
||||
# pylint: disable=redefined-outer-name,unused-argument
|
||||
# pylint: disable=redefined-outer-name,unused-argument,no-member,protected-access
|
||||
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
from livekit import api as livekit_api
|
||||
from rest_framework.test import APIClient
|
||||
|
||||
from ... import utils
|
||||
from ...audit.testing import find_events
|
||||
from ...factories import RecordingFactory, RoomFactory, UserFactory
|
||||
from ...models import Recording, RecordingStatusChoices, RoomAccessLevel
|
||||
from ...livekit.clusters import get_default_livekit_cluster
|
||||
from ...models import Placement, Recording, RecordingStatusChoices, RoomAccessLevel
|
||||
from ...recording.worker.exceptions import RecordingStopError
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
@@ -143,7 +146,9 @@ def test_stop_recording_worker_error(
|
||||
|
||||
response = client.post(f"/api/v1.0/rooms/{room.id}/stop-recording/")
|
||||
|
||||
mock_worker_service_factory.assert_called_once_with(mode="screen_recording")
|
||||
mock_worker_service_factory.assert_called_once_with(
|
||||
mode="screen_recording", cluster=get_default_livekit_cluster()
|
||||
)
|
||||
mock_stop.assert_called_once_with(recording)
|
||||
|
||||
assert response.status_code == 500
|
||||
@@ -179,7 +184,9 @@ def test_stop_recording_success(
|
||||
|
||||
response = client.post(f"/api/v1.0/rooms/{room.id}/stop-recording/")
|
||||
|
||||
mock_worker_service_factory.assert_called_once_with(mode="screen_recording")
|
||||
mock_worker_service_factory.assert_called_once_with(
|
||||
mode="screen_recording", cluster=get_default_livekit_cluster()
|
||||
)
|
||||
mock_stop.assert_called_once_with(recording)
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -189,6 +196,76 @@ def test_stop_recording_success(
|
||||
assert Recording.objects.count() == 1
|
||||
|
||||
|
||||
@mock.patch("core.utils.create_livekit_client")
|
||||
def test_stop_recording_on_the_recording_cluster(
|
||||
mock_create_livekit_client, settings, two_livekit_clusters
|
||||
):
|
||||
"""A recording is stopped on the LiveKit cluster recording the room, even
|
||||
once the room was placed elsewhere."""
|
||||
settings.RECORDING_ENABLE = True
|
||||
_paris, lyon = two_livekit_clusters
|
||||
room = RoomFactory()
|
||||
user = UserFactory()
|
||||
room.accesses.create(user=user, role="owner")
|
||||
recording = RecordingFactory(
|
||||
room=room,
|
||||
status=RecordingStatusChoices.ACTIVE,
|
||||
worker_id="EG_lyon",
|
||||
livekit_cluster="lyon",
|
||||
)
|
||||
Placement.objects.create(room_name=str(room.id), room=room, cluster="paris")
|
||||
client = mock.AsyncMock()
|
||||
client._egress.stop_egress.return_value = mock.Mock(
|
||||
status=livekit_api.EgressStatus.EGRESS_ENDING
|
||||
)
|
||||
mock_create_livekit_client.return_value = client
|
||||
api_client = APIClient()
|
||||
api_client.force_login(user)
|
||||
|
||||
response = api_client.post(f"/api/v1.0/rooms/{room.id}/stop-recording/")
|
||||
|
||||
assert response.status_code == 200
|
||||
mock_create_livekit_client.assert_called_once_with(
|
||||
utils.livekit_api_configuration(lyon)
|
||||
)
|
||||
client._egress.stop_egress.assert_awaited_once_with(
|
||||
livekit_api.StopEgressRequest(egress_id="EG_lyon")
|
||||
)
|
||||
recording.refresh_from_db()
|
||||
assert recording.status == RecordingStatusChoices.STOPPED
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("mock_worker_manager")
|
||||
@pytest.mark.parametrize("livekit_cluster", ["", "lille"])
|
||||
def test_stop_recording_on_the_default_cluster(
|
||||
livekit_cluster, mock_worker_service_factory, settings, two_livekit_clusters
|
||||
):
|
||||
"""A recording naming no LiveKit cluster, as one started before recordings
|
||||
kept theirs, or naming a cluster no longer registered, is stopped on the
|
||||
default cluster, wherever the room is placed."""
|
||||
settings.RECORDING_ENABLE = True
|
||||
paris, _lyon = two_livekit_clusters
|
||||
room = RoomFactory()
|
||||
user = UserFactory()
|
||||
room.accesses.create(user=user, role="owner")
|
||||
RecordingFactory(
|
||||
room=room,
|
||||
status=RecordingStatusChoices.ACTIVE,
|
||||
mode="screen_recording",
|
||||
livekit_cluster=livekit_cluster,
|
||||
)
|
||||
Placement.objects.create(room_name=str(room.id), room=room, cluster="lyon")
|
||||
client = APIClient()
|
||||
client.force_login(user)
|
||||
|
||||
response = client.post(f"/api/v1.0/rooms/{room.id}/stop-recording/")
|
||||
|
||||
assert response.status_code == 200
|
||||
mock_worker_service_factory.assert_called_once_with(
|
||||
mode="screen_recording", cluster=paris
|
||||
)
|
||||
|
||||
|
||||
def test_stop_recording_is_audited(
|
||||
settings, mock_worker_service_factory, mock_worker_manager, audit_events
|
||||
):
|
||||
|
||||
@@ -7,6 +7,7 @@ import uuid
|
||||
from unittest import mock
|
||||
|
||||
from django.conf import settings
|
||||
from django.contrib.auth.models import AnonymousUser
|
||||
|
||||
import pytest
|
||||
from livekit.api import AccessToken, TwirpError, VideoGrants
|
||||
@@ -14,6 +15,7 @@ from rest_framework.test import APIClient
|
||||
|
||||
from core.factories import RoomFactory, UserFactory
|
||||
from core.tests.utils import generate_user_access_token
|
||||
from core.utils import generate_token
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
|
||||
@@ -214,7 +216,10 @@ def test_start_subtitle_wrong_signature(settings, mock_livekit_token):
|
||||
"""Test that tokens signed with incorrect signature are rejected."""
|
||||
|
||||
settings.ROOM_SUBTITLE_ENABLED = True
|
||||
settings.LIVEKIT_CONFIGURATION["api_secret"] = "wrong-secret-padded-to-32-bytes!!"
|
||||
settings.LIVEKIT_CONFIGURATION = {
|
||||
**settings.LIVEKIT_CONFIGURATION,
|
||||
"api_secret": "wrong-secret-padded-to-32-bytes!!",
|
||||
}
|
||||
|
||||
room = RoomFactory()
|
||||
client = APIClient()
|
||||
@@ -231,6 +236,58 @@ def test_start_subtitle_wrong_signature(settings, mock_livekit_token):
|
||||
}
|
||||
|
||||
|
||||
def test_start_subtitle_token_from_another_cluster(
|
||||
settings, mock_livekit_client, two_livekit_clusters
|
||||
):
|
||||
"""Test that a token signed by any registered LiveKit cluster is accepted."""
|
||||
|
||||
settings.ROOM_SUBTITLE_ENABLED = True
|
||||
_paris, lyon = two_livekit_clusters
|
||||
|
||||
room = RoomFactory()
|
||||
token = generate_token(room=str(room.id), user=AnonymousUser(), cluster=lyon)
|
||||
client = APIClient()
|
||||
|
||||
response = client.post(
|
||||
f"/api/v1.0/rooms/{room.id}/start-subtitle/",
|
||||
{},
|
||||
HTTP_AUTHORIZATION=f"X-LiveKit-Token {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"status": "success"}
|
||||
mock_livekit_client.agent_dispatch.create_dispatch.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("two_livekit_clusters")
|
||||
def test_start_subtitle_token_from_an_unknown_cluster(settings, mock_livekit_client):
|
||||
"""Test that a token signed with an API key no cluster uses is rejected."""
|
||||
|
||||
settings.ROOM_SUBTITLE_ENABLED = True
|
||||
|
||||
room = RoomFactory()
|
||||
token = (
|
||||
AccessToken(
|
||||
api_key="key-lille",
|
||||
api_secret="secret-lille-padded-for-minimum-len!-livekit",
|
||||
)
|
||||
.with_grants(VideoGrants(room=str(room.id), room_join=True))
|
||||
.with_identity(str(uuid.uuid4()))
|
||||
.to_jwt()
|
||||
)
|
||||
client = APIClient()
|
||||
|
||||
response = client.post(
|
||||
f"/api/v1.0/rooms/{room.id}/start-subtitle/",
|
||||
{},
|
||||
HTTP_AUTHORIZATION=f"X-LiveKit-Token {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 403
|
||||
assert response.json() == {"detail": "Invalid LiveKit token: Invalid issuer"}
|
||||
mock_livekit_client.agent_dispatch.create_dispatch.assert_not_called()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def user_access_token():
|
||||
"""Generate a valid user access JWT, sent with the "Bearer" scheme."""
|
||||
|
||||
@@ -14,7 +14,10 @@ import pytest
|
||||
from livekit import api
|
||||
|
||||
from ...factories import RecordingFactory
|
||||
from ...livekit.clusters import get_default_livekit_cluster
|
||||
from ...models import Placement
|
||||
from ...services.livekit_events import ActionFailedError, LiveKitEventsService
|
||||
from ..utils import livekit_webhook_token
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -119,6 +122,7 @@ def test_unknown_event_type(client, mock_livekit_config):
|
||||
assert response.json() == {"status": "success"}
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@mock.patch.object(LiveKitEventsService, "_handle_room_finished")
|
||||
def test_handled_event_type(
|
||||
mock_handler,
|
||||
@@ -140,6 +144,106 @@ def test_handled_event_type(
|
||||
assert response.json() == {"status": "success"}
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@mock.patch.object(LiveKitEventsService, "_handle_room_finished", autospec=True)
|
||||
def test_handled_event_from_another_cluster(
|
||||
mock_handler, client, serialized_event_data, two_livekit_clusters
|
||||
):
|
||||
"""Should process a webhook signed by any registered LiveKit cluster.
|
||||
|
||||
The service handling it knows which cluster sent it.
|
||||
"""
|
||||
_paris, lyon = two_livekit_clusters
|
||||
|
||||
response = client.post(
|
||||
"/api/v1.0/rooms/webhooks-livekit/",
|
||||
data=serialized_event_data,
|
||||
content_type="application/json",
|
||||
HTTP_AUTHORIZATION=livekit_webhook_token(
|
||||
serialized_event_data, lyon.api_key, lyon.api_secret
|
||||
),
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"status": "success"}
|
||||
mock_handler.assert_called_once()
|
||||
service = mock_handler.call_args.args[0]
|
||||
assert service.cluster == lyon
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@mock.patch.object(LiveKitEventsService, "_handle_room_finished")
|
||||
def test_room_finished_deletes_its_placement(
|
||||
mock_handler, client, webhook_event_data, serialized_event_data, auth_token
|
||||
):
|
||||
"""Should delete the placement of a LiveKit room that finished where it is
|
||||
placed, and handle the event."""
|
||||
room = webhook_event_data["room"]
|
||||
Placement.objects.create(room_name=room["name"], cluster="default", sid=room["sid"])
|
||||
|
||||
response = client.post(
|
||||
"/api/v1.0/rooms/webhooks-livekit/",
|
||||
data=serialized_event_data,
|
||||
content_type="application/json",
|
||||
HTTP_AUTHORIZATION=auth_token,
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"status": "success"}
|
||||
assert not Placement.objects.exists()
|
||||
mock_handler.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@mock.patch.object(LiveKitEventsService, "_handle_room_finished")
|
||||
def test_room_finished_on_another_cluster_than_placed(
|
||||
mock_handler,
|
||||
client,
|
||||
webhook_event_data,
|
||||
serialized_event_data,
|
||||
two_livekit_clusters,
|
||||
):
|
||||
"""Should acknowledge, and otherwise ignore, the end of a LiveKit room on
|
||||
another cluster than its placement's."""
|
||||
_paris, lyon = two_livekit_clusters
|
||||
room = webhook_event_data["room"]
|
||||
Placement.objects.create(room_name=room["name"], cluster="paris", sid=room["sid"])
|
||||
|
||||
response = client.post(
|
||||
"/api/v1.0/rooms/webhooks-livekit/",
|
||||
data=serialized_event_data,
|
||||
content_type="application/json",
|
||||
HTTP_AUTHORIZATION=livekit_webhook_token(
|
||||
serialized_event_data, lyon.api_key, lyon.api_secret
|
||||
),
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"status": "success"}
|
||||
assert Placement.objects.get().cluster == "paris"
|
||||
mock_handler.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("two_livekit_clusters")
|
||||
@mock.patch.object(LiveKitEventsService, "_handle_room_finished")
|
||||
def test_event_from_an_unknown_cluster(mock_handler, client, serialized_event_data):
|
||||
"""Should return 400 for a webhook signed with an API key no cluster uses."""
|
||||
response = client.post(
|
||||
"/api/v1.0/rooms/webhooks-livekit/",
|
||||
data=serialized_event_data,
|
||||
content_type="application/json",
|
||||
HTTP_AUTHORIZATION=livekit_webhook_token(
|
||||
serialized_event_data,
|
||||
"key-lille",
|
||||
"secret-lille-padded-for-minimum-len!-livekit",
|
||||
),
|
||||
)
|
||||
|
||||
assert response.status_code == 400
|
||||
assert response.json() == {"status": "error"}
|
||||
mock_handler.assert_not_called()
|
||||
|
||||
|
||||
def test_unhandled_event_type(client, mock_livekit_config):
|
||||
"""Should return 200 for event types that have no handler."""
|
||||
event_data = json.dumps({"event": "participant_joined"})
|
||||
@@ -162,6 +266,7 @@ def test_unhandled_event_type(client, mock_livekit_config):
|
||||
assert response.json() == {"status": "success"}
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_action_error(client, mock_livekit_config):
|
||||
"""Should raise exceptions when errors occur during LiveKit webhook processing."""
|
||||
event_data = json.dumps(
|
||||
@@ -253,6 +358,7 @@ def test_egress_ended_keeps_a_partial_transcript(
|
||||
mock_notify.assert_called_once_with(
|
||||
room_name=str(recording.room.id),
|
||||
notification_data={"type": "transcriptionInterrupted"},
|
||||
cluster=get_default_livekit_cluster(),
|
||||
)
|
||||
mock_notify_external_services.assert_called_once_with(recording)
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ Test LiveKitEvents service.
|
||||
"""
|
||||
# pylint: disable=W0621,W0613, W0212, E0611, E1101, C0302
|
||||
|
||||
import json
|
||||
import logging
|
||||
import uuid
|
||||
from datetime import timedelta
|
||||
@@ -16,7 +17,11 @@ from livekit.api import EgressStatus
|
||||
|
||||
from core.audit.testing import find_events
|
||||
from core.factories import RecordingFactory, RoomFactory
|
||||
from core.models import Room
|
||||
from core.livekit.clusters import (
|
||||
LiveKitClusterTokenVerifier,
|
||||
get_default_livekit_cluster,
|
||||
)
|
||||
from core.models import Placement, Room
|
||||
from core.recording.enums import RecordingWorkerEvent
|
||||
from core.recording.services.recording_events import RecordingEventsService
|
||||
from core.services.livekit_events import (
|
||||
@@ -32,6 +37,7 @@ from core.services.livekit_events import (
|
||||
to_recording_event,
|
||||
)
|
||||
from core.services.lobby import LobbyService
|
||||
from core.services.presence import PresenceCache
|
||||
from core.services.room_management import (
|
||||
RoomManagementException,
|
||||
RoomNotFoundException,
|
||||
@@ -40,11 +46,16 @@ from core.services.sip_management import (
|
||||
SIPException,
|
||||
SIPManagement,
|
||||
)
|
||||
from core.tests.utils import livekit_webhook_token
|
||||
from core.utils import NotificationError
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
fake = Faker()
|
||||
|
||||
SID = "RM_hycBMAjmt6Ub"
|
||||
OTHER_SID = "RM_dxSBWkfXbe4v"
|
||||
UUID_FILTER_REGEX = r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_livekit_config(settings):
|
||||
@@ -63,25 +74,93 @@ def service(mock_livekit_config):
|
||||
return LiveKitEventsService()
|
||||
|
||||
|
||||
@mock.patch("livekit.api.TokenVerifier")
|
||||
@mock.patch("livekit.api.WebhookReceiver")
|
||||
def test_initialization(
|
||||
mock_webhook_receiver, mock_token_verifier, mock_livekit_config
|
||||
):
|
||||
"""Should correctly initialize the service with required dependencies."""
|
||||
def test_initialization():
|
||||
"""Should correctly initialize the service with required dependencies.
|
||||
|
||||
api_key = mock_livekit_config["api_key"]
|
||||
api_secret = mock_livekit_config["api_secret"]
|
||||
Webhooks are verified with the keys of whichever LiveKit cluster signed them.
|
||||
"""
|
||||
|
||||
service = LiveKitEventsService()
|
||||
|
||||
mock_token_verifier.assert_called_once_with(api_key, api_secret)
|
||||
mock_webhook_receiver.assert_called_once_with(mock_token_verifier.return_value)
|
||||
assert isinstance(service.token_verifier, LiveKitClusterTokenVerifier)
|
||||
assert isinstance(service.lobby_service, LobbyService)
|
||||
assert isinstance(service.sip_management, SIPManagement)
|
||||
assert isinstance(service.recording_events, RecordingEventsService)
|
||||
|
||||
|
||||
def test_cluster_defaults_to_the_default_cluster(service):
|
||||
"""Before any webhook is received, the service's cluster is the default one.
|
||||
|
||||
Handlers called directly therefore act on the default cluster.
|
||||
"""
|
||||
assert service.cluster == get_default_livekit_cluster()
|
||||
assert service.cluster.name == "default"
|
||||
|
||||
|
||||
def webhook_request(body, api_key, api_secret):
|
||||
"""Return a webhook request whose body is signed with this API key and secret."""
|
||||
request = mock.MagicMock()
|
||||
request.headers = {
|
||||
"Authorization": livekit_webhook_token(body, api_key, api_secret)
|
||||
}
|
||||
request.body = body.encode()
|
||||
return request
|
||||
|
||||
|
||||
def room_event_request(cluster, event, room_name, sid=SID):
|
||||
"""Return the webhook request a LiveKit cluster sends on an event of a room."""
|
||||
body = json.dumps({"event": event, "room": {"name": room_name, "sid": sid}})
|
||||
return webhook_request(body, cluster.api_key, cluster.api_secret)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def room_cleanup(settings):
|
||||
"""Mock the cleanup of a room whose LiveKit room finished.
|
||||
|
||||
Provide the mocks clearing its presence and lobby caches. Telephony and
|
||||
roomkit are disabled, as they are with several LiveKit clusters: the room
|
||||
has no SIP dispatch rule to delete.
|
||||
"""
|
||||
settings.ROOM_TELEPHONY_ENABLED = False
|
||||
settings.ROOMKIT_ENABLED = False
|
||||
with (
|
||||
mock.patch.object(PresenceCache, "clear_room") as clear_presence,
|
||||
mock.patch.object(LobbyService, "clear_room_cache") as clear_lobby,
|
||||
):
|
||||
yield [clear_presence, clear_lobby]
|
||||
|
||||
|
||||
@mock.patch.object(LiveKitEventsService, "_handle_room_finished")
|
||||
def test_receive_identifies_the_cluster(
|
||||
mock_handle_room_finished, two_livekit_clusters
|
||||
):
|
||||
"""The cluster whose API key signed a webhook becomes the service's cluster."""
|
||||
_paris, lyon = two_livekit_clusters
|
||||
body = json.dumps({"event": "room_finished", "room": {"name": str(uuid.uuid4())}})
|
||||
|
||||
service = LiveKitEventsService()
|
||||
service.receive(webhook_request(body, lyon.api_key, lyon.api_secret))
|
||||
|
||||
mock_handle_room_finished.assert_called_once()
|
||||
assert service.cluster == lyon
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("two_livekit_clusters")
|
||||
@mock.patch.object(LiveKitEventsService, "_handle_room_finished")
|
||||
def test_receive_unknown_cluster(mock_handle_room_finished):
|
||||
"""A webhook signed with an API key no cluster uses is an invalid payload."""
|
||||
body = json.dumps({"event": "room_finished", "room": {"name": str(uuid.uuid4())}})
|
||||
request = webhook_request(
|
||||
body, "key-lille", "secret-lille-padded-for-minimum-len!-livekit"
|
||||
)
|
||||
|
||||
service = LiveKitEventsService()
|
||||
with pytest.raises(InvalidPayloadError, match="Invalid webhook payload"):
|
||||
service.receive(request)
|
||||
|
||||
mock_handle_room_finished.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mode", "notification_type"),
|
||||
(
|
||||
@@ -104,10 +183,14 @@ def test_handle_egress_ended_success( # pylint: disable=too-many-arguments, too
|
||||
service._handle_egress_ended(mock_data)
|
||||
|
||||
mock_notify.assert_called_once_with(
|
||||
room_name=str(recording.room.id), notification_data={"type": notification_type}
|
||||
room_name=str(recording.room.id),
|
||||
notification_data={"type": notification_type},
|
||||
cluster=get_default_livekit_cluster(),
|
||||
)
|
||||
mock_update_metadata.assert_called_once_with(
|
||||
str(recording.room.id), remove_keys=["recording_mode", "recording_status"]
|
||||
str(recording.room.id),
|
||||
remove_keys=["recording_mode", "recording_status"],
|
||||
cluster=get_default_livekit_cluster(),
|
||||
)
|
||||
|
||||
recording.refresh_from_db()
|
||||
@@ -137,7 +220,9 @@ def test_handle_egress_updated_success(
|
||||
service._handle_egress_updated(mock_data)
|
||||
|
||||
mock_update_metadata.assert_called_once_with(
|
||||
str(recording.room.id), {"recording_status": status}
|
||||
str(recording.room.id),
|
||||
{"recording_status": status},
|
||||
cluster=get_default_livekit_cluster(),
|
||||
)
|
||||
|
||||
|
||||
@@ -165,7 +250,9 @@ def test_handle_egress_ended_metadata_update_fails( # pylint: disable=too-many-
|
||||
service._handle_egress_ended(mock_data)
|
||||
|
||||
mock_notify.assert_called_once_with(
|
||||
room_name=str(recording.room.id), notification_data={"type": notification_type}
|
||||
room_name=str(recording.room.id),
|
||||
notification_data={"type": notification_type},
|
||||
cluster=get_default_livekit_cluster(),
|
||||
)
|
||||
recording.refresh_from_db()
|
||||
|
||||
@@ -197,9 +284,12 @@ def test_handle_egress_ended_notification_fails(
|
||||
mock_notify.assert_called_once_with(
|
||||
room_name=str(recording.room.id),
|
||||
notification_data={"type": "screenRecordingLimitReached"},
|
||||
cluster=get_default_livekit_cluster(),
|
||||
)
|
||||
mock_update_metadata.assert_called_once_with(
|
||||
str(recording.room.id), remove_keys=["recording_mode", "recording_status"]
|
||||
str(recording.room.id),
|
||||
remove_keys=["recording_mode", "recording_status"],
|
||||
cluster=get_default_livekit_cluster(),
|
||||
)
|
||||
|
||||
recording.refresh_from_db()
|
||||
@@ -254,7 +344,9 @@ def test_handle_egress_ended_recording_should_not_be_saved(
|
||||
|
||||
mock_notify.assert_not_called()
|
||||
mock_update_metadata.assert_called_once_with(
|
||||
str(recording.room.id), remove_keys=["recording_mode", "recording_status"]
|
||||
str(recording.room.id),
|
||||
remove_keys=["recording_mode", "recording_status"],
|
||||
cluster=get_default_livekit_cluster(),
|
||||
)
|
||||
|
||||
recording.refresh_from_db()
|
||||
@@ -289,7 +381,9 @@ def test_handle_egress_ended_complete_does_not_notify_participants(
|
||||
|
||||
mock_notify.assert_not_called()
|
||||
mock_update_metadata.assert_called_once_with(
|
||||
str(recording.room.id), remove_keys=["recording_mode", "recording_status"]
|
||||
str(recording.room.id),
|
||||
remove_keys=["recording_mode", "recording_status"],
|
||||
cluster=get_default_livekit_cluster(),
|
||||
)
|
||||
mock_notify_external_services.assert_called_once_with(recording)
|
||||
|
||||
@@ -354,6 +448,50 @@ def test_handle_egress_ended_does_not_call_metadata_collector_stop_when_conditio
|
||||
mock_collector.stop.assert_not_called()
|
||||
|
||||
|
||||
@mock.patch(
|
||||
"core.recording.services.recording_events.notification_service."
|
||||
"notify_external_services",
|
||||
return_value=True,
|
||||
)
|
||||
@mock.patch("core.utils.create_livekit_client")
|
||||
def test_handle_egress_ended_on_the_recording_cluster(
|
||||
mock_create_livekit_client,
|
||||
mock_notify_external_services,
|
||||
service,
|
||||
two_livekit_clusters,
|
||||
):
|
||||
"""An ended egress is handled on the LiveKit cluster recording the room.
|
||||
|
||||
The room's metadata is cleaned up, the metadata collector's dispatch deleted
|
||||
and participants notified there, even once the room was placed elsewhere.
|
||||
"""
|
||||
paris, lyon = two_livekit_clusters
|
||||
recording = RecordingFactory(
|
||||
worker_id="worker-1",
|
||||
status="active",
|
||||
livekit_cluster="lyon",
|
||||
options={"metadata_collector_dispatch_id": "dispatch-123"},
|
||||
)
|
||||
Placement.objects.create(room_name=str(recording.room.id), cluster="paris")
|
||||
client = mock.AsyncMock()
|
||||
client.room.list_rooms.return_value = mock.Mock(rooms=[mock.Mock(metadata="")])
|
||||
mock_create_livekit_client.return_value = client
|
||||
mock_data = mock.MagicMock()
|
||||
mock_data.egress_info.egress_id = recording.worker_id
|
||||
mock_data.egress_info.status = EgressStatus.EGRESS_LIMIT_REACHED
|
||||
|
||||
assert service.cluster == paris
|
||||
service._handle_egress_ended(mock_data)
|
||||
|
||||
assert mock_create_livekit_client.call_args_list == [mock.call(cluster=lyon)] * 3
|
||||
client.room.update_room_metadata.assert_awaited_once()
|
||||
client.agent_dispatch.delete_dispatch.assert_awaited_once_with(
|
||||
dispatch_id="dispatch-123", room_name=str(recording.room.id)
|
||||
)
|
||||
client.room.send_data.assert_awaited_once()
|
||||
mock_notify_external_services.assert_called_once_with(recording)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("egress_status", "recording_status", "event", "expected_level"),
|
||||
(
|
||||
@@ -638,11 +776,15 @@ def test_handle_egress_ended_interrupted_saves_partial_recording( # noqa: PLR09
|
||||
service._handle_egress_ended(data)
|
||||
|
||||
mock_notify.assert_called_once_with(
|
||||
room_name=str(recording.room.id), notification_data={"type": notification_type}
|
||||
room_name=str(recording.room.id),
|
||||
notification_data={"type": notification_type},
|
||||
cluster=get_default_livekit_cluster(),
|
||||
)
|
||||
mock_notify_external_services.assert_called_once_with(recording)
|
||||
mock_update_metadata.assert_called_once_with(
|
||||
str(recording.room.id), remove_keys=["recording_mode", "recording_status"]
|
||||
str(recording.room.id),
|
||||
remove_keys=["recording_mode", "recording_status"],
|
||||
cluster=get_default_livekit_cluster(),
|
||||
)
|
||||
assert (
|
||||
f"Recording worker reported interrupted for recording {recording.id}"
|
||||
@@ -714,6 +856,7 @@ def test_handle_egress_ended_failed_on_another_error_discards_recording(
|
||||
mock_notify.assert_called_once_with(
|
||||
room_name=str(recording.room.id),
|
||||
notification_data={"type": "transcriptionFailed"},
|
||||
cluster=get_default_livekit_cluster(),
|
||||
)
|
||||
|
||||
recording.refresh_from_db()
|
||||
@@ -1041,7 +1184,9 @@ def test_handle_room_started_closes_deleted_room(
|
||||
|
||||
service._handle_room_started(mock_data)
|
||||
|
||||
mock_delete_room.assert_called_once_with(str(room.id))
|
||||
mock_delete_room.assert_called_once_with(
|
||||
str(room.id), cluster=get_default_livekit_cluster()
|
||||
)
|
||||
mock_ensure_dispatch_rule.assert_not_called()
|
||||
|
||||
|
||||
@@ -1060,7 +1205,9 @@ def test_handle_room_started_ignores_already_closed_deleted_room(
|
||||
|
||||
service._handle_room_started(mock_data)
|
||||
|
||||
mock_delete_room.assert_called_once_with(str(room.id))
|
||||
mock_delete_room.assert_called_once_with(
|
||||
str(room.id), cluster=get_default_livekit_cluster()
|
||||
)
|
||||
|
||||
|
||||
@mock.patch(
|
||||
@@ -1254,6 +1401,173 @@ def test_receive_ignores_connection_test_room(
|
||||
mock_handle_room_finished.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"placed",
|
||||
(
|
||||
pytest.param(None, id="without-placement"),
|
||||
pytest.param({"cluster": "lyon", "sid": ""}, id="without-sid"),
|
||||
pytest.param({"cluster": "lyon", "sid": OTHER_SID}, id="with-another-sid"),
|
||||
pytest.param({"cluster": "paris", "sid": ""}, id="on-another-cluster"),
|
||||
),
|
||||
)
|
||||
def test_receive_room_started_leaves_placements_alone(
|
||||
placed, two_livekit_clusters, settings, audit_events, caplog
|
||||
):
|
||||
"""A LiveKit room that started changes no placement, whether it has one or
|
||||
not, and the event is handled as before.
|
||||
|
||||
The SID of a placement is recorded as the LiveKit room is created for a
|
||||
join, and a LiveKit room live without placement is adopted by the
|
||||
reconciliation.
|
||||
"""
|
||||
settings.ROOM_TELEPHONY_ENABLED = False
|
||||
settings.ROOMKIT_ENABLED = False
|
||||
_paris, lyon = two_livekit_clusters
|
||||
room = RoomFactory()
|
||||
placement = placed and Placement.objects.create(
|
||||
room_name=str(room.id), room=room, **placed
|
||||
)
|
||||
|
||||
with caplog.at_level(logging.INFO):
|
||||
LiveKitEventsService().receive(
|
||||
room_event_request(lyon, "room_started", str(room.id))
|
||||
)
|
||||
|
||||
if placement is None:
|
||||
assert not Placement.objects.exists()
|
||||
else:
|
||||
kept_placement = Placement.objects.get()
|
||||
assert kept_placement.cluster == placed["cluster"]
|
||||
assert kept_placement.sid == placed["sid"]
|
||||
assert kept_placement.updated_at == placement.updated_at
|
||||
assert not find_events(audit_events, "livekit.placement.adopt")
|
||||
assert str(room.id) not in caplog.text
|
||||
room.refresh_from_db()
|
||||
assert room.last_started_at is not None
|
||||
|
||||
|
||||
@mock.patch("core.services.room_management.RoomManagement.delete_room")
|
||||
def test_receive_room_started_closes_a_deleted_room_on_its_cluster(
|
||||
mock_delete_room, two_livekit_clusters
|
||||
):
|
||||
"""A LiveKit room started for a deleted room is closed on the cluster that
|
||||
sent the webhook, and not adopted."""
|
||||
_paris, lyon = two_livekit_clusters
|
||||
room = RoomFactory()
|
||||
room.soft_delete()
|
||||
|
||||
LiveKitEventsService().receive(
|
||||
room_event_request(lyon, "room_started", str(room.id))
|
||||
)
|
||||
|
||||
mock_delete_room.assert_called_once_with(str(room.id), cluster=lyon)
|
||||
assert not Placement.objects.exists()
|
||||
|
||||
|
||||
def test_receive_room_finished_deletes_the_placement(
|
||||
two_livekit_clusters, room_cleanup
|
||||
):
|
||||
"""A LiveKit room that finished on the cluster and with the SID of its
|
||||
placement deletes it, and its room is cleaned up."""
|
||||
_paris, lyon = two_livekit_clusters
|
||||
room_id = uuid.uuid4()
|
||||
Placement.objects.create(room_name=str(room_id), cluster="lyon", sid=SID)
|
||||
Placement.objects.create(room_name="other-room", cluster="lyon", sid=SID)
|
||||
|
||||
LiveKitEventsService().receive(
|
||||
room_event_request(lyon, "room_finished", str(room_id))
|
||||
)
|
||||
|
||||
assert list(Placement.objects.values_list("room_name", flat=True)) == ["other-room"]
|
||||
for cleanup in room_cleanup:
|
||||
cleanup.assert_called_once_with(room_id)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("cluster", "sid"),
|
||||
(("paris", SID), ("lyon", OTHER_SID), ("lyon", "")),
|
||||
)
|
||||
def test_receive_room_finished_elsewhere_than_placed(
|
||||
cluster, sid, two_livekit_clusters, room_cleanup, caplog
|
||||
):
|
||||
"""A LiveKit room that finished on another cluster or with another SID than
|
||||
its placement's leaves the placement and the room as they are.
|
||||
|
||||
The placement points at the LiveKit room the room now uses, where
|
||||
participants admitted from the lobby must not be queued again.
|
||||
"""
|
||||
_paris, lyon = two_livekit_clusters
|
||||
room_id = uuid.uuid4()
|
||||
placement = Placement.objects.create(
|
||||
room_name=str(room_id), cluster=cluster, sid=sid
|
||||
)
|
||||
|
||||
with caplog.at_level(logging.INFO, logger="core.services.livekit_events"):
|
||||
LiveKitEventsService().receive(
|
||||
room_event_request(lyon, "room_finished", str(room_id))
|
||||
)
|
||||
|
||||
kept_placement = Placement.objects.get()
|
||||
assert kept_placement.pk == placement.pk
|
||||
assert kept_placement.cluster == cluster
|
||||
assert kept_placement.sid == sid
|
||||
for cleanup in room_cleanup:
|
||||
cleanup.assert_not_called()
|
||||
assert (
|
||||
f"LiveKit room {room_id} finished on LiveKit cluster lyon with SID {SID}, "
|
||||
f"but is placed on LiveKit cluster {cluster} with SID {sid!r}: ignoring "
|
||||
"the event"
|
||||
) in caplog.text
|
||||
|
||||
|
||||
def test_receive_room_finished_without_placement(two_livekit_clusters, room_cleanup):
|
||||
"""A LiveKit room that finished without placement has its room cleaned up,
|
||||
as before placements existed."""
|
||||
_paris, lyon = two_livekit_clusters
|
||||
room_id = uuid.uuid4()
|
||||
Placement.objects.create(room_name="other-room", cluster="lyon", sid=SID)
|
||||
|
||||
LiveKitEventsService().receive(
|
||||
room_event_request(lyon, "room_finished", str(room_id))
|
||||
)
|
||||
|
||||
assert Placement.objects.filter(room_name="other-room").exists()
|
||||
for cleanup in room_cleanup:
|
||||
cleanup.assert_called_once_with(room_id)
|
||||
|
||||
|
||||
def test_receive_room_finished_without_sid(two_livekit_clusters, room_cleanup):
|
||||
"""A room event without SID leaves placements as they are, and the room is
|
||||
cleaned up as before placements existed."""
|
||||
_paris, lyon = two_livekit_clusters
|
||||
room_id = uuid.uuid4()
|
||||
Placement.objects.create(room_name=str(room_id), cluster="lyon", sid=SID)
|
||||
|
||||
LiveKitEventsService().receive(
|
||||
room_event_request(lyon, "room_finished", str(room_id), sid="")
|
||||
)
|
||||
|
||||
assert Placement.objects.filter(room_name=str(room_id)).exists()
|
||||
for cleanup in room_cleanup:
|
||||
cleanup.assert_called_once_with(room_id)
|
||||
|
||||
|
||||
def test_receive_room_finished_filtered_out_deletes_the_placement(
|
||||
two_livekit_clusters, room_cleanup, settings
|
||||
):
|
||||
"""The placement of a filtered out room is deleted too, although the room
|
||||
is not cleaned up."""
|
||||
settings.LIVEKIT_WEBHOOK_EVENTS_FILTER_REGEX = UUID_FILTER_REGEX
|
||||
_paris, lyon = two_livekit_clusters
|
||||
Placement.objects.create(room_name="my-room", cluster="lyon", sid=SID)
|
||||
|
||||
LiveKitEventsService().receive(room_event_request(lyon, "room_finished", "my-room"))
|
||||
|
||||
assert not Placement.objects.exists()
|
||||
for cleanup in room_cleanup:
|
||||
cleanup.assert_not_called()
|
||||
|
||||
|
||||
@mock.patch("core.services.presence.cache.delete")
|
||||
def test_participant_left_clearing_gated_by_setting(mock_delete, service, settings):
|
||||
"""PRESENCE_CLEAR_ON_PARTICIPANT_LEFT toggles eager presence invalidation."""
|
||||
|
||||
@@ -17,6 +17,7 @@ from freezegun import freeze_time
|
||||
|
||||
from core import utils
|
||||
from core.factories import RoomFactory, UserFactory, UserResourceAccessFactory
|
||||
from core.livekit.clusters import get_default_livekit_cluster
|
||||
from core.models import RoleChoices, RoomAccessLevel
|
||||
from core.services.lobby import (
|
||||
LobbyParticipant,
|
||||
@@ -254,6 +255,7 @@ def test_request_entry_public_room(
|
||||
configuration=room.configuration,
|
||||
participant_id=participant_id,
|
||||
role=None,
|
||||
cluster=get_default_livekit_cluster(),
|
||||
)
|
||||
|
||||
|
||||
@@ -295,6 +297,7 @@ def test_request_entry_trusted_room(
|
||||
configuration=room.configuration,
|
||||
participant_id=participant_id,
|
||||
role=None,
|
||||
cluster=get_default_livekit_cluster(),
|
||||
)
|
||||
|
||||
|
||||
@@ -401,6 +404,7 @@ def test_request_entry_accepted_participant(
|
||||
configuration=room.configuration,
|
||||
participant_id="test-participant-id",
|
||||
role=None,
|
||||
cluster=get_default_livekit_cluster(),
|
||||
)
|
||||
|
||||
|
||||
@@ -481,6 +485,7 @@ def test_request_entry_participant_with_role(
|
||||
configuration=room.configuration,
|
||||
participant_id="test-participant-id",
|
||||
role="administrator",
|
||||
cluster=get_default_livekit_cluster(),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Tests for the ParticipantsManagement service."""
|
||||
|
||||
from unittest import mock
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from core.models import Placement
|
||||
from core.services.participants_management import ParticipantsManagement
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
|
||||
|
||||
PARTICIPANT_CALLS = [
|
||||
pytest.param(
|
||||
lambda room_name, **kwargs: ParticipantsManagement().mute(
|
||||
room_name, "participant", "track-sid", **kwargs
|
||||
),
|
||||
id="mute",
|
||||
),
|
||||
pytest.param(
|
||||
lambda room_name, **kwargs: ParticipantsManagement().remove(
|
||||
room_name, "participant", **kwargs
|
||||
),
|
||||
id="remove",
|
||||
),
|
||||
pytest.param(
|
||||
lambda room_name, **kwargs: ParticipantsManagement().update(
|
||||
room_name, "participant", attributes={"room_role": "member"}, **kwargs
|
||||
),
|
||||
id="update",
|
||||
),
|
||||
pytest.param(
|
||||
lambda room_name, **kwargs: ParticipantsManagement().check_if_in_meeting(
|
||||
room_name, "participant", **kwargs
|
||||
),
|
||||
id="check_if_in_meeting",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"manage_participant",
|
||||
[
|
||||
*PARTICIPANT_CALLS,
|
||||
pytest.param(
|
||||
lambda room_name: ParticipantsManagement().check_if_in_meeting_cached(
|
||||
room_name, "participant"
|
||||
),
|
||||
id="check_if_in_meeting_cached",
|
||||
),
|
||||
],
|
||||
)
|
||||
@mock.patch("core.utils.create_livekit_client")
|
||||
def test_participants_managed_on_the_placed_cluster(
|
||||
mock_create_livekit_client,
|
||||
manage_participant,
|
||||
two_livekit_clusters,
|
||||
django_assert_num_queries,
|
||||
):
|
||||
"""Participants are managed on the LiveKit cluster their room is placed on."""
|
||||
_paris, lyon = two_livekit_clusters
|
||||
room_name = str(uuid4())
|
||||
Placement.objects.create(room_name=room_name, cluster="lyon")
|
||||
mock_create_livekit_client.return_value = mock.AsyncMock()
|
||||
|
||||
with django_assert_num_queries(1):
|
||||
manage_participant(room_name)
|
||||
|
||||
mock_create_livekit_client.assert_called_once_with(cluster=lyon)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("manage_participant", PARTICIPANT_CALLS)
|
||||
@mock.patch("core.utils.create_livekit_client")
|
||||
def test_participants_managed_on_the_given_cluster(
|
||||
mock_create_livekit_client,
|
||||
manage_participant,
|
||||
two_livekit_clusters,
|
||||
django_assert_num_queries,
|
||||
):
|
||||
"""Participants are managed on the cluster given, whatever the placement."""
|
||||
paris, _lyon = two_livekit_clusters
|
||||
room_name = str(uuid4())
|
||||
Placement.objects.create(room_name=room_name, cluster="lyon")
|
||||
mock_create_livekit_client.return_value = mock.AsyncMock()
|
||||
|
||||
with django_assert_num_queries(0):
|
||||
manage_participant(room_name, cluster=paris)
|
||||
|
||||
mock_create_livekit_client.assert_called_once_with(cluster=paris)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("two_livekit_clusters")
|
||||
@mock.patch("core.utils.create_livekit_client")
|
||||
def test_check_if_in_meeting_without_room_or_identity(
|
||||
mock_create_livekit_client, django_assert_num_queries
|
||||
):
|
||||
"""Without a room or an identity, nobody is looked for, nor any placement."""
|
||||
with django_assert_num_queries(0):
|
||||
assert ParticipantsManagement().check_if_in_meeting("", "participant") is False
|
||||
assert ParticipantsManagement().check_if_in_meeting("room", "") is False
|
||||
|
||||
mock_create_livekit_client.assert_not_called()
|
||||
@@ -8,7 +8,7 @@ import pytest
|
||||
from livekit.api import TwirpError
|
||||
|
||||
from core.factories import RoomFactory
|
||||
from core.models import Room, RoomAccessLevel
|
||||
from core.models import Placement, Room, RoomAccessLevel
|
||||
from core.services.room_management import (
|
||||
RoomManagement,
|
||||
RoomManagementException,
|
||||
@@ -297,3 +297,61 @@ def test_sync_room_metadata_pushes_configuration_and_access_level(mock_update_me
|
||||
"access_level": RoomAccessLevel.RESTRICTED,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def live_room_client():
|
||||
"""Return a LiveKit client mock on which the room is live, without metadata."""
|
||||
client = mock.AsyncMock()
|
||||
client.room.list_rooms.return_value = mock.Mock(rooms=[mock.Mock(metadata="")])
|
||||
return client
|
||||
|
||||
|
||||
ROOM_CALLS = [
|
||||
pytest.param(
|
||||
lambda room_name, **kwargs: RoomManagement.update_metadata(
|
||||
room_name, {"key": "value"}, **kwargs
|
||||
),
|
||||
id="update_metadata",
|
||||
),
|
||||
pytest.param(RoomManagement.delete_room, id="delete_room"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@pytest.mark.parametrize("manage_room", ROOM_CALLS)
|
||||
@mock.patch("core.services.room_management.utils.create_livekit_client")
|
||||
def test_room_managed_on_the_placed_cluster(
|
||||
mock_create_livekit_client,
|
||||
manage_room,
|
||||
two_livekit_clusters,
|
||||
django_assert_num_queries,
|
||||
):
|
||||
"""A LiveKit room is managed on the LiveKit cluster it is placed on."""
|
||||
_paris, lyon = two_livekit_clusters
|
||||
Placement.objects.create(room_name="room-abc", cluster="lyon")
|
||||
mock_create_livekit_client.return_value = live_room_client()
|
||||
|
||||
with django_assert_num_queries(1):
|
||||
manage_room("room-abc")
|
||||
|
||||
mock_create_livekit_client.assert_called_once_with(cluster=lyon)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@pytest.mark.parametrize("manage_room", ROOM_CALLS)
|
||||
@mock.patch("core.services.room_management.utils.create_livekit_client")
|
||||
def test_room_managed_on_the_given_cluster(
|
||||
mock_create_livekit_client,
|
||||
manage_room,
|
||||
two_livekit_clusters,
|
||||
django_assert_num_queries,
|
||||
):
|
||||
"""A LiveKit room is managed on the cluster given, whatever its placement."""
|
||||
paris, _lyon = two_livekit_clusters
|
||||
Placement.objects.create(room_name="room-abc", cluster="lyon")
|
||||
mock_create_livekit_client.return_value = live_room_client()
|
||||
|
||||
with django_assert_num_queries(0):
|
||||
manage_room("room-abc", cluster=paris)
|
||||
|
||||
mock_create_livekit_client.assert_called_once_with(cluster=paris)
|
||||
|
||||
@@ -8,6 +8,7 @@ from unittest import mock
|
||||
import pytest
|
||||
|
||||
from core.factories import RoomFactory
|
||||
from core.models import Placement
|
||||
from core.services.subtitle import SubtitleService
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
@@ -37,6 +38,38 @@ def test_start_subtitle_settings(mock_livekit_client, settings):
|
||||
assert call_args.room == str(room.id)
|
||||
|
||||
|
||||
@mock.patch("core.utils.create_livekit_client")
|
||||
def test_start_subtitle_on_the_placed_cluster(
|
||||
mock_create_livekit_client, two_livekit_clusters, django_assert_num_queries
|
||||
):
|
||||
"""The agent is dispatched on the LiveKit cluster the room is placed on."""
|
||||
_paris, lyon = two_livekit_clusters
|
||||
room = RoomFactory()
|
||||
Placement.objects.create(room_name=str(room.id), room=room, cluster="lyon")
|
||||
mock_create_livekit_client.return_value = mock.AsyncMock()
|
||||
|
||||
with django_assert_num_queries(1):
|
||||
SubtitleService().start_subtitle(room)
|
||||
|
||||
mock_create_livekit_client.assert_called_once_with(cluster=lyon)
|
||||
|
||||
|
||||
@mock.patch("core.utils.create_livekit_client")
|
||||
def test_start_subtitle_on_the_given_cluster(
|
||||
mock_create_livekit_client, two_livekit_clusters, django_assert_num_queries
|
||||
):
|
||||
"""The agent is dispatched on the cluster given, whatever the placement."""
|
||||
paris, _lyon = two_livekit_clusters
|
||||
room = RoomFactory()
|
||||
Placement.objects.create(room_name=str(room.id), room=room, cluster="lyon")
|
||||
mock_create_livekit_client.return_value = mock.AsyncMock()
|
||||
|
||||
with django_assert_num_queries(0):
|
||||
SubtitleService().start_subtitle(room, cluster=paris)
|
||||
|
||||
mock_create_livekit_client.assert_called_once_with(cluster=paris)
|
||||
|
||||
|
||||
def test_stop_subtitle_not_implemented():
|
||||
"""Test that stop_subtitle raises NotImplementedError."""
|
||||
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
"""Tests for connection test Celery tasks."""
|
||||
|
||||
import logging
|
||||
from unittest import mock
|
||||
|
||||
from django.test.utils import override_settings
|
||||
import pytest
|
||||
|
||||
from core.livekit.clusters import get_default_livekit_cluster
|
||||
from core.services.room_management import (
|
||||
RoomManagementException,
|
||||
RoomNotFoundException,
|
||||
@@ -13,11 +15,50 @@ from core.tasks.connection_test import delete_connection_test_room
|
||||
|
||||
@mock.patch("core.tasks.connection_test.RoomManagement.delete_room")
|
||||
def test_delete_connection_test_room_calls_room_management(mock_delete_room, settings):
|
||||
"""RoomManagement.delete_room is called for connection-test rooms."""
|
||||
"""RoomManagement.delete_room is called for connection-test rooms, on the
|
||||
default LiveKit cluster when the task names none, as tasks queued before
|
||||
connection test rooms were created on a chosen cluster do."""
|
||||
settings.CONNECTION_TEST_ROOM_PREFIX = "connection-test"
|
||||
delete_connection_test_room("connection-test-abc")
|
||||
|
||||
mock_delete_room.assert_called_once_with("connection-test-abc")
|
||||
mock_delete_room.assert_called_once_with(
|
||||
"connection-test-abc", cluster=get_default_livekit_cluster()
|
||||
)
|
||||
|
||||
|
||||
@mock.patch("core.tasks.connection_test.RoomManagement.delete_room")
|
||||
def test_delete_connection_test_room_on_its_cluster(
|
||||
mock_delete_room, settings, two_livekit_clusters
|
||||
):
|
||||
"""A connection-test room is deleted on the LiveKit cluster it was created on."""
|
||||
settings.CONNECTION_TEST_ROOM_PREFIX = "connection-test"
|
||||
_paris, lyon = two_livekit_clusters
|
||||
|
||||
delete_connection_test_room("connection-test-abc", "lyon")
|
||||
|
||||
mock_delete_room.assert_called_once_with("connection-test-abc", cluster=lyon)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("two_livekit_clusters")
|
||||
@mock.patch("core.tasks.connection_test.RoomManagement.delete_room")
|
||||
def test_delete_connection_test_room_on_unregistered_cluster(
|
||||
mock_delete_room, settings, caplog
|
||||
):
|
||||
"""A connection-test room created on a LiveKit cluster since removed from
|
||||
the registry is deleted on the default cluster, with a warning."""
|
||||
settings.CONNECTION_TEST_ROOM_PREFIX = "connection-test"
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="core.tasks.connection_test"):
|
||||
delete_connection_test_room("connection-test-abc", "lille")
|
||||
|
||||
mock_delete_room.assert_called_once_with(
|
||||
"connection-test-abc", cluster=get_default_livekit_cluster()
|
||||
)
|
||||
assert get_default_livekit_cluster().name == "paris"
|
||||
assert (
|
||||
"Connection test room 'connection-test-abc' was created on LiveKit cluster "
|
||||
"lille, which is not registered: deleting it on the default cluster paris."
|
||||
) in caplog.text
|
||||
|
||||
|
||||
@mock.patch("core.tasks.connection_test.RoomManagement.delete_room")
|
||||
@@ -35,9 +76,11 @@ def test_delete_connection_test_room_ignores_missing_room(mock_delete_room, sett
|
||||
settings.CONNECTION_TEST_ROOM_PREFIX = "connection-test"
|
||||
mock_delete_room.side_effect = RoomNotFoundException("Room does not exist")
|
||||
|
||||
delete_connection_test_room("connection-test-gone")
|
||||
delete_connection_test_room("connection-test-gone", "default")
|
||||
|
||||
mock_delete_room.assert_called_once_with("connection-test-gone")
|
||||
mock_delete_room.assert_called_once_with(
|
||||
"connection-test-gone", cluster=get_default_livekit_cluster()
|
||||
)
|
||||
|
||||
|
||||
@mock.patch("core.tasks.connection_test.RoomManagement.delete_room")
|
||||
@@ -46,6 +89,8 @@ def test_delete_connection_test_room_logs_other_failures(mock_delete_room, setti
|
||||
settings.CONNECTION_TEST_ROOM_PREFIX = "connection-test"
|
||||
mock_delete_room.side_effect = RoomManagementException("Could not delete room")
|
||||
|
||||
delete_connection_test_room("connection-test-fail")
|
||||
delete_connection_test_room("connection-test-fail", "default")
|
||||
|
||||
mock_delete_room.assert_called_once_with("connection-test-fail")
|
||||
mock_delete_room.assert_called_once_with(
|
||||
"connection-test-fail", cluster=get_default_livekit_cluster()
|
||||
)
|
||||
|
||||
@@ -1,11 +1,38 @@
|
||||
"""Test the LiveKit reconnection settings exposed by the config endpoint."""
|
||||
"""Test the LiveKit settings exposed by the config endpoint."""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from rest_framework.test import APIClient
|
||||
|
||||
from core.tests.utils import livekit_cluster_entry
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
|
||||
|
||||
def test_api_config_retrieve_livekit_url(settings):
|
||||
"""Without LIVEKIT_CLUSTERS, browsers connect to LIVEKIT_CONFIGURATION's URL."""
|
||||
response = APIClient().get("/api/v1.0/config/")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["livekit"]["url"] == settings.LIVEKIT_CONFIGURATION["url"]
|
||||
|
||||
|
||||
def test_api_config_retrieve_livekit_url_default_cluster(settings):
|
||||
"""Browsers are given the URL of the default LiveKit cluster, not its API URL."""
|
||||
settings.LIVEKIT_CLUSTERS = json.dumps(
|
||||
[
|
||||
livekit_cluster_entry("paris", api_url="http://livekit.paris.svc:7880"),
|
||||
livekit_cluster_entry("lyon"),
|
||||
]
|
||||
)
|
||||
|
||||
response = APIClient().get("/api/v1.0/config/")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["livekit"]["url"] == "wss://paris.livekit.test"
|
||||
|
||||
|
||||
def test_api_config_retrieve_livekit_reconnect_defaults():
|
||||
"""Without any override, the frontend gets the default backoff."""
|
||||
response = APIClient().get("/api/v1.0/config/")
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Test diagnostics API endpoints."""
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from unittest import mock
|
||||
|
||||
@@ -15,6 +16,10 @@ from core.api.throttling import (
|
||||
ConnectionTestUserRateThrottle,
|
||||
)
|
||||
from core.factories import UserFactory
|
||||
from core.livekit.clusters import get_livekit_cluster
|
||||
from core.livekit.exceptions import LiveKitRoomCreationError
|
||||
from core.models import Placement
|
||||
from core.tests.utils import livekit_cluster_entry, random_remote_addr
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
|
||||
@@ -111,11 +116,78 @@ def test_api_diagnostics_connection_token_for_authenticated_user(settings, clien
|
||||
assert payload["exp"] - payload["nbf"] == 300
|
||||
|
||||
|
||||
def test_api_diagnostics_connection_chosen_cluster(
|
||||
settings, fake_livekit_room_creation
|
||||
):
|
||||
"""The connection is tested on the LiveKit cluster the placement strategy
|
||||
picks, where the room is created, without any placement.
|
||||
|
||||
Browsers are given its URL, not the API URL Django calls it on.
|
||||
"""
|
||||
lyon = livekit_cluster_entry("lyon", api_url="http://livekit.lyon.svc:7880")
|
||||
settings.LIVEKIT_CLUSTERS = json.dumps([livekit_cluster_entry("paris"), lyon])
|
||||
|
||||
with mock.patch("core.livekit.strategies.randbelow", return_value=1):
|
||||
response = APIClient(REMOTE_ADDR=random_remote_addr()).post(
|
||||
"/api/v1.0/diagnostics/connection/"
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
config = response.json()["livekit"]
|
||||
assert config["url"] == lyon["url"]
|
||||
payload = jwt.decode(config["token"], lyon["api_secret"], algorithms=["HS256"])
|
||||
assert payload["iss"] == lyon["api_key"]
|
||||
assert payload["video"]["room"] == config["room"]
|
||||
fake_livekit_room_creation.assert_called_once_with(
|
||||
config["room"], get_livekit_cluster("lyon")
|
||||
)
|
||||
assert not Placement.objects.exists()
|
||||
|
||||
|
||||
@mock.patch("core.api.viewsets.delete_connection_test_room.apply_async")
|
||||
def test_api_diagnostics_connection_room_creation_failure(
|
||||
mock_apply_async, settings, fake_livekit_room_creation
|
||||
):
|
||||
"""A connection cannot be tested for now when its LiveKit cluster fails to
|
||||
create the room: no deletion is scheduled then."""
|
||||
settings.CELERY_ENABLED = True
|
||||
fake_livekit_room_creation.side_effect = LiveKitRoomCreationError("Timed out")
|
||||
|
||||
response = APIClient(REMOTE_ADDR=random_remote_addr()).post(
|
||||
"/api/v1.0/diagnostics/connection/"
|
||||
)
|
||||
|
||||
assert response.status_code == 503
|
||||
assert response.json() == {
|
||||
"detail": "This room cannot be joined for now, please try again."
|
||||
}
|
||||
mock_apply_async.assert_not_called()
|
||||
|
||||
|
||||
def test_api_diagnostics_connection_without_candidate(
|
||||
settings, fake_livekit_room_creation
|
||||
):
|
||||
"""A connection cannot be tested for now when no LiveKit cluster may receive
|
||||
new placements."""
|
||||
settings.LIVEKIT_CLUSTERS = json.dumps([livekit_cluster_entry("paris", weight=0)])
|
||||
|
||||
response = APIClient(REMOTE_ADDR=random_remote_addr()).post(
|
||||
"/api/v1.0/diagnostics/connection/"
|
||||
)
|
||||
|
||||
assert response.status_code == 503
|
||||
assert response.json() == {
|
||||
"detail": "This room cannot be joined for now, please try again."
|
||||
}
|
||||
fake_livekit_room_creation.assert_not_called()
|
||||
|
||||
|
||||
@mock.patch("core.api.viewsets.delete_connection_test_room.apply_async")
|
||||
def test_api_diagnostics_connection_schedules_room_deletion(
|
||||
mock_apply_async, settings, client
|
||||
):
|
||||
"""When Celery is enabled, schedule a hard room delete after max age."""
|
||||
"""When Celery is enabled, schedule a hard room delete after max age, on the
|
||||
LiveKit cluster the room was created on."""
|
||||
|
||||
settings.CELERY_ENABLED = True
|
||||
settings.CONNECTION_TEST_TOKEN_TTL_SECONDS = 300
|
||||
@@ -126,7 +198,7 @@ def test_api_diagnostics_connection_schedules_room_deletion(
|
||||
|
||||
assert response.status_code == 200
|
||||
room = response.json()["livekit"]["room"]
|
||||
mock_apply_async.assert_called_once_with(args=[room], countdown=310)
|
||||
mock_apply_async.assert_called_once_with(args=[room, "default"], countdown=310)
|
||||
|
||||
|
||||
@mock.patch("core.api.viewsets.delete_connection_test_room.apply_async")
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
"""Tests of what the exception reporter shows of the LiveKit cluster settings."""
|
||||
|
||||
import json
|
||||
import sys
|
||||
|
||||
from django.test import RequestFactory
|
||||
from django.views.debug import (
|
||||
get_default_exception_reporter_filter,
|
||||
technical_500_response,
|
||||
)
|
||||
|
||||
from core.tests.utils import livekit_cluster_entry
|
||||
|
||||
SECRET = "distinctive-cluster-api-secret"
|
||||
|
||||
|
||||
def clusters_with_secret():
|
||||
"""A LIVEKIT_CLUSTERS value whose API secret is easy to search for."""
|
||||
entry = livekit_cluster_entry("paris")
|
||||
entry["api_secret"] = SECRET
|
||||
return json.dumps([entry])
|
||||
|
||||
|
||||
def raised_exc_info():
|
||||
"""The exception info of a raised error."""
|
||||
try:
|
||||
raise RuntimeError("boom")
|
||||
except RuntimeError:
|
||||
return sys.exc_info()
|
||||
|
||||
|
||||
def test_exception_reporter_hides_livekit_clusters_in_safe_settings(settings):
|
||||
"""The safe settings show LIVEKIT_CLUSTERS as stars, not the secrets it holds."""
|
||||
settings.LIVEKIT_CLUSTERS = clusters_with_secret()
|
||||
|
||||
safe = get_default_exception_reporter_filter().get_safe_settings()
|
||||
|
||||
assert SECRET not in json.dumps(safe, default=str)
|
||||
assert safe["LIVEKIT_CLUSTERS"] == "********************"
|
||||
|
||||
|
||||
def test_exception_reporter_hides_livekit_clusters_on_technical_500_page(settings):
|
||||
"""The technical 500 page of a deployment with DEBUG on lists no cluster secret."""
|
||||
settings.DEBUG = True
|
||||
settings.LIVEKIT_CLUSTERS = clusters_with_secret()
|
||||
request = RequestFactory().get("/")
|
||||
exc_info = raised_exc_info()
|
||||
|
||||
response = technical_500_response(request, *exc_info)
|
||||
|
||||
content = response.content.decode()
|
||||
assert response.status_code == 500
|
||||
assert "LIVEKIT_CLUSTERS" in content
|
||||
assert SECRET not in content
|
||||
@@ -0,0 +1,80 @@
|
||||
"""
|
||||
Unit tests for the Placement model
|
||||
"""
|
||||
|
||||
from django.core.exceptions import ValidationError
|
||||
from django.db import connection
|
||||
|
||||
import pytest
|
||||
|
||||
from core.factories import RoomFactory
|
||||
from core.models import Placement
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
|
||||
|
||||
def test_models_placements_str():
|
||||
"""The str representation should name the LiveKit room and its cluster."""
|
||||
placement = Placement.objects.create(room_name="my-room", cluster="paris")
|
||||
assert str(placement) == "my-room on paris"
|
||||
|
||||
|
||||
def test_models_placements_defaults():
|
||||
"""A new placement has no room, no SID, no relocation, no absence and epoch 0."""
|
||||
placement = Placement.objects.create(room_name="my-room", cluster="paris")
|
||||
|
||||
placement.refresh_from_db()
|
||||
assert placement.room is None
|
||||
assert placement.sid == ""
|
||||
assert placement.epoch == 0
|
||||
assert placement.previous_cluster == ""
|
||||
assert placement.relocated_at is None
|
||||
assert placement.absent_since is None
|
||||
|
||||
|
||||
def test_models_placements_room_name_unique():
|
||||
"""A LiveKit room has a single placement, also when writes skip validation."""
|
||||
Placement.objects.create(room_name="my-room", cluster="paris")
|
||||
|
||||
with pytest.raises(ValidationError) as excinfo:
|
||||
Placement.objects.create(room_name="my-room", cluster="lyon")
|
||||
|
||||
assert "Placement with this LiveKit room name already exists." in str(excinfo.value)
|
||||
|
||||
Placement.objects.bulk_create(
|
||||
[Placement(room_name="my-room", cluster="lyon")], ignore_conflicts=True
|
||||
)
|
||||
|
||||
assert Placement.objects.get().cluster == "paris"
|
||||
|
||||
|
||||
def test_models_placements_room_deleted():
|
||||
"""A placement outlives its room's record, and then points to no room."""
|
||||
room = RoomFactory()
|
||||
room_name = str(room.pk)
|
||||
placement = Placement.objects.create(
|
||||
room_name=room_name, room=room, cluster="paris"
|
||||
)
|
||||
|
||||
room.delete()
|
||||
|
||||
placement.refresh_from_db()
|
||||
assert placement.room is None
|
||||
assert placement.room_name == room_name
|
||||
assert placement.cluster == "paris"
|
||||
|
||||
|
||||
def test_models_placements_room_deleted_without_the_collector():
|
||||
"""The database unsets the room of a placement when a deletion bypasses Django."""
|
||||
room = RoomFactory()
|
||||
placement = Placement.objects.create(
|
||||
room_name=str(room.pk), room=room, cluster="paris"
|
||||
)
|
||||
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute("DELETE FROM meet_room WHERE resource_id = %s", [room.pk])
|
||||
cursor.execute("DELETE FROM meet_resource WHERE id = %s", [room.pk])
|
||||
|
||||
placement.refresh_from_db()
|
||||
assert placement.room is None
|
||||
assert placement.cluster == "paris"
|
||||
@@ -2,7 +2,11 @@
|
||||
Unit tests for the Recording model
|
||||
"""
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
|
||||
from django.core.exceptions import ValidationError
|
||||
from django.db import connection
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -290,3 +294,69 @@ def test_models_recording_is_saved_false_error_states(status):
|
||||
"""Test is_saved property returns False for error statuses."""
|
||||
recording = RecordingFactory(status=status)
|
||||
assert recording.is_saved is False
|
||||
|
||||
|
||||
def test_models_recording_livekit_cluster_default():
|
||||
"""A recording names no LiveKit cluster by default."""
|
||||
recording = RecordingFactory()
|
||||
assert recording.livekit_cluster == ""
|
||||
|
||||
|
||||
def test_models_recording_livekit_cluster_unnamed_by_previous_release():
|
||||
"""A rollback can still start recordings: the previous release inserts them
|
||||
without naming a LiveKit cluster, which leaves them on the default cluster."""
|
||||
room = RoomFactory()
|
||||
recording_id = uuid.uuid4()
|
||||
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"INSERT INTO meet_recording (id, created_at, updated_at, room_id, "
|
||||
"status, worker_id, mode, options, external_process_id) VALUES "
|
||||
"(%s, NOW(), NOW(), %s, 'initiated', NULL, 'screen_recording', '{}', NULL)",
|
||||
[recording_id, room.id],
|
||||
)
|
||||
|
||||
assert Recording.objects.get(id=recording_id).livekit_cluster == ""
|
||||
|
||||
|
||||
def test_models_recording_get_livekit_cluster(two_livekit_clusters):
|
||||
"""A recording is on the registered LiveKit cluster it names."""
|
||||
_paris, lyon = two_livekit_clusters
|
||||
recording = RecordingFactory(livekit_cluster="lyon")
|
||||
|
||||
assert recording.get_livekit_cluster() == lyon
|
||||
|
||||
|
||||
def test_models_recording_get_livekit_cluster_unnamed(two_livekit_clusters, caplog):
|
||||
"""A recording naming no LiveKit cluster, as one started before recordings
|
||||
kept theirs, is on the default cluster."""
|
||||
paris, _lyon = two_livekit_clusters
|
||||
recording = RecordingFactory(livekit_cluster="")
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="core.models"):
|
||||
assert recording.get_livekit_cluster() == paris
|
||||
|
||||
assert "not registered" not in caplog.text
|
||||
|
||||
|
||||
def test_models_recording_get_livekit_cluster_unregistered(
|
||||
two_livekit_clusters, caplog
|
||||
):
|
||||
"""A recording on a LiveKit cluster no longer registered is on the default
|
||||
cluster, which is logged."""
|
||||
paris, _lyon = two_livekit_clusters
|
||||
recording = RecordingFactory(livekit_cluster="lille")
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="core.models"):
|
||||
assert recording.get_livekit_cluster() == paris
|
||||
|
||||
assert (
|
||||
f"Recording {recording.id} is on LiveKit cluster lille, which is not "
|
||||
"registered: managing it on the default cluster paris"
|
||||
) in caplog.text
|
||||
|
||||
|
||||
def test_models_recording_livekit_cluster_too_long():
|
||||
"""A LiveKit cluster name is at most 63 characters long."""
|
||||
with pytest.raises(ValidationError, match="livekit_cluster"):
|
||||
RecordingFactory(livekit_cluster="a" * 64)
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Unit tests for the validation of the LIVEKIT_CLUSTERS setting."""
|
||||
|
||||
import json
|
||||
import re
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
from core.tests.utils import livekit_cluster_entry
|
||||
|
||||
from meet import settings
|
||||
|
||||
|
||||
def test_livekit_clusters_invalid_is_refused():
|
||||
"""An invalid list of clusters stops the settings loading, naming the problem."""
|
||||
clusters = json.dumps([livekit_cluster_entry("paris", weight=-1)])
|
||||
|
||||
with mock.patch.object(settings.Test, "LIVEKIT_CLUSTERS", clusters):
|
||||
with pytest.raises(ValueError, match=re.escape("LIVEKIT_CLUSTERS[0].weight")):
|
||||
settings.Test.post_setup()
|
||||
|
||||
|
||||
def test_livekit_clusters_valid_is_accepted():
|
||||
"""A valid list of clusters is accepted while telephony and roomkit are off."""
|
||||
clusters = json.dumps(
|
||||
[livekit_cluster_entry("paris"), livekit_cluster_entry("lyon")]
|
||||
)
|
||||
|
||||
with (
|
||||
mock.patch.object(settings.Test, "LIVEKIT_CLUSTERS", clusters),
|
||||
mock.patch.object(settings.Test, "ROOM_TELEPHONY_ENABLED", False),
|
||||
mock.patch.object(settings.Test, "ROOMKIT_ENABLED", False),
|
||||
):
|
||||
settings.Test.post_setup()
|
||||
|
||||
|
||||
def test_livekit_clusters_unset_skips_validation():
|
||||
"""Unset, as in the Build configuration which has no LiveKit environment.
|
||||
|
||||
LIVEKIT_CONFIGURATION, which the registry then falls back to, is not checked.
|
||||
"""
|
||||
with (
|
||||
mock.patch.object(settings.Test, "LIVEKIT_CLUSTERS", None),
|
||||
mock.patch.object(
|
||||
settings.Test,
|
||||
"LIVEKIT_CONFIGURATION",
|
||||
{"url": None, "api_key": None, "api_secret": None},
|
||||
),
|
||||
):
|
||||
settings.Test.post_setup()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"telephony,roomkit,enabled",
|
||||
[
|
||||
(True, False, "ROOM_TELEPHONY_ENABLED"),
|
||||
(False, True, "ROOMKIT_ENABLED"),
|
||||
(True, True, "ROOM_TELEPHONY_ENABLED and ROOMKIT_ENABLED"),
|
||||
],
|
||||
ids=["telephony", "roomkit", "both"],
|
||||
)
|
||||
def test_livekit_clusters_several_with_telephony_are_refused(
|
||||
telephony, roomkit, enabled
|
||||
):
|
||||
"""Several clusters with telephony or roomkit stop the settings loading.
|
||||
|
||||
Phone and roomkit callers reach the SIP server of one cluster before their PIN
|
||||
is known, so Django cannot route them. The message names the enabled settings.
|
||||
"""
|
||||
clusters = json.dumps(
|
||||
[livekit_cluster_entry("paris"), livekit_cluster_entry("lyon")]
|
||||
)
|
||||
|
||||
with (
|
||||
mock.patch.object(settings.Test, "LIVEKIT_CLUSTERS", clusters),
|
||||
mock.patch.object(settings.Test, "ROOM_TELEPHONY_ENABLED", telephony),
|
||||
mock.patch.object(settings.Test, "ROOMKIT_ENABLED", roomkit),
|
||||
):
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match=f"^{enabled} cannot be combined with more than one LiveKit "
|
||||
"cluster, and LIVEKIT_CLUSTERS lists 2:",
|
||||
):
|
||||
settings.Test.post_setup()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"clusters",
|
||||
[None, json.dumps([livekit_cluster_entry("paris")])],
|
||||
ids=["unset", "listed"],
|
||||
)
|
||||
def test_livekit_clusters_single_with_telephony_is_accepted(clusters):
|
||||
"""Telephony and roomkit work with a single cluster.
|
||||
|
||||
It is either listed in LIVEKIT_CLUSTERS or, when that is unset, built from
|
||||
LIVEKIT_CONFIGURATION.
|
||||
"""
|
||||
with (
|
||||
mock.patch.object(settings.Test, "LIVEKIT_CLUSTERS", clusters),
|
||||
mock.patch.object(settings.Test, "ROOM_TELEPHONY_ENABLED", True),
|
||||
mock.patch.object(settings.Test, "ROOMKIT_ENABLED", True),
|
||||
):
|
||||
settings.Test.post_setup()
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Unit tests for the LIVEKIT_API_TIMEOUT_SECONDS setting."""
|
||||
"""Unit tests for the timeouts of LiveKit API calls in the settings."""
|
||||
|
||||
from unittest import mock
|
||||
|
||||
@@ -12,3 +12,13 @@ def test_livekit_api_timeout_zero_is_refused():
|
||||
with mock.patch.object(Test, "LIVEKIT_API_TIMEOUT_SECONDS", 0):
|
||||
with pytest.raises(ValueError, match="LIVEKIT_API_TIMEOUT_SECONDS"):
|
||||
Test.post_setup()
|
||||
|
||||
|
||||
def test_livekit_create_room_timeout_zero_is_refused():
|
||||
"""A zero timeout would fail every LiveKit room creation, and with it every
|
||||
join, so it stops the settings loading."""
|
||||
with mock.patch.object(Test, "LIVEKIT_CREATE_ROOM_TIMEOUT_SECONDS", 0):
|
||||
with pytest.raises(
|
||||
ValueError, match="LIVEKIT_CREATE_ROOM_TIMEOUT_SECONDS must be at least 1"
|
||||
):
|
||||
Test.post_setup()
|
||||
|
||||
@@ -3,9 +3,12 @@ Test utils functions
|
||||
"""
|
||||
|
||||
# pylint: disable=W0621
|
||||
import asyncio
|
||||
import contextlib
|
||||
import json
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
from unittest import mock
|
||||
|
||||
from django.conf import settings
|
||||
@@ -14,18 +17,25 @@ from django.contrib.auth.models import AnonymousUser
|
||||
import jwt
|
||||
import pytest
|
||||
from asgiref.sync import async_to_sync
|
||||
from livekit.api import TwirpError
|
||||
from livekit.api import ( # pylint: disable=no-name-in-module
|
||||
DeleteRoomRequest,
|
||||
TwirpError,
|
||||
)
|
||||
|
||||
from core.factories import UserFactory
|
||||
from core.livekit.clusters import get_livekit_cluster
|
||||
from core.models import Placement
|
||||
from core.services.participants_management import (
|
||||
ParticipantsManagement,
|
||||
ParticipantsManagementException,
|
||||
)
|
||||
from core.services.room_management import RoomManagement, RoomManagementException
|
||||
from core.services.sip_management import SIPException, SIPManagement
|
||||
from core.tests.utils import livekit_cluster_entry
|
||||
from core.utils import (
|
||||
NotificationError,
|
||||
create_livekit_client,
|
||||
generate_livekit_config,
|
||||
generate_token,
|
||||
notify_participants,
|
||||
)
|
||||
@@ -42,6 +52,20 @@ def decode_token(token: str) -> dict:
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def two_clusters(settings):
|
||||
"""Register the Paris and Lyon LiveKit clusters, Paris being the default.
|
||||
|
||||
Django calls each of them on an API URL apart from the one browsers use.
|
||||
"""
|
||||
settings.LIVEKIT_CLUSTERS = json.dumps(
|
||||
[
|
||||
livekit_cluster_entry("paris", api_url="http://livekit.paris.svc:7880"),
|
||||
livekit_cluster_entry("lyon", api_url="http://livekit.lyon.svc:7880"),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_generate_token_authenticated_uses_full_name():
|
||||
"""The token's display name should default to the user's full name."""
|
||||
user = UserFactory(full_name="Jane Doe")
|
||||
@@ -116,6 +140,49 @@ def test_anonymous_falls_back_to_anonymous_label():
|
||||
assert claims["name"] == "Anonymous"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("two_clusters")
|
||||
@pytest.mark.parametrize(
|
||||
("name", "expected"), [(None, "paris"), ("paris", "paris"), ("lyon", "lyon")]
|
||||
)
|
||||
def test_generate_token_cluster(name, expected):
|
||||
"""A token is signed with the API key and secret of its cluster.
|
||||
|
||||
It is for the default cluster when none is given.
|
||||
"""
|
||||
cluster = get_livekit_cluster(name) if name else None
|
||||
expected_cluster = get_livekit_cluster(expected)
|
||||
|
||||
token = generate_token(room="my-room", user=AnonymousUser(), cluster=cluster)
|
||||
|
||||
claims = jwt.decode(token, expected_cluster.api_secret, algorithms=["HS256"])
|
||||
assert claims["iss"] == expected_cluster.api_key
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("two_clusters")
|
||||
@pytest.mark.parametrize("name", ["paris", "lyon"])
|
||||
def test_generate_livekit_config_cluster(name):
|
||||
"""Browsers get the URL of the cluster and a token signed for it."""
|
||||
cluster = get_livekit_cluster(name)
|
||||
|
||||
config = generate_livekit_config(
|
||||
room_id="my-room", user=AnonymousUser(), username="Guest42", cluster=cluster
|
||||
)
|
||||
|
||||
assert config["url"] == cluster.url
|
||||
assert config["room"] == "my-room"
|
||||
claims = jwt.decode(config["token"], cluster.api_secret, algorithms=["HS256"])
|
||||
assert claims["iss"] == cluster.api_key
|
||||
assert claims["video"]["room"] == "my-room"
|
||||
|
||||
|
||||
def test_generate_livekit_config_without_cluster():
|
||||
"""No credentials are built without the cluster the LiveKit room is placed on."""
|
||||
arguments = {"room_id": "my-room", "user": AnonymousUser(), "username": "Guest42"}
|
||||
|
||||
with pytest.raises(TypeError, match="cluster"):
|
||||
generate_livekit_config(**arguments)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_livekit(request):
|
||||
"""A LiveKit address that takes the connection, sends `request.param`, then waits."""
|
||||
@@ -187,6 +254,63 @@ def test_create_livekit_client_lets_an_answer_through(fake_livekit, settings):
|
||||
RoomManagement.delete_room("room-abc")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def slow_livekit():
|
||||
"""A LiveKit address that answers every call half a second after it is made."""
|
||||
server = socket.create_server(("127.0.0.1", 0))
|
||||
held = []
|
||||
|
||||
def answer_later(connection):
|
||||
time.sleep(0.5)
|
||||
with contextlib.suppress(OSError):
|
||||
connection.sendall(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
|
||||
|
||||
def accept():
|
||||
while True:
|
||||
try:
|
||||
connection, _ = server.accept()
|
||||
except OSError:
|
||||
return
|
||||
held.append(connection)
|
||||
threading.Thread(
|
||||
target=answer_later, args=(connection,), daemon=True
|
||||
).start()
|
||||
|
||||
threading.Thread(target=accept, daemon=True).start()
|
||||
yield f"http://127.0.0.1:{server.getsockname()[1]:d}"
|
||||
server.close()
|
||||
for connection in held:
|
||||
connection.close()
|
||||
|
||||
|
||||
def test_create_livekit_client_timeout(slow_livekit, settings):
|
||||
"""A client given a timeout gives up after it, other clients keep the set one."""
|
||||
settings.LIVEKIT_CONFIGURATION = {
|
||||
**settings.LIVEKIT_CONFIGURATION,
|
||||
"url": slow_livekit,
|
||||
}
|
||||
settings.LIVEKIT_API_TIMEOUT_SECONDS = 5
|
||||
|
||||
async def delete_room(client):
|
||||
try:
|
||||
return await client.room.delete_room(DeleteRoomRequest(room="room-abc"))
|
||||
finally:
|
||||
await client.aclose()
|
||||
|
||||
@async_to_sync
|
||||
async def delete_rooms():
|
||||
patient = create_livekit_client()
|
||||
hasty = create_livekit_client(timeout=0.1)
|
||||
return await asyncio.gather(
|
||||
delete_room(patient), delete_room(hasty), return_exceptions=True
|
||||
)
|
||||
|
||||
patient, hasty = delete_rooms()
|
||||
|
||||
assert not isinstance(patient, BaseException)
|
||||
assert isinstance(hasty, TimeoutError)
|
||||
|
||||
|
||||
class TimingOutClient:
|
||||
"""A LiveKit client whose every call times out."""
|
||||
|
||||
@@ -320,6 +444,34 @@ def test_create_livekit_client_custom_configuration():
|
||||
assert url_of_new_client().startswith("http://mock-url.com")
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("two_clusters")
|
||||
@pytest.mark.parametrize(("name", "expected"), [(None, "paris"), ("lyon", "lyon")])
|
||||
def test_create_livekit_client_cluster(name, expected):
|
||||
"""A client calls its cluster on the API URL, with the cluster's key and secret.
|
||||
|
||||
The default cluster is called when none is given.
|
||||
"""
|
||||
cluster = get_livekit_cluster(name) if name else None
|
||||
expected_cluster = get_livekit_cluster(expected)
|
||||
|
||||
@async_to_sync
|
||||
async def target_of_new_client():
|
||||
client = create_livekit_client(cluster=cluster)
|
||||
room_service = client.room
|
||||
await client.aclose()
|
||||
return (
|
||||
room_service._client.host, # pylint: disable=protected-access
|
||||
room_service.api_key,
|
||||
room_service.api_secret,
|
||||
)
|
||||
|
||||
assert target_of_new_client() == (
|
||||
expected_cluster.api_url,
|
||||
expected_cluster.api_key,
|
||||
expected_cluster.api_secret,
|
||||
)
|
||||
|
||||
|
||||
@mock.patch("core.utils.create_livekit_client")
|
||||
def test_notify_participants_error(mock_create_livekit_client):
|
||||
"""Test participant notification with API error."""
|
||||
@@ -420,3 +572,37 @@ def test_notify_participants_success(mock_create_livekit_client):
|
||||
|
||||
# Verify aclose was called
|
||||
mock_api_instance.aclose.assert_called_once()
|
||||
|
||||
|
||||
@mock.patch("core.utils.create_livekit_client")
|
||||
def test_notify_participants_on_the_placed_cluster(
|
||||
mock_create_livekit_client, two_livekit_clusters, django_assert_num_queries
|
||||
):
|
||||
"""Participants are notified on the LiveKit cluster their room is placed on."""
|
||||
_paris, lyon = two_livekit_clusters
|
||||
Placement.objects.create(room_name="room-number-1", cluster="lyon")
|
||||
mock_create_livekit_client.return_value = mock.AsyncMock()
|
||||
|
||||
with django_assert_num_queries(1):
|
||||
notify_participants(room_name="room-number-1", notification_data={"foo": "foo"})
|
||||
|
||||
mock_create_livekit_client.assert_called_once_with(cluster=lyon)
|
||||
|
||||
|
||||
@mock.patch("core.utils.create_livekit_client")
|
||||
def test_notify_participants_on_the_given_cluster(
|
||||
mock_create_livekit_client, two_livekit_clusters, django_assert_num_queries
|
||||
):
|
||||
"""Participants are notified on the cluster given, whatever the placement."""
|
||||
paris, _lyon = two_livekit_clusters
|
||||
Placement.objects.create(room_name="room-number-1", cluster="lyon")
|
||||
mock_create_livekit_client.return_value = mock.AsyncMock()
|
||||
|
||||
with django_assert_num_queries(0):
|
||||
notify_participants(
|
||||
room_name="room-number-1",
|
||||
notification_data={"foo": "foo"},
|
||||
cluster=paris,
|
||||
)
|
||||
|
||||
mock_create_livekit_client.assert_called_once_with(cluster=paris)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Shared helpers for tests in the Meet core application"""
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
@@ -7,10 +8,13 @@ from datetime import datetime, timedelta, timezone
|
||||
from django.conf import settings
|
||||
|
||||
import jwt
|
||||
from livekit.api import AccessToken
|
||||
|
||||
from core.factories import ApplicationFactory
|
||||
from core.models import ApplicationScope
|
||||
|
||||
CREATED_LIVEKIT_ROOM_SID = "RM_created"
|
||||
|
||||
|
||||
def generate_user_access_token(user, application=None, **overrides):
|
||||
"""Generate a valid user access JWT signed with the token secret.
|
||||
@@ -57,6 +61,32 @@ def random_remote_addr():
|
||||
)
|
||||
|
||||
|
||||
def livekit_cluster_entry(name, **overrides):
|
||||
"""Describe a LiveKit cluster as an entry of LIVEKIT_CLUSTERS.
|
||||
|
||||
Each cluster gets its own API key, and a secret long enough to sign tokens.
|
||||
"""
|
||||
return {
|
||||
"name": name,
|
||||
"url": f"wss://{name}.livekit.test",
|
||||
"api_key": f"key-{name}",
|
||||
"api_secret": f"secret-{name}-padded-for-minimum-len!-livekit",
|
||||
**overrides,
|
||||
}
|
||||
|
||||
|
||||
def livekit_webhook_token(body, api_key, api_secret):
|
||||
"""Return the token a LiveKit cluster sends with a webhook body.
|
||||
|
||||
It is signed with this API key and secret, and holds the body's SHA-256.
|
||||
"""
|
||||
token = AccessToken(api_key, api_secret)
|
||||
token.claims.sha256 = base64.b64encode(
|
||||
hashlib.sha256(body.encode()).digest()
|
||||
).decode()
|
||||
return token.to_jwt()
|
||||
|
||||
|
||||
def transit_code_target(code):
|
||||
"""Return the audit target of a code: its SHA-256, never the code itself."""
|
||||
return {
|
||||
|
||||
+90
-22
@@ -36,6 +36,8 @@ from livekit.api import ( # pylint: disable=E0611
|
||||
VideoGrants,
|
||||
)
|
||||
|
||||
from core.livekit.clusters import LiveKitClusterConfig, get_default_livekit_cluster
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -70,6 +72,8 @@ def generate_token( # noqa: PLR0917
|
||||
role: Optional[str] = None,
|
||||
participant_id: Optional[str] = None,
|
||||
ttl: Optional[timedelta] = None,
|
||||
*,
|
||||
cluster: Optional[LiveKitClusterConfig] = None,
|
||||
) -> str:
|
||||
"""Generate a LiveKit access token for a user in a specific room.
|
||||
|
||||
@@ -86,11 +90,17 @@ def generate_token( # noqa: PLR0917
|
||||
participant_id (Optional[str]): Stable identifier for anonymous users;
|
||||
used as identity when user.is_anonymous.
|
||||
ttl (Optional[timedelta]): Token validity duration. Defaults to LiveKit SDK default.
|
||||
cluster (Optional[LiveKitClusterConfig]): The LiveKit cluster the token is for,
|
||||
signed with its API key and secret. Defaults to the default
|
||||
cluster.
|
||||
|
||||
Returns:
|
||||
str: The LiveKit JWT access token.
|
||||
"""
|
||||
|
||||
if cluster is None:
|
||||
cluster = get_default_livekit_cluster()
|
||||
|
||||
is_admin_or_owner = role in ("owner", "administrator")
|
||||
if is_admin_or_owner:
|
||||
sources = settings.LIVEKIT_DEFAULT_SOURCES
|
||||
@@ -124,10 +134,7 @@ def generate_token( # noqa: PLR0917
|
||||
display_name = (username or default_username) if can_edit else default_username
|
||||
|
||||
token = (
|
||||
AccessToken(
|
||||
api_key=settings.LIVEKIT_CONFIGURATION["api_key"],
|
||||
api_secret=settings.LIVEKIT_CONFIGURATION["api_secret"],
|
||||
)
|
||||
AccessToken(api_key=cluster.api_key, api_secret=cluster.api_secret)
|
||||
.with_grants(video_grants)
|
||||
.with_identity(identity)
|
||||
.with_name(display_name)
|
||||
@@ -153,6 +160,8 @@ def generate_livekit_config( # noqa: PLR0917
|
||||
color: Optional[str] = None,
|
||||
configuration: Optional[dict] = None,
|
||||
participant_id: Optional[str] = None,
|
||||
*,
|
||||
cluster: LiveKitClusterConfig,
|
||||
) -> dict:
|
||||
"""Generate LiveKit configuration for room access.
|
||||
|
||||
@@ -165,9 +174,12 @@ def generate_livekit_config( # noqa: PLR0917
|
||||
configuration (Optional[dict]): Room configuration dict that can override default settings.
|
||||
participant_id (Optional[str]): Stable identifier for anonymous users;
|
||||
used as identity when user.is_anonymous.
|
||||
cluster (LiveKitClusterConfig): The LiveKit cluster to connect to,
|
||||
the one the room's LiveKit room is placed on.
|
||||
|
||||
Returns:
|
||||
dict: LiveKit configuration with URL, room and access token
|
||||
dict: LiveKit configuration with the URL browsers connect to on the
|
||||
cluster, the room and an access token signed for that cluster
|
||||
"""
|
||||
|
||||
sources = None
|
||||
@@ -175,7 +187,7 @@ def generate_livekit_config( # noqa: PLR0917
|
||||
sources = configuration.get("can_publish_sources", None)
|
||||
|
||||
return {
|
||||
"url": settings.LIVEKIT_CONFIGURATION["url"],
|
||||
"url": cluster.url,
|
||||
"room": room_id,
|
||||
"token": generate_token(
|
||||
room=room_id,
|
||||
@@ -185,6 +197,7 @@ def generate_livekit_config( # noqa: PLR0917
|
||||
sources=sources,
|
||||
role=role,
|
||||
participant_id=participant_id,
|
||||
cluster=cluster,
|
||||
),
|
||||
}
|
||||
|
||||
@@ -219,16 +232,21 @@ def generate_s3_authorization_headers(key):
|
||||
return request
|
||||
|
||||
|
||||
async def bound_livekit_request(request, handler):
|
||||
"""Give up on a LiveKit request, its answer included, after the set timeout.
|
||||
def bound_livekit_requests(timeout: float):
|
||||
"""Return a client middleware giving up on a LiveKit request, its answer
|
||||
included, after `timeout` seconds.
|
||||
|
||||
The SDK passes timeout=None to aiohttp on every call, which means no limit,
|
||||
so a timeout set on the session never applies.
|
||||
"""
|
||||
async with asyncio.timeout(settings.LIVEKIT_API_TIMEOUT_SECONDS):
|
||||
response = await handler(request)
|
||||
await response.read()
|
||||
return response
|
||||
|
||||
async def bound_livekit_request(request, handler):
|
||||
async with asyncio.timeout(timeout):
|
||||
response = await handler(request)
|
||||
await response.read()
|
||||
return response
|
||||
|
||||
return bound_livekit_request
|
||||
|
||||
|
||||
class LiveKitClient(LiveKitAPI):
|
||||
@@ -239,16 +257,46 @@ class LiveKitClient(LiveKitAPI):
|
||||
await self._session.close()
|
||||
|
||||
|
||||
def create_livekit_client(custom_configuration=None):
|
||||
"""Create and return a configured LiveKit API client."""
|
||||
def livekit_api_configuration(cluster: Optional[LiveKitClusterConfig] = None) -> dict:
|
||||
"""Return the URL, API key and secret Django calls a LiveKit cluster with.
|
||||
|
||||
The URL is the cluster's api_url, not the one browsers connect to. The
|
||||
default cluster is used when none is given.
|
||||
"""
|
||||
if cluster is None:
|
||||
cluster = get_default_livekit_cluster()
|
||||
return {
|
||||
"url": cluster.api_url,
|
||||
"api_key": cluster.api_key,
|
||||
"api_secret": cluster.api_secret,
|
||||
}
|
||||
|
||||
|
||||
def create_livekit_client(
|
||||
custom_configuration: Optional[dict] = None,
|
||||
*,
|
||||
cluster: Optional[LiveKitClusterConfig] = None,
|
||||
timeout: Optional[float] = None,
|
||||
) -> LiveKitClient:
|
||||
"""Create a LiveKit API client calling a LiveKit cluster.
|
||||
|
||||
Args:
|
||||
custom_configuration (Optional[dict]): The URL, API key and secret to call
|
||||
LiveKit with, in place of the cluster's.
|
||||
cluster (Optional[LiveKitClusterConfig]): The LiveKit cluster to call.
|
||||
Defaults to the default cluster.
|
||||
timeout (Optional[float]): Seconds after which each request of this client
|
||||
gives up. Defaults to LIVEKIT_API_TIMEOUT_SECONDS.
|
||||
"""
|
||||
configuration = custom_configuration or livekit_api_configuration(cluster)
|
||||
if timeout is None:
|
||||
timeout = settings.LIVEKIT_API_TIMEOUT_SECONDS
|
||||
|
||||
connector = None if settings.LIVEKIT_VERIFY_SSL else aiohttp.TCPConnector(ssl=False)
|
||||
session = aiohttp.ClientSession(
|
||||
connector=connector, middlewares=(bound_livekit_request,)
|
||||
connector=connector, middlewares=(bound_livekit_requests(timeout),)
|
||||
)
|
||||
|
||||
# Use default configuration if none provided
|
||||
configuration = custom_configuration or settings.LIVEKIT_CONFIGURATION
|
||||
|
||||
return LiveKitClient(session=session, **configuration)
|
||||
|
||||
|
||||
@@ -256,11 +304,31 @@ class NotificationError(Exception):
|
||||
"""Notification delivery to room participants fails."""
|
||||
|
||||
|
||||
@async_to_sync
|
||||
async def notify_participants(room_name: str, notification_data: dict):
|
||||
"""Send notification data to all participants in a LiveKit room."""
|
||||
def notify_participants(
|
||||
room_name: str,
|
||||
notification_data: dict,
|
||||
*,
|
||||
cluster: Optional[LiveKitClusterConfig] = None,
|
||||
):
|
||||
"""Send notification data to all participants in a LiveKit room.
|
||||
|
||||
lkapi = create_livekit_client()
|
||||
The LiveKit room is reached on `cluster`, by default the LiveKit cluster it
|
||||
is placed on.
|
||||
"""
|
||||
# pylint: disable-next=import-outside-toplevel,cyclic-import
|
||||
from core.livekit.placements import PlacementService # noqa: PLC0415
|
||||
|
||||
cluster = cluster or PlacementService.cluster_for(room_name)
|
||||
_notify_participants(room_name, notification_data, cluster=cluster)
|
||||
|
||||
|
||||
@async_to_sync
|
||||
async def _notify_participants(
|
||||
room_name: str, notification_data: dict, *, cluster: LiveKitClusterConfig
|
||||
):
|
||||
"""Send notification data to a LiveKit room on a LiveKit cluster."""
|
||||
|
||||
lkapi = create_livekit_client(cluster=cluster)
|
||||
|
||||
try:
|
||||
room_response = await lkapi.room.list_rooms(
|
||||
|
||||
@@ -29,6 +29,8 @@ from pydantic import ValidationError as PydanticValidationError
|
||||
from sentry_sdk.integrations.django import DjangoIntegration
|
||||
from sentry_sdk.integrations.logging import ignore_logger
|
||||
|
||||
from core.livekit.specs import parse_livekit_clusters
|
||||
|
||||
# Build paths inside the project like this: BASE_DIR / 'subdir'.
|
||||
BASE_DIR = path.dirname(path.dirname(path.abspath(__file__)))
|
||||
|
||||
@@ -467,6 +469,11 @@ class Base(Configuration):
|
||||
environ_name="CONNECTION_TEST_THROTTLE_RATES",
|
||||
environ_prefix=None,
|
||||
),
|
||||
"unregistered_room_placement": values.Value(
|
||||
default="60/minute",
|
||||
environ_name="UNREGISTERED_ROOM_PLACEMENT_THROTTLE_RATES",
|
||||
environ_prefix=None,
|
||||
),
|
||||
},
|
||||
}
|
||||
MONITORED_THROTTLE_FAILURE_CALLBACK = (
|
||||
@@ -778,6 +785,18 @@ class Base(Configuration):
|
||||
),
|
||||
"url": values.Value(environ_name="LIVEKIT_API_URL", environ_prefix=None),
|
||||
}
|
||||
LIVEKIT_CLUSTERS = SecretFileValue(
|
||||
None, environ_name="LIVEKIT_CLUSTERS", environ_prefix=None
|
||||
)
|
||||
LIVEKIT_PLACEMENT_STRATEGY = values.Value(
|
||||
"core.livekit.strategies.WeightedRandomStrategy",
|
||||
environ_name="LIVEKIT_PLACEMENT_STRATEGY",
|
||||
environ_prefix=None,
|
||||
)
|
||||
DEFAULT_EXCEPTION_REPORTER_FILTER = (
|
||||
"core.exception_reporter.LiveKitSafeExceptionReporterFilter"
|
||||
)
|
||||
|
||||
LIVEKIT_FORCE_WSS_PROTOCOL = values.BooleanValue(
|
||||
False, environ_name="LIVEKIT_FORCE_WSS_PROTOCOL", environ_prefix=None
|
||||
)
|
||||
@@ -851,6 +870,9 @@ class Base(Configuration):
|
||||
LIVEKIT_API_TIMEOUT_SECONDS = values.PositiveIntegerValue(
|
||||
10, environ_name="LIVEKIT_API_TIMEOUT_SECONDS", environ_prefix=None
|
||||
)
|
||||
LIVEKIT_CREATE_ROOM_TIMEOUT_SECONDS = values.PositiveIntegerValue(
|
||||
3, environ_name="LIVEKIT_CREATE_ROOM_TIMEOUT_SECONDS", environ_prefix=None
|
||||
)
|
||||
# Regex to filter webhook events by room name. Only matching events are processed.
|
||||
LIVEKIT_WEBHOOK_EVENTS_FILTER_REGEX = values.Value(
|
||||
None, environ_name="LIVEKIT_WEBHOOK_EVENTS_FILTER_REGEX", environ_prefix=None
|
||||
@@ -1593,6 +1615,33 @@ class Base(Configuration):
|
||||
f"'{cls.AUDIT_LOG_DATA_STREAM_NAMESPACE}' must not contain '-'."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _check_livekit_clusters(cls):
|
||||
"""Ensure LIVEKIT_CLUSTERS, when set, lists valid LiveKit clusters.
|
||||
|
||||
Unset, the registry holds a single cluster built from
|
||||
LIVEKIT_CONFIGURATION, so every source of clusters is covered.
|
||||
|
||||
Telephony and roomkit need a single cluster, so LIVEKIT_CLUSTERS may list
|
||||
more than one only while both are disabled.
|
||||
"""
|
||||
if cls.LIVEKIT_CLUSTERS is None:
|
||||
return
|
||||
clusters = parse_livekit_clusters(cls.LIVEKIT_CLUSTERS)
|
||||
enabled = [
|
||||
name
|
||||
for name in ("ROOM_TELEPHONY_ENABLED", "ROOMKIT_ENABLED")
|
||||
if getattr(cls, name)
|
||||
]
|
||||
if len(clusters) > 1 and enabled:
|
||||
raise ValueError(
|
||||
f"{' and '.join(enabled)} cannot be combined with more than one "
|
||||
f"LiveKit cluster, and LIVEKIT_CLUSTERS lists {len(clusters)}: "
|
||||
"phone and roomkit callers reach the SIP server of one cluster "
|
||||
"before their PIN is known, so Django cannot route them to the "
|
||||
"cluster their room is placed on."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _configure_database_pool(cls):
|
||||
"""Inject the psycopg pool options into the default database when enabled."""
|
||||
@@ -1653,9 +1702,14 @@ class Base(Configuration):
|
||||
cls._check_recording_encoding_maps()
|
||||
cls._check_audit_log_data_stream_namespace()
|
||||
cls._configure_database_pool()
|
||||
cls._check_livekit_clusters()
|
||||
|
||||
if cls.LIVEKIT_API_TIMEOUT_SECONDS < 1:
|
||||
raise ValueError("LIVEKIT_API_TIMEOUT_SECONDS must be at least 1")
|
||||
for timeout_setting in (
|
||||
"LIVEKIT_API_TIMEOUT_SECONDS",
|
||||
"LIVEKIT_CREATE_ROOM_TIMEOUT_SECONDS",
|
||||
):
|
||||
if getattr(cls, timeout_setting) < 1:
|
||||
raise ValueError(f"{timeout_setting} must be at least 1")
|
||||
|
||||
if (
|
||||
cls.SUMMARY_SERVICE_VERSION == 1
|
||||
@@ -1821,6 +1875,7 @@ class Test(Base):
|
||||
"api_secret": "secret-key-padded-for-minimum-len!-livekit",
|
||||
"url": "http://127.0.0.1.nip.io:7880",
|
||||
}
|
||||
LIVEKIT_CLUSTERS = None
|
||||
|
||||
APPLICATION_ENABLED = True
|
||||
APPLICATION_JWT_SECRET_KEY = "secret-key-padded-for-minimum-len!-application" # noqa:S105
|
||||
|
||||
Reference in New Issue
Block a user