Compare commits

..

13 Commits

Author SHA1 Message Date
lebaudantoine 98d0b48015 fixup! wip 2026-09-02 23:47:39 +02:00
lebaudantoine 7412c02341 fixup! wip enhance analytics 2026-09-02 23:47:09 +02:00
lebaudantoine f2b3af6633 fixup! wip 2026-09-02 23:12:35 +02:00
lebaudantoine 2aafd93b46 fixup! wip 2026-09-02 22:46:15 +02:00
lebaudantoine 87931f50cd fixup! wip try to enhance error handling 2026-09-02 22:30:16 +02:00
lebaudantoine 35cb114160 wip try to enhance error handling 2026-09-02 20:48:16 +02:00
lebaudantoine c783fefa57 fixup! wip 2026-09-02 20:36:33 +02:00
lebaudantoine 09a0238936 wip propose back a no effect options to help user deactivate their effect 2026-09-02 20:09:52 +02:00
lebaudantoine 03870ad1fa wip enhance analytics 2026-09-02 19:54:44 +02:00
lebaudantoine f37992a089 fixup! wip 2026-09-02 19:08:57 +02:00
lebaudantoine c7c6d20b91 wip silence
info log which are purely informational and log with std err
and failure that mediapipe also raises as real js exceptions
2026-09-02 19:05:12 +02:00
lebaudantoine 2a71e5b836 wip 2026-09-02 18:53:33 +02:00
lebaudantoine 089d016d12 🐛(frontend) cache supportsBackgroundProcessors WebGL2 probe result
`supportsBackgroundProcessors()` creates a live WebGL2 context on
every call and never releases it. The method is called from render
paths (e.g. the Effects button on the join screen), so without
caching, each re-render leaks a context until the browser hits its
live-context limit.

This is one of the ways MediaPipe later fails with:

  "emscripten_webgl_create_context() returned error 0"

Support cannot change within a session, so probe once and cache the
result.
2026-09-02 18:26:12 +02:00
20 changed files with 535 additions and 1054 deletions
-1
View File
@@ -12,7 +12,6 @@ and this project adheres to
- ✨(frontend) add 1080p sending resolution option #1660
- ✨(backend) add Traefik support via configurable media-auth url header #1649
- ✨(backend) update a room's attributes from the external API
### Fixed
+3 -78
View File
@@ -16,11 +16,11 @@ info:
* `rooms:list` – List rooms accessible to the delegated user.
* `rooms:retrieve` – Retrieve details of a specific room.
* `rooms:create` – Create new rooms.
* `rooms:update` – Update the access level and configuration of existing rooms.
* `rooms:update` – **Coming soon** Update existing rooms, e.g., add attendees to a room.
* `rooms:delete` – **Coming soon** Delete rooms generated by the application.
#### Upcoming Features
* **Add attendees to a room:** You will be able to update a room to include a list of attendees, allowing them to bypass the lobby system automatically.
* **Delete application-generated rooms:** Rooms created via the application can be deleted when no longer needed.
@@ -310,67 +310,6 @@ paths:
'404':
$ref: '#/components/responses/RoomNotFoundError'
patch:
tags:
- Rooms
summary: Update a room
description: |
Partially updates a room. Only the delegated user's rooms where they are
administrator or owner can be updated; any other role gets a `403`.
**Updatable fields:** `access_level` and `configuration`. Every other field
(`id`, `name`, `slug`, `pin_code`) is read-only and silently ignored when sent.
`configuration` is replaced as a whole, it is not merged with the stored one.
Send the complete object you want the room to end up with.
Full replacement (`PUT`) is not supported. Use `PATCH` instead.
operationId: updateRoom
security:
- BearerAuth: [rooms:update]
parameters:
- name: id
in: path
required: true
description: Room UUID
schema:
type: string
format: uuid
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/RoomUpdate'
examples:
accessLevelOnly:
summary: Change the access level
value:
access_level: "restricted"
configurationOnly:
summary: Replace the room configuration
value:
configuration:
everyone_can_mute: true
responses:
'200':
description: Room updated successfully
content:
application/json:
schema:
$ref: '#/components/schemas/Room'
'400':
$ref: '#/components/responses/BadRequestError'
'401':
$ref: '#/components/responses/UnauthorizedError'
'403':
$ref: '#/components/responses/ForbiddenError'
'404':
$ref: '#/components/responses/RoomNotFoundError'
'405':
description: |
Method not allowed, `PUT` is not supported on this endpoint.
components:
securitySchemes:
BearerAuth:
@@ -447,17 +386,6 @@ components:
configuration:
$ref: '#/components/schemas/RoomConfiguration'
RoomUpdate:
type: object
description: |
Fields that can be updated on an existing room. Both are optional, omitted
fields keep their current value.
properties:
access_level:
$ref: '#/components/schemas/RoomAccessLevel'
configuration:
$ref: '#/components/schemas/RoomConfiguration'
RoomConfiguration:
type: object
description: |
@@ -499,9 +427,6 @@ components:
- `public`: Anyone with the room link can join directly, no authentication required.
- `trusted`: Authenticated users join directly. Unauthenticated users wait in the lobby for approval.
- `restricted`: Only participants explicitly trusted by the owner bypass the lobby. Everyone else waits for approval regardless of authentication.
`public` is rejected with a `400` unless the deployment explicitly enables it
for this API. This applies both when creating a room and when updating one.
example: "trusted"
Room:
+1 -76
View File
@@ -20,7 +20,7 @@ info:
* `lasuite_visio:rooms:list` – List rooms accessible to the delegated user.
* `lasuite_visio:rooms:retrieve` – Retrieve details of a specific room.
* `lasuite_visio:rooms:create` – Create new rooms.
* `lasuite_visio:rooms:update` – Update the access level and configuration of existing rooms.
* `lasuite_visio:rooms:update` – **Coming soon** Update existing rooms, e.g., add attendees to a room.
* `lasuite_visio:rooms:delete` – **Coming soon** Delete rooms generated by the application.
#### Upcoming Features
@@ -206,67 +206,6 @@ paths:
'404':
$ref: '#/components/responses/RoomNotFoundError'
patch:
tags:
- Rooms
summary: Update a room
description: |
Partially updates a room. Only rooms where the user is administrator or
owner can be updated; any other role gets a `403`.
**Updatable fields:** `access_level` and `configuration`. Every other field
(`id`, `name`, `slug`, `pin_code`) is read-only and silently ignored when sent.
`configuration` is replaced as a whole, it is not merged with the stored one.
Send the complete object you want the room to end up with.
Full replacement (`PUT`) is not supported. Use `PATCH` instead.
operationId: updateRoom
security:
- BearerAuth: [rooms:update]
parameters:
- name: id
in: path
required: true
description: Room UUID
schema:
type: string
format: uuid
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/RoomUpdate'
examples:
accessLevelOnly:
summary: Change the access level
value:
access_level: "restricted"
configurationOnly:
summary: Replace the room configuration
value:
configuration:
everyone_can_mute: true
responses:
'200':
description: Room updated successfully
content:
application/json:
schema:
$ref: '#/components/schemas/Room'
'400':
$ref: '#/components/responses/BadRequestError'
'401':
$ref: '#/components/responses/UnauthorizedError'
'403':
$ref: '#/components/responses/ForbiddenError'
'404':
$ref: '#/components/responses/RoomNotFoundError'
'405':
description: |
Method not allowed, `PUT` is not supported on this endpoint.
components:
securitySchemes:
BearerAuth:
@@ -288,17 +227,6 @@ components:
configuration:
$ref: '#/components/schemas/RoomConfiguration'
RoomUpdate:
type: object
description: |
Fields that can be updated on an existing room. Both are optional, omitted
fields keep their current value.
properties:
access_level:
$ref: '#/components/schemas/RoomAccessLevel'
configuration:
$ref: '#/components/schemas/RoomConfiguration'
RoomConfiguration:
type: object
description: |
@@ -340,9 +268,6 @@ components:
- `public`: Anyone with the room link can join directly, no authentication required.
- `trusted`: Authenticated users join directly. Unauthenticated users wait in the lobby for approval.
- `restricted`: Only participants explicitly trusted by the owner bypass the lobby. Everyone else waits for approval regardless of authentication.
`public` is rejected with a `400` unless the deployment explicitly enables it
for this API. This applies both when creating a room and when updating one.
example: "trusted"
Room:
-1
View File
@@ -8,7 +8,6 @@ class AnalyticsEvent(StrEnum):
# Rooms
ROOM_CREATED = "room_created"
ROOM_UPDATED = "room_updated"
# Roomkit (meeting-room SIP devices)
ROOMKIT_JOINED = "roomkit_joined"
+25 -2
View File
@@ -75,7 +75,11 @@ from core.services.participants_management import (
ParticipantsManagementException,
)
from core.services.room_creation import RoomCreation
from core.services.room_management import RoomManagement
from core.services.room_management import (
RoomManagement,
RoomManagementException,
RoomNotFoundException,
)
from core.services.room_roles import (
RoomRoleError,
RoomRoleService,
@@ -352,7 +356,26 @@ class RoomViewSet(
):
return
RoomManagement.sync_room_metadata(room)
metadata = {
"configuration": room.configuration,
"access_level": room.access_level,
}
try:
RoomManagement().update_metadata(
room_name=str(room.id),
metadata=metadata,
)
except RoomNotFoundException:
logger.info(
"LiveKit room %s does not exist yet, skipping metadata sync",
room.id,
)
except RoomManagementException:
logger.warning(
"Failed to sync metadata to LiveKit for room %s",
room.id,
)
@decorators.action(
detail=True,
+21 -64
View File
@@ -1,6 +1,5 @@
"""External API endpoints"""
import copy
from logging import getLogger
from django.conf import settings
@@ -26,7 +25,6 @@ from rest_framework import (
from core import analytics, api, models
from core.api.feature_flag import FeatureFlag
from core.services.jwt_token import JwtTokenService
from core.services.room_management import RoomManagement
from ..services.provisional_user_service import (
ProvisionalUserCreationDisabledError,
@@ -144,7 +142,6 @@ class RoomViewSet(
mixins.CreateModelMixin,
mixins.RetrieveModelMixin,
mixins.ListModelMixin,
mixins.UpdateModelMixin,
viewsets.GenericViewSet,
):
"""Application-delegated API for room management.
@@ -157,12 +154,8 @@ class RoomViewSet(
- list: List rooms the user has access to (requires 'rooms:list' scope)
- retrieve: Get room details (requires 'rooms:retrieve' scope)
- create: Create a new room owned by the user (requires 'rooms:create' scope)
- partial_update: Update a room's access level and configuration, for
administrators and owners only (requires 'rooms:update' scope)
"""
http_method_names = ["get", "post", "patch", "head", "options"]
authentication_classes = [
authentication.ApplicationJWTAuthentication,
authentication.AddonsJWTAuthentication,
@@ -196,39 +189,7 @@ class RoomViewSet(
serializer = self.get_serializer(queryset, many=True)
return drf_response.Response(serializer.data)
def _track_room_event(self, room, event, **extra_properties):
"""Log a room operation for auditing and forward it to analytics."""
auth_method = type(self.request.successful_authenticator).__name__
client_id = (self.request.auth or {}).get("client_id", "unknown")
# Log for auditing
details = "".join(f", {key}={value}" for key, value in extra_properties.items())
logger.info(
"Room %s via application: room_id=%s, user_id=%s, client_id=%s, auth_method=%s%s",
event.removeprefix("room_"),
room.id,
self.request.user.id,
client_id,
auth_method,
details,
)
analytics.capture(
self.request.user,
event,
{
"room_id": str(room.pk),
"access_level": room.access_level,
"client_id": client_id,
"external_api": True,
"auth_method": auth_method,
**extra_properties,
"$set": {"email": self.request.user.email},
},
)
def perform_create(self, serializer: serializers.RoomSerializer):
def perform_create(self, serializer):
"""Set the current user as owner of the newly created room."""
room = serializer.save()
models.ResourceAccess.objects.create(
@@ -237,31 +198,27 @@ class RoomViewSet(
role=models.RoleChoices.OWNER,
)
self._track_room_event(room, analytics.AnalyticsEvent.ROOM_CREATED)
auth_method = type(self.request.successful_authenticator).__name__
client_id = (self.request.auth or {}).get("client_id", "unknown")
def perform_update(self, serializer: serializers.RoomSerializer):
"""Persist the room update, sync it to LiveKit, then log and track it."""
previous_values = {
"access_level": serializer.instance.access_level,
"configuration": copy.deepcopy(serializer.instance.configuration),
}
room = serializer.save()
# Report the fields that actually changed, not the ones that were submitted.
updated_fields = sorted(
field
for field, previous_value in previous_values.items()
if getattr(room, field) != previous_value
# Log for auditing
logger.info(
"Room created via application: room_id=%s, user_id=%s, client_id=%s, auth_method=%s",
room.id,
self.request.user.id,
client_id,
auth_method,
)
if updated_fields:
RoomManagement.sync_room_metadata(room)
self._track_room_event(
room,
analytics.AnalyticsEvent.ROOM_UPDATED,
updated_fields=updated_fields,
previous_access_level=previous_values["access_level"],
analytics.capture(
self.request.user,
analytics.AnalyticsEvent.ROOM_CREATED,
{
"room_id": str(room.pk),
"access_level": room.access_level,
"client_id": client_id,
"external_api": True,
"auth_method": auth_method,
"$set": {"email": self.request.user.email},
},
)
+4
View File
@@ -48,6 +48,8 @@ class ResourceFactory(factory.django.DjangoModelFactory):
else:
UserResourceAccessFactory(resource=self, user=item[0], role=item[1])
self.save()
class UserResourceAccessFactory(factory.django.DjangoModelFactory):
"""Create fake resource user accesses for testing."""
@@ -95,6 +97,8 @@ class RecordingFactory(factory.django.DjangoModelFactory):
recording=self, user=item[0], role=item[1]
)
self.save()
class UserRecordingAccessFactory(factory.django.DjangoModelFactory):
"""Create fake recording user accesses for testing."""
@@ -44,7 +44,7 @@ class RecordingEventsService:
recording_status = status_mapping.get(egress_status)
if recording_status:
try:
RoomManagement.update_metadata(
RoomManagement().update_metadata(
room_name, {"recording_status": recording_status}
)
except RoomNotFoundException:
@@ -68,7 +68,7 @@ class WorkerServiceMediator:
mode = recording.options.get("original_mode", None) or recording.mode
try:
RoomManagement.update_metadata(
RoomManagement().update_metadata(
room_name, {"recording_mode": mode, "recording_status": "starting"}
)
except RoomNotFoundException:
+1 -1
View File
@@ -192,7 +192,7 @@ class LiveKitEventsService:
try:
room_name = str(recording.room.id)
RoomManagement.update_metadata(
RoomManagement().update_metadata(
room_name, remove_keys=["recording_mode", "recording_status"]
)
except RoomNotFoundException:
+2 -33
View File
@@ -30,10 +30,9 @@ class RoomNotFoundException(RoomManagementException):
class RoomManagement:
"""Service for managing LiveKit rooms."""
@classmethod
@async_to_sync
async def update_metadata(
cls,
self,
room_name: str,
metadata: Optional[Dict] = None,
remove_keys: Optional[list[str]] = None,
@@ -91,9 +90,8 @@ class RoomManagement:
finally:
await lkapi.aclose()
@classmethod
@async_to_sync
async def delete_room(cls, room_name: str):
async def delete_room(self, room_name: str):
"""Delete a LiveKit room and disconnect all participants.
Raises:
@@ -118,32 +116,3 @@ class RoomManagement:
raise RoomManagementException("Could not delete room") from e
finally:
await lkapi.aclose()
@classmethod
def sync_room_metadata(cls, room):
"""Push a room's configuration and access level to its LiveKit room metadata.
Failures are swallowed: a room that is not live yet, or a LiveKit hiccup,
should never fail the request that triggered the update.
"""
metadata = {
"configuration": room.configuration,
"access_level": room.access_level,
}
try:
cls.update_metadata(
room_name=str(room.id),
metadata=metadata,
)
except RoomNotFoundException:
logger.info(
"LiveKit room %s does not exist yet, skipping metadata sync",
room.id,
)
except RoomManagementException:
logger.warning(
"Failed to sync metadata to LiveKit for room %s",
room.id,
)
+1 -1
View File
@@ -31,7 +31,7 @@ def delete_connection_test_room(room_name: str):
return
try:
RoomManagement.delete_room(room_name)
RoomManagement().delete_room(room_name)
except RoomNotFoundException:
# Room may already be gone after empty/departure timeout.
logger.info("Connection test room '%s' already gone.", room_name)
@@ -381,11 +381,38 @@ def test_api_rooms_update_administrators_of_another():
assert other_room.slug == "old-name"
@pytest.mark.parametrize("exception", [RoomNotFoundException, RoomManagementException])
@patch.object(RoomManagement, "update_metadata")
def test_api_rooms_update_livekit_sync_failure(mock_update_metadata, exception):
"""Should not fail the API request when the LiveKit metadata sync fails."""
mock_update_metadata.side_effect = exception
@patch.object(RoomManagement, "update_metadata", side_effect=RoomNotFoundException)
def test_api_rooms_update_livekit_room_not_found(mock_update_metadata):
"""Should not fail the API request when the LiveKit room does not exist yet."""
user = UserFactory()
room = RoomFactory(
users=[(user, random.choice(["administrator", "owner"]))],
configuration={},
)
client = APIClient()
client.force_login(user)
response = client.patch(
f"/api/v1.0/rooms/{room.id!s}/",
{"configuration": {"can_publish_sources": ["camera"]}},
format="json",
)
assert response.status_code == 200
room.refresh_from_db()
assert room.configuration == {"can_publish_sources": ["camera"]}
mock_update_metadata.assert_called_once_with(
room_name=str(room.id),
metadata={
"access_level": room.access_level,
"configuration": {"can_publish_sources": ["camera"]},
},
)
@patch.object(RoomManagement, "update_metadata", side_effect=RoomManagementException)
def test_api_rooms_update_livekit_sync_failure(mock_update_metadata):
"""Should not fail the API request when the LiveKit metadata sync fails."""
user = UserFactory()
room = RoomFactory(
users=[(user, random.choice(["administrator", "owner"]))],
@@ -5,8 +5,6 @@ from unittest import mock
import pytest
from livekit.api import TwirpError
from core.factories import RoomFactory
from core.models import RoomAccessLevel
from core.services.room_management import (
RoomManagement,
RoomManagementException,
@@ -22,7 +20,7 @@ def test_delete_room_calls_livekit(mock_create_livekit_client):
mock_api.aclose = mock.AsyncMock()
mock_create_livekit_client.return_value = mock_api
RoomManagement.delete_room("room-abc")
RoomManagement().delete_room("room-abc")
mock_api.room.delete_room.assert_awaited_once()
request = mock_api.room.delete_room.await_args.args[0]
@@ -41,7 +39,7 @@ def test_delete_room_raises_not_found(mock_create_livekit_client):
mock_create_livekit_client.return_value = mock_api
with pytest.raises(RoomNotFoundException):
RoomManagement.delete_room("missing-room")
RoomManagement().delete_room("missing-room")
mock_api.aclose.assert_awaited_once()
@@ -57,25 +55,6 @@ def test_delete_room_raises_management_exception(mock_create_livekit_client):
mock_create_livekit_client.return_value = mock_api
with pytest.raises(RoomManagementException):
RoomManagement.delete_room("room-abc")
RoomManagement().delete_room("room-abc")
mock_api.aclose.assert_awaited_once()
@mock.patch.object(RoomManagement, "update_metadata")
def test_sync_room_metadata_pushes_configuration_and_access_level(mock_update_metadata):
"""The room's configuration and access level are forwarded to LiveKit."""
room = RoomFactory.build(
access_level=RoomAccessLevel.RESTRICTED,
configuration={"everyone_can_mute": True},
)
RoomManagement.sync_room_metadata(room)
mock_update_metadata.assert_called_once_with(
room_name=str(room.id),
metadata={
"configuration": {"everyone_can_mute": True},
"access_level": RoomAccessLevel.RESTRICTED,
},
)
@@ -16,17 +16,8 @@ import responses
from lasuite.oidc_resource_server.authentication import ResourceServerAuthentication
from rest_framework.test import APIClient
from core.analytics import AnalyticsEvent
from core.factories import ApplicationFactory, RoomFactory, UserFactory
from core.models import (
Application,
ApplicationScope,
RoleChoices,
Room,
RoomAccessLevel,
User,
)
from core.services.room_management import RoomManagement
from core.models import ApplicationScope, RoleChoices, Room, RoomAccessLevel, User
pytestmark = pytest.mark.django_db
@@ -889,509 +880,6 @@ def test_api_rooms_create_public_access_level_when_default_is_public(settings):
assert response.data["access_level"] == RoomAccessLevel.PUBLIC
@mock.patch("core.external_api.viewsets.analytics.capture")
def test_api_rooms_create_tracks_analytics(mock_capture):
"""Creating a room should emit a ROOM_CREATED analytics event."""
user = UserFactory()
token = generate_test_token(user, [ApplicationScope.ROOMS_CREATE])
application = Application.objects.get()
client = APIClient()
client.credentials(HTTP_AUTHORIZATION=f"Bearer {token}")
response = client.post(
"/external-api/v1.0/rooms/",
{"access_level": RoomAccessLevel.RESTRICTED},
format="json",
)
assert response.status_code == 201
mock_capture.assert_called_once()
captured_user, event, properties = mock_capture.call_args[0]
assert captured_user == user
assert event == AnalyticsEvent.ROOM_CREATED
assert properties == {
"room_id": response.data["id"],
"access_level": RoomAccessLevel.RESTRICTED,
"client_id": str(application.client_id),
"external_api": True,
"auth_method": "ApplicationJWTAuthentication",
"$set": {"email": user.email},
}
def test_api_rooms_update_requires_authentication():
"""Updating a room without authentication should return 401."""
room = RoomFactory(users=[(UserFactory(), RoleChoices.OWNER)])
client = APIClient()
response = client.patch(
f"/external-api/v1.0/rooms/{room.id}/",
{"access_level": RoomAccessLevel.RESTRICTED},
format="json",
)
assert response.status_code == 401
def test_api_rooms_update_requires_scope():
"""Updating a room requires the ROOMS_UPDATE scope."""
user = UserFactory()
room = RoomFactory(users=[(user, RoleChoices.OWNER)])
# Token without ROOMS_UPDATE scope
token = generate_test_token(user, [ApplicationScope.ROOMS_RETRIEVE])
client = APIClient()
client.credentials(HTTP_AUTHORIZATION=f"Bearer {token}")
response = client.patch(
f"/external-api/v1.0/rooms/{room.id}/",
{"access_level": RoomAccessLevel.RESTRICTED},
format="json",
)
assert response.status_code == 403
assert (
"insufficient permissions. required scope: rooms:update"
in str(response.data).lower()
)
def test_api_rooms_update_no_scope():
"""Updating a room without any scope should return 403."""
user = UserFactory()
room = RoomFactory(users=[(user, RoleChoices.OWNER)])
token = generate_test_token(user, [])
client = APIClient()
client.credentials(HTTP_AUTHORIZATION=f"Bearer {token}")
response = client.patch(
f"/external-api/v1.0/rooms/{room.id}/",
{"access_level": RoomAccessLevel.RESTRICTED},
format="json",
)
assert response.status_code == 403
assert "insufficient permissions." in str(response.data).lower()
@mock.patch.object(RoomManagement, "update_metadata")
def test_api_rooms_update_owner_success(mock_update_metadata, settings):
"""An owner should be able to update the access level and the configuration."""
settings.APPLICATION_BASE_URL = "http://your-application.com"
user = UserFactory()
room = RoomFactory(
users=[(user, RoleChoices.OWNER)],
access_level=RoomAccessLevel.TRUSTED,
configuration={},
)
token = generate_test_token(user, [ApplicationScope.ROOMS_UPDATE])
client = APIClient()
client.credentials(HTTP_AUTHORIZATION=f"Bearer {token}")
response = client.patch(
f"/external-api/v1.0/rooms/{room.id}/",
{
"access_level": RoomAccessLevel.RESTRICTED,
"configuration": {"everyone_can_mute": True},
},
format="json",
)
assert response.status_code == 200
assert response.data["id"] == str(room.id)
assert response.data["access_level"] == RoomAccessLevel.RESTRICTED
assert response.data["configuration"] == {"everyone_can_mute": True}
assert response.data["url"] == f"http://your-application.com/{room.slug}"
room.refresh_from_db()
assert room.access_level == RoomAccessLevel.RESTRICTED
assert room.configuration == {"everyone_can_mute": True}
mock_update_metadata.assert_called_once_with(
room_name=str(room.id),
metadata={
"configuration": {"everyone_can_mute": True},
"access_level": RoomAccessLevel.RESTRICTED,
},
)
@mock.patch.object(RoomManagement, "update_metadata")
def test_api_rooms_update_replaces_configuration(mock_update_metadata):
"""The configuration is replaced as a whole, it is not merged with the stored one."""
user = UserFactory()
room = RoomFactory(
users=[(user, RoleChoices.OWNER)],
configuration={"can_publish_sources": ["camera"], "everyone_can_mute": True},
)
token = generate_test_token(user, [ApplicationScope.ROOMS_UPDATE])
client = APIClient()
client.credentials(HTTP_AUTHORIZATION=f"Bearer {token}")
response = client.patch(
f"/external-api/v1.0/rooms/{room.id}/",
{"configuration": {"everyone_can_mute": False}},
format="json",
)
assert response.status_code == 200
# The keys missing from the payload are dropped, not kept.
assert response.data["configuration"] == {"everyone_can_mute": False}
room.refresh_from_db()
assert room.configuration == {"everyone_can_mute": False}
mock_update_metadata.assert_called_once_with(
room_name=str(room.id),
metadata={
"configuration": {"everyone_can_mute": False},
"access_level": room.access_level,
},
)
@mock.patch.object(RoomManagement, "update_metadata")
def test_api_rooms_update_administrator_success(mock_update_metadata):
"""An administrator should be able to update a room."""
user = UserFactory()
room = RoomFactory(
users=[(user, RoleChoices.ADMIN)],
access_level=RoomAccessLevel.TRUSTED,
)
token = generate_test_token(user, [ApplicationScope.ROOMS_UPDATE])
client = APIClient()
client.credentials(HTTP_AUTHORIZATION=f"Bearer {token}")
response = client.patch(
f"/external-api/v1.0/rooms/{room.id}/",
{"access_level": RoomAccessLevel.RESTRICTED},
format="json",
)
assert response.status_code == 200
room.refresh_from_db()
assert room.access_level == RoomAccessLevel.RESTRICTED
mock_update_metadata.assert_called_once()
@mock.patch.object(RoomManagement, "update_metadata")
def test_api_rooms_update_put_not_allowed(mock_update_metadata):
"""PUT is not exposed: full replacement is not supported, only PATCH is."""
user = UserFactory()
room = RoomFactory(
users=[(user, RoleChoices.OWNER)],
access_level=RoomAccessLevel.TRUSTED,
configuration={},
)
token = generate_test_token(user, [ApplicationScope.ROOMS_UPDATE])
client = APIClient()
client.credentials(HTTP_AUTHORIZATION=f"Bearer {token}")
response = client.put(
f"/external-api/v1.0/rooms/{room.id}/",
{
"access_level": RoomAccessLevel.RESTRICTED,
"configuration": {"everyone_can_mute": True},
},
format="json",
)
assert response.status_code == 405
room.refresh_from_db()
assert room.access_level == RoomAccessLevel.TRUSTED
assert room.configuration == {}
mock_update_metadata.assert_not_called()
@pytest.mark.parametrize("role", [RoleChoices.MEMBER, None])
@mock.patch.object(RoomManagement, "update_metadata")
def test_api_rooms_update_without_privileges(mock_update_metadata, role):
"""Members and users without any role should not be able to update a room."""
user = UserFactory()
users = [(user, role)] if role else []
room = RoomFactory(users=users, access_level=RoomAccessLevel.TRUSTED)
token = generate_test_token(user, [ApplicationScope.ROOMS_UPDATE])
client = APIClient()
client.credentials(HTTP_AUTHORIZATION=f"Bearer {token}")
response = client.patch(
f"/external-api/v1.0/rooms/{room.id}/",
{"access_level": RoomAccessLevel.RESTRICTED},
format="json",
)
assert response.status_code == 403
room.refresh_from_db()
assert room.access_level == RoomAccessLevel.TRUSTED
mock_update_metadata.assert_not_called()
@mock.patch.object(RoomManagement, "update_metadata")
def test_api_rooms_update_readonly_enforcement(mock_update_metadata):
"""Read-only fields provided on update should be ignored, the slug stays immutable."""
user = UserFactory()
room = RoomFactory(
users=[(user, RoleChoices.OWNER)],
access_level=RoomAccessLevel.TRUSTED,
)
expected_id, expected_name = str(room.id), room.name
expected_slug, expected_pin_code = room.slug, room.pin_code
token = generate_test_token(user, [ApplicationScope.ROOMS_UPDATE])
client = APIClient()
client.credentials(HTTP_AUTHORIZATION=f"Bearer {token}")
response = client.patch(
f"/external-api/v1.0/rooms/{room.id}/",
{
"id": str(uuid.uuid4()),
"name": "fake-name",
"slug": "fake-slug",
"pin_code": "000000",
"access_level": RoomAccessLevel.RESTRICTED,
},
format="json",
)
assert response.status_code == 200
assert response.data["id"] == expected_id
assert response.data["name"] == expected_name
assert response.data["slug"] == expected_slug
room.refresh_from_db()
assert str(room.id) == expected_id
assert room.name == expected_name
assert room.slug == expected_slug
assert room.pin_code == expected_pin_code
# The one writable field in the payload was applied
assert room.access_level == RoomAccessLevel.RESTRICTED
mock_update_metadata.assert_called_once()
@mock.patch.object(RoomManagement, "update_metadata")
def test_api_rooms_update_rejects_invalid_configuration(mock_update_metadata):
"""Updating a room with unsupported configuration keys should fail."""
user = UserFactory()
room = RoomFactory(users=[(user, RoleChoices.OWNER)], configuration={})
token = generate_test_token(user, [ApplicationScope.ROOMS_UPDATE])
client = APIClient()
client.credentials(HTTP_AUTHORIZATION=f"Bearer {token}")
response = client.patch(
f"/external-api/v1.0/rooms/{room.id}/",
{"configuration": {"unsupported_flag": True}},
format="json",
)
assert response.status_code == 400
assert "extra inputs are not permitted" in str(response.data).lower()
room.refresh_from_db()
assert room.configuration == {}
mock_update_metadata.assert_not_called()
@pytest.mark.parametrize(
"invalid_configuration",
[
{"can_publish_sources": ["invalid-source"]},
{"everyone_can_mute": "invalid-value"},
],
)
@mock.patch.object(RoomManagement, "update_metadata")
def test_api_rooms_update_rejects_invalid_configuration_values(
mock_update_metadata, invalid_configuration
):
"""Updating a room with invalid configuration values should fail."""
user = UserFactory()
room = RoomFactory(users=[(user, RoleChoices.OWNER)], configuration={})
token = generate_test_token(user, [ApplicationScope.ROOMS_UPDATE])
client = APIClient()
client.credentials(HTTP_AUTHORIZATION=f"Bearer {token}")
response = client.patch(
f"/external-api/v1.0/rooms/{room.id}/",
{"configuration": invalid_configuration},
format="json",
)
assert response.status_code == 400
room.refresh_from_db()
assert room.configuration == {}
mock_update_metadata.assert_not_called()
@mock.patch.object(RoomManagement, "update_metadata")
def test_api_rooms_update_public_access_disabled_by_default(mock_update_metadata):
"""Switching a room to public should be disabled for the external API by default."""
user = UserFactory()
room = RoomFactory(
users=[(user, RoleChoices.OWNER)],
access_level=RoomAccessLevel.TRUSTED,
)
token = generate_test_token(user, [ApplicationScope.ROOMS_UPDATE])
client = APIClient()
client.credentials(HTTP_AUTHORIZATION=f"Bearer {token}")
response = client.patch(
f"/external-api/v1.0/rooms/{room.id}/",
{"access_level": RoomAccessLevel.PUBLIC},
format="json",
)
assert response.status_code == 400
assert "public rooms are disabled" in str(response.data).lower()
room.refresh_from_db()
assert room.access_level == RoomAccessLevel.TRUSTED
mock_update_metadata.assert_not_called()
@mock.patch.object(RoomManagement, "update_metadata")
def test_api_rooms_update_public_access_enabled_with_settings(
mock_update_metadata, settings
):
"""Switching a room to public should be allowed when explicitly enabled."""
settings.EXTERNAL_API_ALLOW_PUBLIC_ACCESS = True
user = UserFactory()
room = RoomFactory(
users=[(user, RoleChoices.OWNER)],
access_level=RoomAccessLevel.TRUSTED,
)
token = generate_test_token(user, [ApplicationScope.ROOMS_UPDATE])
client = APIClient()
client.credentials(HTTP_AUTHORIZATION=f"Bearer {token}")
response = client.patch(
f"/external-api/v1.0/rooms/{room.id}/",
{"access_level": RoomAccessLevel.PUBLIC},
format="json",
)
assert response.status_code == 200
assert response.data["access_level"] == RoomAccessLevel.PUBLIC
room.refresh_from_db()
assert room.access_level == RoomAccessLevel.PUBLIC
mock_update_metadata.assert_called_once()
@mock.patch("core.external_api.viewsets.analytics.capture")
@mock.patch.object(RoomManagement, "update_metadata")
def test_api_rooms_update_unchanged_skips_livekit_sync(
mock_update_metadata, mock_capture
):
"""An update that changes nothing should not sync metadata nor report changes."""
user = UserFactory()
room = RoomFactory(
users=[(user, RoleChoices.OWNER)],
access_level=RoomAccessLevel.TRUSTED,
configuration={"everyone_can_mute": True},
)
token = generate_test_token(user, [ApplicationScope.ROOMS_UPDATE])
client = APIClient()
client.credentials(HTTP_AUTHORIZATION=f"Bearer {token}")
response = client.patch(
f"/external-api/v1.0/rooms/{room.id}/",
{
"access_level": RoomAccessLevel.TRUSTED,
"configuration": {"everyone_can_mute": True},
},
format="json",
)
assert response.status_code == 200
mock_update_metadata.assert_not_called()
# The event is still emitted for auditing, but reports an empty delta.
_, _, properties = mock_capture.call_args[0]
assert properties["updated_fields"] == []
@mock.patch("core.external_api.viewsets.analytics.capture")
@mock.patch.object(RoomManagement, "update_metadata")
def test_api_rooms_update_tracks_analytics(mock_update_metadata, mock_capture):
"""Updating a room should emit a ROOM_UPDATED analytics event."""
user = UserFactory()
room = RoomFactory(
users=[(user, RoleChoices.OWNER)],
access_level=RoomAccessLevel.TRUSTED,
configuration={},
)
token = generate_test_token(user, [ApplicationScope.ROOMS_UPDATE])
application = Application.objects.get()
client = APIClient()
client.credentials(HTTP_AUTHORIZATION=f"Bearer {token}")
response = client.patch(
f"/external-api/v1.0/rooms/{room.id}/",
{
"access_level": RoomAccessLevel.RESTRICTED,
"configuration": {"everyone_can_mute": True},
},
format="json",
)
assert response.status_code == 200
mock_capture.assert_called_once()
captured_user, event, properties = mock_capture.call_args[0]
assert captured_user == user
assert event == AnalyticsEvent.ROOM_UPDATED
assert properties == {
"room_id": str(room.pk),
"access_level": RoomAccessLevel.RESTRICTED,
"updated_fields": ["access_level", "configuration"],
"previous_access_level": RoomAccessLevel.TRUSTED,
"client_id": str(application.client_id),
"external_api": True,
"auth_method": "ApplicationJWTAuthentication",
"$set": {"email": user.email},
}
mock_update_metadata.assert_called_once()
def test_api_rooms_response_no_url(settings):
"""Response should not include url field when APPLICATION_BASE_URL is None."""
settings.APPLICATION_BASE_URL = None
@@ -2009,106 +1497,6 @@ def test_resource_server_denies_access_with_insufficient_scopes(settings):
assert response.status_code == 403
@responses.activate
@mock.patch.object(RoomManagement, "update_metadata")
def test_resource_server_updates_room_with_prefixed_scope(
mock_update_metadata, settings
):
"""A resource server token carrying the prefixed update scope should be accepted."""
user = UserFactory(sub="very-specific-sub")
room = RoomFactory(
users=[(user, RoleChoices.OWNER)],
access_level=RoomAccessLevel.TRUSTED,
)
settings.OIDC_RS_CLIENT_ID = "some_client_id"
settings.OIDC_RS_CLIENT_SECRET = "some_client_secret"
settings.OIDC_RS_SCOPES_PREFIX = "lasuite_meet"
settings.OIDC_OP_URL = "https://oidc.example.com"
settings.OIDC_VERIFY_SSL = False
settings.OIDC_TIMEOUT = 5
settings.OIDC_PROXY = None
settings.OIDC_OP_JWKS_ENDPOINT = "https://oidc.example.com/jwks"
settings.OIDC_OP_INTROSPECTION_ENDPOINT = "https://oidc.example.com/introspect"
responses.add(
responses.POST,
"https://oidc.example.com/introspect",
json={
"iss": "https://oidc.example.com",
"aud": "some_client_id", # settings.OIDC_RS_CLIENT_ID
"sub": "very-specific-sub",
"client_id": "some_service_provider",
"scope": "openid lasuite_meet lasuite_meet:rooms:update",
"active": True,
},
)
client = APIClient()
client.credentials(HTTP_AUTHORIZATION="Bearer some_token")
response = client.patch(
f"/external-api/v1.0/rooms/{room.id}/",
{"access_level": RoomAccessLevel.RESTRICTED},
format="json",
)
assert response.status_code == 200
room.refresh_from_db()
assert room.access_level == RoomAccessLevel.RESTRICTED
mock_update_metadata.assert_called_once()
@responses.activate
def test_resource_server_denies_room_update_without_update_scope(settings):
"""A resource server token without the update scope should be denied."""
user = UserFactory(sub="very-specific-sub")
room = RoomFactory(
users=[(user, RoleChoices.OWNER)],
access_level=RoomAccessLevel.TRUSTED,
)
settings.OIDC_RS_CLIENT_ID = "some_client_id"
settings.OIDC_RS_CLIENT_SECRET = "some_client_secret"
settings.OIDC_RS_SCOPES_PREFIX = "lasuite_meet"
settings.OIDC_OP_URL = "https://oidc.example.com"
settings.OIDC_VERIFY_SSL = False
settings.OIDC_TIMEOUT = 5
settings.OIDC_PROXY = None
settings.OIDC_OP_JWKS_ENDPOINT = "https://oidc.example.com/jwks"
settings.OIDC_OP_INTROSPECTION_ENDPOINT = "https://oidc.example.com/introspect"
responses.add(
responses.POST,
"https://oidc.example.com/introspect",
json={
"iss": "https://oidc.example.com",
"aud": "some_client_id", # settings.OIDC_RS_CLIENT_ID
"sub": "very-specific-sub",
"client_id": "some_service_provider",
"scope": "openid lasuite_meet lasuite_meet:rooms:retrieve",
"active": True,
},
)
client = APIClient()
client.credentials(HTTP_AUTHORIZATION="Bearer some_token")
response = client.patch(
f"/external-api/v1.0/rooms/{room.id}/",
{"access_level": RoomAccessLevel.RESTRICTED},
format="json",
)
assert response.status_code == 403
room.refresh_from_db()
assert room.access_level == RoomAccessLevel.TRUSTED
# ==============================
# Addons
# ==============================
@@ -2160,32 +1548,6 @@ def test_api_rooms_create_with_valid_addons_token():
assert room.get_role(user) == RoleChoices.OWNER
@mock.patch.object(RoomManagement, "update_metadata")
def test_api_rooms_update_with_valid_addons_token(mock_update_metadata):
"""Updating a room with a valid addons token should succeed."""
user = UserFactory()
room = RoomFactory(
users=[(user, RoleChoices.OWNER)],
access_level=RoomAccessLevel.TRUSTED,
)
token = generate_addons_test_token(user, [ApplicationScope.ROOMS_UPDATE])
client = APIClient()
client.credentials(HTTP_AUTHORIZATION=f"Bearer {token}")
response = client.patch(
f"/external-api/v1.0/rooms/{room.id}/",
{"access_level": RoomAccessLevel.RESTRICTED},
format="json",
)
assert response.status_code == 200
room.refresh_from_db()
assert room.access_level == RoomAccessLevel.RESTRICTED
mock_update_metadata.assert_called_once()
def test_api_rooms_addons_token_inactive_user():
"""Addons token for an inactive user should return 401."""
user = UserFactory(is_active=False)
@@ -8,6 +8,28 @@ const IGNORED_EXCEPTION_PATTERNS = [
// the close reason is already logged by the SDK.
// See: https://github.com/livekit/client-sdk-js/issues/2062
/^Event captured as exception with keys: isTrusted$/,
// MediaPipe's WASM writes its native logs to stderr, which Emscripten
// routes to console.error, which PostHog's console capture then promotes
// to an $exception — even though nothing was thrown. Two flavors:
//
// 1. "INFO: ..." lines are purely informational. In particular
// "INFO: Created TensorFlow Lite XNNPACK delegate for CPU." is a
// SUCCESS message: TFLite prints it when it lazily initializes CPU
// inference on the first segmented frame. It fires on every effects
// init, on every browser and delegate (the GPU delegate still
// instantiates the CPU/XNNPACK delegate for non-delegated ops), so it
// was our single noisiest "error" while carrying zero signal.
/^INFO: /,
//
// 2. absl-formatted log lines, e.g.
// "E0901 19:21:45.443000 1880752 gl_graph_runner_internal.cc:260]
// StartGraph failed: ..."
// (severity letter E/W/I/F + MMDD + timestamp). These are the stderr
// *copies* of failures that MediaPipe also raises as real JS
// exceptions, which we already capture via reportError / thrown
// errors. Dropping them de-duplicates each incident (previously
// counted 2-3x) without losing the actual error report.
/^[EWIF]\d{4} \d{2}:\d{2}:\d{2}\./,
]
const shouldIgnoreException = (value: unknown): boolean =>
@@ -17,7 +17,7 @@ import {
type ProcessorType,
MEDIAPIPE_PATH_WASM,
} from '.'
import { captureEvent } from '@/features/analytics/telemetry.ts'
import { captureEvent, reportError } from '@/features/analytics/telemetry'
const PROCESSING_WIDTH = 256
const PROCESSING_HEIGHT = 144
@@ -26,6 +26,40 @@ const SEGMENTATION_MASK_CANVAS_ID = 'background-blur-local-segmentation'
const BLUR_CANVAS_ID = 'background-blur-local'
const DEFAULT_BLUR = '10'
const CONCEALING_BLUR = '25'
const FRAME_INTERVAL_MS = 1000 / 30
// After this many consecutive failed frames, stop segmenting and fall back to
// publishing a fully blurred frame: the user keeps a live camera instead of a
// frozen one, without ever exposing the surroundings they chose to conceal.
const MAX_CONSECUTIVE_ERRORS = 5
let webgl2Supported: boolean | undefined
/**
* MediaPipe's ImageSegmenter requires a WebGL2 context on the web even with
* `delegate: 'CPU'` (only inference runs on CPU; the mask post-processing in
* TensorsToSegmentationCalculator is GL-based). Without this check, machines
* with WebGL disabled or blocklisted fail at StartGraph with
* `emscripten_webgl_create_context() returned error 0`.
*
* The result is cached and the probe context is explicitly released so that
* repeated support checks do not count against the browser's limit on live
* WebGL contexts.
*/
const isWebGL2Supported = () => {
if (webgl2Supported === undefined) {
try {
const canvas = document.createElement('canvas')
const gl = canvas.getContext('webgl2')
webgl2Supported = !!gl
gl?.getExtension('WEBGL_lose_context')?.loseContext()
} catch {
webgl2Supported = false
}
}
return webgl2Supported
}
/**
* This implementation of video blurring is made to be run on CPU for browser that are
@@ -42,14 +76,12 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
source?: MediaStreamTrack
sourceSettings?: MediaTrackSettings
videoElement?: HTMLVideoElement
videoElementLoaded?: boolean
// Canvas containing the video processing result, of which we extract as stream.
outputCanvas?: HTMLCanvasElement
outputCanvasCtx?: CanvasRenderingContext2D
imageSegmenter?: ImageSegmenter
imageSegmenterResult?: ImageSegmenterResult
// Canvas used for resizing video source and projecting mask.
segmentationMaskCanvas?: HTMLCanvasElement
@@ -66,6 +98,13 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
type: ProcessorType
virtualBackgroundImage?: HTMLImageElement
private virtualBackgroundImagePath?: string
private destroyed = false
private degraded = false
private consecutiveErrors = 0
private processing?: Promise<void>
private onVideoLoaded?: () => void
constructor(opts: ProcessorConfig) {
this.name = 'blur'
this.options = opts
@@ -73,7 +112,10 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
}
static get isSupported() {
return navigator.userAgent.toLowerCase().includes('firefox')
return (
navigator.userAgent.toLowerCase().includes('firefox') &&
isWebGL2Supported()
)
}
async init(opts: ProcessorOptions<Track.Kind>) {
@@ -81,6 +123,10 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
throw new Error('Element is required for processing')
}
this.destroyed = false
this.degraded = false
this.consecutiveErrors = 0
this.source = opts.track as MediaStreamTrack
this.sourceSettings = this.source!.getSettings()
this.videoElement = opts.element as HTMLVideoElement
@@ -97,26 +143,56 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
this.processedTrack = tracks[0]
this.segmentationMask = new ImageData(PROCESSING_WIDTH, PROCESSING_HEIGHT)
const t0 = performance.now()
await this.initSegmenter()
const segmenterInitMs = Math.round(performance.now() - t0)
this._initWorker()
captureEvent('firefox-blurring-init', {})
captureEvent('legacy-background-processor', {
effect_type: this.options.type,
hw_concurrency: navigator.hardwareConcurrency,
video_width: this.videoElement?.videoWidth,
video_height: this.videoElement?.videoHeight,
segmenter_init_ms: segmenterInitMs,
})
}
_initVirtualBackgroundImage() {
if (this.options.type !== 'virtual') {
if (this.options.type !== 'virtual' || !this.options.imagePath) {
return
}
const needsUpdate =
this.options.imagePath &&
if (
this.virtualBackgroundImage &&
this.virtualBackgroundImage.src !== this.options.imagePath
if (this.options.imagePath || needsUpdate) {
this.virtualBackgroundImage = document.createElement('img')
this.virtualBackgroundImage.crossOrigin = 'anonymous'
this.virtualBackgroundImage.src = this.options.imagePath!
this.virtualBackgroundImagePath === this.options.imagePath
) {
return
}
const image = document.createElement('img')
image.crossOrigin = 'anonymous'
image.src = this.options.imagePath
// Surface load failures once instead of letting drawImage throw on a
// broken image inside the processing loop.
image.decode().catch((error) => {
reportError('effects_processor_failure', error, {
context: 'Failed to load virtual background image',
image_path:
this.options.type === 'virtual' ? this.options.imagePath : undefined,
})
})
this.virtualBackgroundImage = image
this.virtualBackgroundImagePath = this.options.imagePath
}
_isVirtualBackgroundImageReady() {
return (
!!this.virtualBackgroundImage &&
this.virtualBackgroundImage.complete &&
this.virtualBackgroundImage.naturalWidth > 0
)
}
async update(opts: ProcessorConfig): Promise<void> {
@@ -129,26 +205,26 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
name: 'Blurring',
})
this.timerWorker.onmessage = (data) => this.onTimerMessage(data)
// When hiding camera then showing it again, the onloadeddata callback is not fired again.
if (this.videoElementLoaded) {
this.timerWorker!.postMessage({
id: SET_TIMEOUT,
timeMs: 1000 / 30,
})
const startLoop = () => {
this.onVideoLoaded = undefined
this._syncOutputCanvasSize()
this._scheduleNextFrame()
}
if (this.videoElement!.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) {
startLoop()
} else {
this.videoElement!.onloadeddata = () => {
this.videoElementLoaded = true
this.timerWorker!.postMessage({
id: SET_TIMEOUT,
timeMs: 1000 / 30,
})
}
this.onVideoLoaded = startLoop
this.videoElement!.addEventListener('loadeddata', this.onVideoLoaded, {
once: true,
})
}
}
onTimerMessage(response: { data: { id: number } }) {
if (response.data.id === TIMEOUT_TICK) {
this.process()
this.processing = this.process()
}
}
@@ -194,30 +270,47 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
*/
async segment() {
const startTimeMs = performance.now()
return new Promise<void>((resolve) => {
this.imageSegmenter!.segmentForVideo(
this.sourceImageData!,
startTimeMs,
(result: ImageSegmenterResult) => {
this.imageSegmenterResult = result
resolve()
}
)
return new Promise<void>((resolve, reject) => {
try {
this.imageSegmenter!.segmentForVideo(
this.sourceImageData!,
startTimeMs,
(result: ImageSegmenterResult) => {
try {
// The mask is only valid for the duration of this callback:
// MediaPipe frees the underlying WASM memory as soon as it
// returns, so the data must be copied out synchronously here.
this._applyMaskToAlphaChannel(result)
resolve()
} catch (error) {
reject(error)
}
}
)
} catch (error) {
reject(error)
}
})
}
/**
* TODO: future improvement with WebGL.
*/
async blur() {
if (this.options.type !== 'blur') {
throw new Error('Blurring is only supported for blur background')
_applyMaskToAlphaChannel(result: ImageSegmenterResult) {
const categoryMask = result.categoryMask
if (!categoryMask) {
return
}
const mask = this.imageSegmenterResult!.categoryMask!.getAsUint8Array()
for (let i = 0; i < mask.length; ++i) {
this.segmentationMask!.data[i * 4 + 3] = 255 - mask[i]
const mask = categoryMask.getAsUint8Array()
const alpha = this.segmentationMask!.data
const length = Math.min(mask.length, alpha.length / 4)
for (let i = 0; i < length; ++i) {
alpha[i * 4 + 3] = 255 - mask[i]
}
}
/**
* Composite the segmentation mask over the output canvas: mask first, then
* the clear body, leaving the background to be filled by the caller.
*/
_compositeMaskAndBody() {
this.segmentationMaskCanvasCtx!.putImageData(this.segmentationMask!, 0, 0)
this.outputCanvasCtx!.globalCompositeOperation = 'copy'
@@ -240,6 +333,16 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
this.outputCanvasCtx!.globalCompositeOperation = 'source-in'
this.outputCanvasCtx!.filter = 'none'
this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0)
}
/**
* TODO: future improvement with WebGL.
*/
async blur() {
if (this.options.type !== 'blur') {
throw new Error('Blurring is only supported for blur background')
}
this._compositeMaskAndBody()
// Draw blurry background.
this.outputCanvasCtx!.globalCompositeOperation = 'destination-over'
@@ -251,87 +354,150 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
* TODO: future improvement with WebGL.
*/
async drawVirtualBackground() {
const mask = this.imageSegmenterResult!.categoryMask!.getAsUint8Array()
for (let i = 0; i < mask.length; ++i) {
this.segmentationMask!.data[i * 4 + 3] = 255 - mask[i]
}
this._compositeMaskAndBody()
this.segmentationMaskCanvasCtx!.putImageData(this.segmentationMask!, 0, 0)
this.outputCanvasCtx!.globalCompositeOperation = 'copy'
this.outputCanvasCtx!.filter = 'blur(8px)'
// Put opacity mask.
this.outputCanvasCtx!.drawImage(
this.segmentationMaskCanvas!,
0,
0,
PROCESSING_WIDTH,
PROCESSING_HEIGHT,
0,
0,
this.videoElement!.videoWidth,
this.videoElement!.videoHeight
)
// Draw clear body.
this.outputCanvasCtx!.globalCompositeOperation = 'source-in'
this.outputCanvasCtx!.filter = 'none'
this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0)
// Draw virtual background.
this.outputCanvasCtx!.globalCompositeOperation = 'destination-over'
this.outputCanvasCtx!.drawImage(
this.virtualBackgroundImage!,
0,
0,
this.outputCanvas!.width,
this.outputCanvas!.height
)
this.outputCanvasCtx!.filter = 'none'
if (this._isVirtualBackgroundImageReady()) {
// Draw virtual background.
this.outputCanvasCtx!.drawImage(
this.virtualBackgroundImage!,
0,
0,
this.outputCanvas!.width,
this.outputCanvas!.height
)
} else {
// Image not decoded (yet, or failed to load): fill the background with
// a heavy blur instead. Never fall back to the raw video here — the
// user selected this effect to conceal their surroundings, so the
// fallback must keep concealing them.
this.outputCanvasCtx!.filter = `blur(${CONCEALING_BLUR}px)`
this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0)
}
}
/**
* Draw the whole frame heavily blurred (person included). Used when
* segmentation is broken: the outgoing video keeps flowing instead of
* freezing on a stale frame, while the surroundings the user chose to
* conceal stay concealed. Requires no segmenter, only one filtered draw.
*/
_drawDegradedFrame() {
this.outputCanvasCtx!.globalCompositeOperation = 'copy'
this.outputCanvasCtx!.filter = `blur(${CONCEALING_BLUR}px)`
this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0)
this.outputCanvasCtx!.filter = 'none'
}
async process() {
await this.sizeSource()
await this.segment()
if (this.options.type === 'blur') {
await this.blur()
} else {
await this.drawVirtualBackground()
if (this.destroyed) {
return
}
this.timerWorker!.postMessage({
try {
this._syncOutputCanvasSize()
// No decoded frame available (e.g. right after a device switch): skip
// this tick rather than processing a 0x0 source.
if (
!this.videoElement ||
this.videoElement.videoWidth === 0 ||
this.videoElement.videoHeight === 0
) {
this._scheduleNextFrame()
return
}
if (this.degraded) {
this._drawDegradedFrame()
this._scheduleNextFrame()
return
}
await this.sizeSource()
await this.segment()
if (this.destroyed) {
return
}
if (this.options.type === 'blur') {
await this.blur()
} else {
await this.drawVirtualBackground()
}
this.consecutiveErrors = 0
} catch (error) {
if (this.destroyed) {
return
}
this.consecutiveErrors += 1
if (this.consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) {
// Degrade to a fully blurred frame: a live camera beats a frozen
// one, and concealment must survive the failure.
this.degraded = true
reportError('effects_processor_failure', error, {
context:
'Background processing failed repeatedly, falling back to fully blurred video',
consecutive_errors: this.consecutiveErrors,
})
this.imageSegmenter?.close()
this.imageSegmenter = undefined
}
}
this._scheduleNextFrame()
}
_scheduleNextFrame() {
if (this.destroyed) {
return
}
this.timerWorker?.postMessage({
id: SET_TIMEOUT,
timeMs: 1000 / 30,
timeMs: FRAME_INTERVAL_MS,
})
}
_createMainCanvas() {
this.outputCanvas = document.querySelector(
'canvas#background-blur-local'
) as HTMLCanvasElement
if (!this.outputCanvas) {
this.outputCanvas = this._createCanvas(
BLUR_CANVAS_ID,
this.sourceSettings!.width!,
this.sourceSettings!.height!
)
/**
* Keep the output canvas in sync with the actual decoded video dimensions.
* `MediaStreamTrack.getSettings()` can be incomplete or stale on Firefox,
* so the video element is the source of truth.
*/
_syncOutputCanvasSize() {
const width = this.videoElement?.videoWidth
const height = this.videoElement?.videoHeight
if (!width || !height || !this.outputCanvas) {
return
}
if (
this.outputCanvas.width !== width ||
this.outputCanvas.height !== height
) {
this.outputCanvas.width = width
this.outputCanvas.height = height
}
}
_createMainCanvas() {
const width =
this.sourceSettings?.width || this.videoElement?.videoWidth || 1280
const height =
this.sourceSettings?.height || this.videoElement?.videoHeight || 720
this.outputCanvas = this._createCanvas(BLUR_CANVAS_ID, width, height)
this.outputCanvasCtx = this.outputCanvas.getContext('2d')!
}
_createMaskCanvas() {
this.segmentationMaskCanvas = document.querySelector(
`#${SEGMENTATION_MASK_CANVAS_ID}`
) as HTMLCanvasElement
if (!this.segmentationMaskCanvas) {
this.segmentationMaskCanvas = this._createCanvas(
SEGMENTATION_MASK_CANVAS_ID,
PROCESSING_WIDTH,
PROCESSING_HEIGHT
)
}
this.segmentationMaskCanvasCtx =
this.segmentationMaskCanvas.getContext('2d')!
this.segmentationMaskCanvas = this._createCanvas(
SEGMENTATION_MASK_CANVAS_ID,
PROCESSING_WIDTH,
PROCESSING_HEIGHT
)
// getImageData is called on this canvas 30 times per second: opt out of
// GPU backing to avoid a costly readback on every frame.
this.segmentationMaskCanvasCtx = this.segmentationMaskCanvas.getContext(
'2d',
{ willReadFrequently: true }
)!
}
_createCanvas(id: string, width: number, height: number) {
@@ -348,11 +514,39 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
}
async destroy() {
this.destroyed = true
this.timerWorker?.postMessage({
id: CLEAR_TIMEOUT,
})
// Let any in-flight frame finish before releasing the resources it uses,
// so segmentForVideo is never called on a closed segmenter.
try {
await this.processing
} catch {
// Failures are already handled inside process().
}
this.processing = undefined
if (this.onVideoLoaded && this.videoElement) {
this.videoElement.removeEventListener('loadeddata', this.onVideoLoaded)
}
this.onVideoLoaded = undefined
this.timerWorker?.terminate()
this.timerWorker = undefined
this.imageSegmenter?.close()
this.imageSegmenter = undefined
this.processedTrack?.stop()
this.processedTrack = undefined
this.outputCanvas = undefined
this.outputCanvasCtx = undefined
this.segmentationMaskCanvas = undefined
this.segmentationMaskCanvasCtx = undefined
this.sourceImageData = undefined
}
}
@@ -6,6 +6,7 @@ import type { Track, TrackProcessor } from 'livekit-client'
import { BackgroundCustomProcessor } from './BackgroundCustomProcessor'
import { UnifiedBackgroundTrackProcessor } from './UnifiedBackgroundTrackProcessor'
import { FaceLandmarksOptions } from './FaceLandmarksProcessor'
import { captureEvent } from '@/features/analytics/telemetry'
export const SELFIE_SEGMENTER_MODEL_PATH =
'/assets/mediapipe/models/selfie_segmenter_landscape.tflite'
@@ -31,15 +32,29 @@ export interface BackgroundProcessorInterface extends TrackProcessor<Track.Kind>
options: ProcessorConfig
}
let unsupportedReported = false
export class BackgroundProcessorFactory {
private static _isSupported?: boolean
static hasModernApiSupport() {
return ProcessorWrapper.hasModernApiSupport
}
static isSupported() {
return (
supportsBackgroundProcessors() || BackgroundCustomProcessor.isSupported
)
if (this._isSupported === undefined) {
this._isSupported =
supportsBackgroundProcessors() || BackgroundCustomProcessor.isSupported
}
if (!this._isSupported && !unsupportedReported) {
unsupportedReported = true
captureEvent('background-processor-unsupported', {
path: 'isSupported',
})
}
return this._isSupported
}
static getProcessor(
@@ -48,6 +63,8 @@ export class BackgroundProcessorFactory {
const isBlur = config.type === ProcessorType.BLUR
const isVirtual = config.type === ProcessorType.VIRTUAL
return new BackgroundCustomProcessor(config)
if (!isBlur && !isVirtual) return undefined
if (supportsBackgroundProcessors()) {
@@ -58,6 +75,12 @@ export class BackgroundProcessorFactory {
return new BackgroundCustomProcessor(config)
}
if (!unsupportedReported) {
captureEvent('background-processor-unsupported', {
path: 'getProcessor',
})
}
return undefined
}
@@ -25,7 +25,11 @@ import {
} from '@/features/files/api/listFiles.ts'
import { useCreateFile } from '@/features/files/api/createFile.ts'
import { FileTrigger } from 'react-aria-components'
import { RiDeleteBinLine, RiImageAddFill } from '@remixicon/react'
import {
RiDeleteBinLine,
RiImageAddFill,
RiProhibitedLine,
} from '@remixicon/react'
import { useDeleteFile } from '@/features/files/api/deleteFile.ts'
import { useUser } from '@/features/auth/api/useUser'
import { ApiFileItem } from '@/features/files/api/types.ts'
@@ -197,10 +201,23 @@ export const EffectsConfiguration = ({
*
* We arrive in this condition when we enter the room with the camera already off.
*/
const newProcessorTmp = BackgroundProcessorFactory.getProcessor(config)!
await toggle(true, {
processor: newProcessorTmp,
})
try {
const newProcessorTmp =
BackgroundProcessorFactory.getProcessor(config)!
await toggle(true, {
processor: newProcessorTmp,
})
} catch (error) {
reportError('effects_processor_failure', error, {
context: 'Error applying effect while enabling camera:',
})
saveProcessorConfig(undefined)
try {
await toggle(true)
} catch {
// Camera errors are handled by the toggle's own error path.
}
}
setTimeout(() => setProcessorPending(false))
return
}
@@ -242,6 +259,14 @@ export const EffectsConfiguration = ({
reportError('effects_processor_failure', error, {
context: 'Error applying effect:',
})
try {
if (videoTrack.getProcessor()) {
await videoTrack.stopProcessor()
}
} catch {
// Best effort: the processor may already be broken.
}
saveProcessorConfig(undefined)
} finally {
// Without setTimeout the DOM is not refreshing when updating the options.
setTimeout(() => setProcessorPending(false))
@@ -250,6 +275,24 @@ export const EffectsConfiguration = ({
[enabled, selectedId, toggle, updateEffectStatusMessage, videoTrack]
)
const clearEffect = useCallback(async () => {
if (selectedId === 'none') return
setProcessorPending(true)
try {
if (videoTrack?.getProcessor()) {
await videoTrack.stopProcessor()
}
saveProcessorConfig(undefined)
announceEffectStatusMessage(t('blur.status.none'))
} catch (error) {
reportError('effects_processor_failure', error, {
context: 'Error clearing effect:',
})
} finally {
setTimeout(() => setProcessorPending(false))
}
}, [announceEffectStatusMessage, selectedId, t, videoTrack])
const { data: appConfig } = useConfig()
const { isLoggedIn } = useUser()
const canUploadBackground =
@@ -647,6 +690,17 @@ export const EffectsConfiguration = ({
gap: '1.25rem',
})}
>
<ToggleButton
variant="bigSquare"
aria-label={t('clear')}
tooltip={t('clear')}
isDisabled={processorOptions.isDisabled}
onChange={clearEffect}
isSelected={selectedId === 'none'}
data-attr="toggle-effect-none"
>
<RiProhibitedLine />
</ToggleButton>
{processorOptions.blurBased.map(({ Icon, ...option }) => (
<ToggleButton
key={option.id}
@@ -22,10 +22,12 @@ import { VOICE_AUDIO_CONSTRAINTS } from '../utils/constants'
import {
saveAudioInputDeviceId,
saveAudioInputEnabled,
saveProcessorConfig,
saveVideoInputDeviceId,
saveVideoInputEnabled,
userChoicesStore,
} from '@/stores/userChoices'
import { reportError } from '@/features/analytics/telemetry'
import { useSyncTrackDeviceId } from './useSyncTrackDeviceId'
// Module-level: effect dependencies, must be referentially stable.
@@ -221,15 +223,32 @@ export function useJoinTracks(): {
[audioDeviceId]
)
const createVideo = useCallback(
() =>
createLocalVideoTrack({
const createVideo = useCallback(async () => {
const processor =
BackgroundProcessorFactory.fromProcessorConfig(processorConfig)
if (!processor) {
return createLocalVideoTrack({ deviceId: videoDeviceId })
}
try {
return await createLocalVideoTrack({
deviceId: videoDeviceId,
processor:
BackgroundProcessorFactory.fromProcessorConfig(processorConfig),
}),
[videoDeviceId, processorConfig]
)
processor,
})
} catch (error) {
// A camera problem (permission, device missing/busy) is not the
// effect's fault: let the normal media error handling deal with it
// without touching the user's saved effect.
const e = getMediaDeviceFailure(error as Error)
if (e !== MediaDeviceFailure.Other && !!e) {
throw error
}
reportError('effects_processor_failure', error, {
context: 'Restoring saved effect failed, retrying without it',
})
saveProcessorConfig(undefined)
return createLocalVideoTrack({ deviceId: videoDeviceId })
}
}, [videoDeviceId, processorConfig])
const audioTrack = useLocalTrack({
ready: audioReady,