Compare commits

..

1 Commits

Author SHA1 Message Date
lebaudantoine 7e8e98d737 🩹(changelog) fix changelog entry ordering
Restore the correct order of entries in the CHANGELOG, which got
shuffled somewhere between rebases.
2026-09-01 22:07:38 +02:00
20 changed files with 164 additions and 638 deletions
-9
View File
@@ -8,15 +8,6 @@ and this project adheres to
## [Unreleased]
### Added
- ✨(frontend) add 1080p sending resolution option #1660
- ✨(backend) add Traefik support via configurable media-auth url header #1649
### Fixed
- 🐛(frontend) keep the sending resolution picked while the camera is off #1667
## [1.30.0] - 2026-09-01
### Added
-1
View File
@@ -65,7 +65,6 @@ RUN apk update && apk upgrade \
musl \
musl-utils \
zlib>=1.3.2-r0 \
libexpat>=2.8.4-r0 \
&& apk del curl
USER nginx
+7 -17
View File
@@ -1076,10 +1076,9 @@ class RecordingViewSet(
def _auth_get_original_url(self, request):
"""
Extracts and parses the original URL from the configured header.
Extracts and parses the original URL from the "HTTP_X_ORIGINAL_URL" header.
Raises PermissionDenied if the header is missing.
The original url is passed by the reverse proxy in the header named by the
MEDIA_AUTH_ORIGINAL_URL_HEADER setting, which defaults to "HTTP_X_ORIGINAL_URL".
The original url is passed by nginx in the "HTTP_X_ORIGINAL_URL" header.
See corresponding ingress configuration in Helm chart and read about the
nginx.ingress.kubernetes.io/auth-url annotation to understand how the Nginx ingress
is configured to do this.
@@ -1089,13 +1088,9 @@ class RecordingViewSet(
reasons.
"""
# Extract the original URL from the request header
original_url = request.META.get(settings.MEDIA_AUTH_ORIGINAL_URL_HEADER)
original_url = request.META.get("HTTP_X_ORIGINAL_URL")
if not original_url:
logger.warning(
"Missing %s header in subrequest. Set MEDIA_AUTH_ORIGINAL_URL_HEADER "
"to the header your reverse proxy sends.",
settings.MEDIA_AUTH_ORIGINAL_URL_HEADER,
)
logger.warning("Missing HTTP_X_ORIGINAL_URL header in subrequest")
raise drf_exceptions.PermissionDenied()
logger.debug("Original url: '%s'", original_url)
@@ -1420,8 +1415,7 @@ class FileViewSet(
Authorize access based on the original URL of an Nginx subrequest
and user permissions. Returns a dictionary of URL parameters if authorized.
The original url is passed by the reverse proxy in the header named by the
MEDIA_AUTH_ORIGINAL_URL_HEADER setting, which defaults to "HTTP_X_ORIGINAL_URL".
The original url is passed by nginx in the "HTTP_X_ORIGINAL_URL" header.
See corresponding ingress configuration in Helm chart and read about the
nginx.ingress.kubernetes.io/auth-url annotation to understand how the Nginx ingress
is configured to do this.
@@ -1440,13 +1434,9 @@ class FileViewSet(
- PermissionDenied if authorization fails.
"""
# Extract the original URL from the request header
original_url = request.META.get(settings.MEDIA_AUTH_ORIGINAL_URL_HEADER)
original_url = request.META.get("HTTP_X_ORIGINAL_URL")
if not original_url:
logger.warning(
"Missing %s header in subrequest. Set MEDIA_AUTH_ORIGINAL_URL_HEADER "
"to the header your reverse proxy sends.",
settings.MEDIA_AUTH_ORIGINAL_URL_HEADER,
)
logger.warning("Missing HTTP_X_ORIGINAL_URL header in subrequest")
raise drf_exceptions.PermissionDenied()
parsed_url = urlparse(original_url)
@@ -7,7 +7,6 @@ from urllib.parse import quote, urlparse
from django.conf import settings
from django.core.files.storage import default_storage
from django.test import override_settings
from django.utils import timezone
import pytest
@@ -144,59 +143,3 @@ def test_api_files_media_auth_own_file_deleted():
)
assert response.status_code == 403
@override_settings(MEDIA_AUTH_ORIGINAL_URL_HEADER="HTTP_X_FORWARDED_URI")
def test_api_files_media_auth_custom_original_url_header():
"""
Authorization should honour the configured original-url header.
Covers the attachment subrequest path, which resolves the header separately
from the recording one. Reverse proxies other than nginx-ingress use
different headers: Traefik's ForwardAuth sends X-Forwarded-Uri and cannot
emit X-Original-URL at all.
"""
user = factories.UserFactory()
file = factories.FileFactory(
type=models.FileTypeChoices.BACKGROUND_IMAGE,
update_upload_state=models.FileUploadStateChoices.READY,
creator=user,
)
client = APIClient()
client.force_login(user)
default_storage.save(file.file_key, BytesIO(b"my prose"))
original_url = f"http://localhost/media/{file.file_key:s}"
response = client.get(
"/api/v1.0/files/media-auth/", HTTP_X_FORWARDED_URI=original_url
)
assert response.status_code == 200
assert "AWS4-HMAC-SHA256 Credential=" in response["Authorization"]
@override_settings(MEDIA_AUTH_ORIGINAL_URL_HEADER="HTTP_X_FORWARDED_URI")
def test_api_files_media_auth_default_header_ignored_when_reconfigured():
"""
Only the configured header should be honoured, never a hardcoded fallback.
"""
user = factories.UserFactory()
file = factories.FileFactory(
type=models.FileTypeChoices.BACKGROUND_IMAGE,
update_upload_state=models.FileUploadStateChoices.READY,
creator=user,
)
client = APIClient()
client.force_login(user)
original_url = f"http://localhost/media/{file.file_key:s}"
response = client.get(
"/api/v1.0/files/media-auth/", HTTP_X_ORIGINAL_URL=original_url
)
assert response.status_code == 403
@@ -8,7 +8,6 @@ from uuid import uuid4
from django.conf import settings
from django.core.files.storage import default_storage
from django.test import override_settings
from django.utils import timezone
import pytest
@@ -283,63 +282,3 @@ def test_api_recordings_media_auth_success_administrator(mode):
timeout=1,
)
assert response.content.decode("utf-8") == "my prose"
def test_api_recordings_media_auth_missing_header():
"""
Test that a subrequest without the configured original-url header is rejected.
"""
user = UserFactory()
client = APIClient()
client.force_login(user)
response = client.get("/api/v1.0/recordings/media-auth/")
assert response.status_code == 403
@override_settings(MEDIA_AUTH_ORIGINAL_URL_HEADER="HTTP_X_FORWARDED_URI")
def test_api_recordings_media_auth_custom_original_url_header():
"""
Test that the header carrying the original URL can be configured.
Reverse proxies other than nginx-ingress use different headers: Traefik's
ForwardAuth sends X-Forwarded-Uri and cannot emit X-Original-URL at all.
"""
user = UserFactory()
client = APIClient()
client.force_login(user)
original_url = f"http://localhost/media/recordings/{uuid4()!s}.mp4"
response = client.get(
"/api/v1.0/recordings/media-auth/", HTTP_X_FORWARDED_URI=original_url
)
# The header was read and parsed: we get as far as looking the recording up,
# rather than being rejected for a missing header.
assert response.status_code == 404
@override_settings(MEDIA_AUTH_ORIGINAL_URL_HEADER="HTTP_X_FORWARDED_URI")
def test_api_recordings_media_auth_default_header_ignored_when_reconfigured():
"""
Test that only the configured header is honoured.
Guards against the header being read from a hardcoded name in parallel with
the setting.
"""
user = UserFactory()
client = APIClient()
client.force_login(user)
original_url = f"http://localhost/media/recordings/{uuid4()!s}.mp4"
response = client.get(
"/api/v1.0/recordings/media-auth/", HTTP_X_ORIGINAL_URL=original_url
)
assert response.status_code == 403
-9
View File
@@ -129,15 +129,6 @@ class Base(Configuration):
MEDIA_BASE_URL = values.Value(
"", environ_name="MEDIA_BASE_URL", environ_prefix=None
)
# Header the reverse proxy uses to pass the original request URL to the
# media-auth subrequest views. nginx-ingress sends X-Original-URL, which is
# the default. Other proxies use different headers -- Traefik's ForwardAuth,
# for instance, sends X-Forwarded-Uri and cannot emit X-Original-URL at all.
MEDIA_AUTH_ORIGINAL_URL_HEADER = values.Value(
default="HTTP_X_ORIGINAL_URL",
environ_name="MEDIA_AUTH_ORIGINAL_URL_HEADER",
environ_prefix=None,
)
SITE_ID = 1
-1
View File
@@ -53,7 +53,6 @@ RUN apk update && apk upgrade \
musl \
musl-utils \
zlib>=1.3.2-r0 \
libexpat>=2.8.4-r0 \
&& apk del curl
USER nginx
@@ -8,28 +8,6 @@ const IGNORED_EXCEPTION_PATTERNS = [
// the close reason is already logged by the SDK.
// See: https://github.com/livekit/client-sdk-js/issues/2062
/^Event captured as exception with keys: isTrusted$/,
// MediaPipe's WASM writes its native logs to stderr, which Emscripten
// routes to console.error, which PostHog's console capture then promotes
// to an $exception — even though nothing was thrown. Two flavors:
//
// 1. "INFO: ..." lines are purely informational. In particular
// "INFO: Created TensorFlow Lite XNNPACK delegate for CPU." is a
// SUCCESS message: TFLite prints it when it lazily initializes CPU
// inference on the first segmented frame. It fires on every effects
// init, on every browser and delegate (the GPU delegate still
// instantiates the CPU/XNNPACK delegate for non-delegated ops), so it
// was our single noisiest "error" while carrying zero signal.
/^INFO: /,
//
// 2. absl-formatted log lines, e.g.
// "E0901 19:21:45.443000 1880752 gl_graph_runner_internal.cc:260]
// StartGraph failed: ..."
// (severity letter E/W/I/F + MMDD + timestamp). These are the stderr
// *copies* of failures that MediaPipe also raises as real JS
// exceptions, which we already capture via reportError / thrown
// errors. Dropping them de-duplicates each incident (previously
// counted 2-3x) without losing the actual error report.
/^[EWIF]\d{4} \d{2}:\d{2}:\d{2}\./,
]
const shouldIgnoreException = (value: unknown): boolean =>
@@ -17,7 +17,7 @@ import {
type ProcessorType,
MEDIAPIPE_PATH_WASM,
} from '.'
import { captureEvent, reportError } from '@/features/analytics/telemetry'
import { captureEvent } from '@/features/analytics/telemetry.ts'
const PROCESSING_WIDTH = 256
const PROCESSING_HEIGHT = 144
@@ -26,40 +26,6 @@ const SEGMENTATION_MASK_CANVAS_ID = 'background-blur-local-segmentation'
const BLUR_CANVAS_ID = 'background-blur-local'
const DEFAULT_BLUR = '10'
const CONCEALING_BLUR = '25'
const FRAME_INTERVAL_MS = 1000 / 30
// After this many consecutive failed frames, stop segmenting and fall back to
// publishing a fully blurred frame: the user keeps a live camera instead of a
// frozen one, without ever exposing the surroundings they chose to conceal.
const MAX_CONSECUTIVE_ERRORS = 5
let webgl2Supported: boolean | undefined
/**
* MediaPipe's ImageSegmenter requires a WebGL2 context on the web even with
* `delegate: 'CPU'` (only inference runs on CPU; the mask post-processing in
* TensorsToSegmentationCalculator is GL-based). Without this check, machines
* with WebGL disabled or blocklisted fail at StartGraph with
* `emscripten_webgl_create_context() returned error 0`.
*
* The result is cached and the probe context is explicitly released so that
* repeated support checks do not count against the browser's limit on live
* WebGL contexts.
*/
const isWebGL2Supported = () => {
if (webgl2Supported === undefined) {
try {
const canvas = document.createElement('canvas')
const gl = canvas.getContext('webgl2')
webgl2Supported = !!gl
gl?.getExtension('WEBGL_lose_context')?.loseContext()
} catch {
webgl2Supported = false
}
}
return webgl2Supported
}
/**
* This implementation of video blurring is made to be run on CPU for browser that are
@@ -76,12 +42,14 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
source?: MediaStreamTrack
sourceSettings?: MediaTrackSettings
videoElement?: HTMLVideoElement
videoElementLoaded?: boolean
// Canvas containing the video processing result, of which we extract as stream.
outputCanvas?: HTMLCanvasElement
outputCanvasCtx?: CanvasRenderingContext2D
imageSegmenter?: ImageSegmenter
imageSegmenterResult?: ImageSegmenterResult
// Canvas used for resizing video source and projecting mask.
segmentationMaskCanvas?: HTMLCanvasElement
@@ -98,13 +66,6 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
type: ProcessorType
virtualBackgroundImage?: HTMLImageElement
private virtualBackgroundImagePath?: string
private destroyed = false
private degraded = false
private consecutiveErrors = 0
private processing?: Promise<void>
private onVideoLoaded?: () => void
constructor(opts: ProcessorConfig) {
this.name = 'blur'
this.options = opts
@@ -112,10 +73,7 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
}
static get isSupported() {
return (
navigator.userAgent.toLowerCase().includes('firefox') &&
isWebGL2Supported()
)
return navigator.userAgent.toLowerCase().includes('firefox')
}
async init(opts: ProcessorOptions<Track.Kind>) {
@@ -123,10 +81,6 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
throw new Error('Element is required for processing')
}
this.destroyed = false
this.degraded = false
this.consecutiveErrors = 0
this.source = opts.track as MediaStreamTrack
this.sourceSettings = this.source!.getSettings()
this.videoElement = opts.element as HTMLVideoElement
@@ -143,56 +97,26 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
this.processedTrack = tracks[0]
this.segmentationMask = new ImageData(PROCESSING_WIDTH, PROCESSING_HEIGHT)
const t0 = performance.now()
await this.initSegmenter()
const segmenterInitMs = Math.round(performance.now() - t0)
this._initWorker()
captureEvent('legacy-background-processor', {
effect_type: this.options.type,
hw_concurrency: navigator.hardwareConcurrency,
video_width: this.videoElement?.videoWidth,
video_height: this.videoElement?.videoHeight,
segmenter_init_ms: segmenterInitMs,
})
captureEvent('firefox-blurring-init', {})
}
_initVirtualBackgroundImage() {
if (this.options.type !== 'virtual' || !this.options.imagePath) {
if (this.options.type !== 'virtual') {
return
}
if (
const needsUpdate =
this.options.imagePath &&
this.virtualBackgroundImage &&
this.virtualBackgroundImagePath === this.options.imagePath
) {
return
this.virtualBackgroundImage.src !== this.options.imagePath
if (this.options.imagePath || needsUpdate) {
this.virtualBackgroundImage = document.createElement('img')
this.virtualBackgroundImage.crossOrigin = 'anonymous'
this.virtualBackgroundImage.src = this.options.imagePath!
}
const image = document.createElement('img')
image.crossOrigin = 'anonymous'
image.src = this.options.imagePath
// Surface load failures once instead of letting drawImage throw on a
// broken image inside the processing loop.
image.decode().catch((error) => {
reportError('effects_processor_failure', error, {
context: 'Failed to load virtual background image',
image_path:
this.options.type === 'virtual' ? this.options.imagePath : undefined,
})
})
this.virtualBackgroundImage = image
this.virtualBackgroundImagePath = this.options.imagePath
}
_isVirtualBackgroundImageReady() {
return (
!!this.virtualBackgroundImage &&
this.virtualBackgroundImage.complete &&
this.virtualBackgroundImage.naturalWidth > 0
)
}
async update(opts: ProcessorConfig): Promise<void> {
@@ -205,26 +129,26 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
name: 'Blurring',
})
this.timerWorker.onmessage = (data) => this.onTimerMessage(data)
const startLoop = () => {
this.onVideoLoaded = undefined
this._syncOutputCanvasSize()
this._scheduleNextFrame()
}
if (this.videoElement!.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) {
startLoop()
} else {
this.onVideoLoaded = startLoop
this.videoElement!.addEventListener('loadeddata', this.onVideoLoaded, {
once: true,
// When hiding camera then showing it again, the onloadeddata callback is not fired again.
if (this.videoElementLoaded) {
this.timerWorker!.postMessage({
id: SET_TIMEOUT,
timeMs: 1000 / 30,
})
} else {
this.videoElement!.onloadeddata = () => {
this.videoElementLoaded = true
this.timerWorker!.postMessage({
id: SET_TIMEOUT,
timeMs: 1000 / 30,
})
}
}
}
onTimerMessage(response: { data: { id: number } }) {
if (response.data.id === TIMEOUT_TICK) {
this.processing = this.process()
this.process()
}
}
@@ -270,47 +194,30 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
*/
async segment() {
const startTimeMs = performance.now()
return new Promise<void>((resolve, reject) => {
try {
this.imageSegmenter!.segmentForVideo(
this.sourceImageData!,
startTimeMs,
(result: ImageSegmenterResult) => {
try {
// The mask is only valid for the duration of this callback:
// MediaPipe frees the underlying WASM memory as soon as it
// returns, so the data must be copied out synchronously here.
this._applyMaskToAlphaChannel(result)
resolve()
} catch (error) {
reject(error)
}
}
)
} catch (error) {
reject(error)
}
return new Promise<void>((resolve) => {
this.imageSegmenter!.segmentForVideo(
this.sourceImageData!,
startTimeMs,
(result: ImageSegmenterResult) => {
this.imageSegmenterResult = result
resolve()
}
)
})
}
_applyMaskToAlphaChannel(result: ImageSegmenterResult) {
const categoryMask = result.categoryMask
if (!categoryMask) {
return
}
const mask = categoryMask.getAsUint8Array()
const alpha = this.segmentationMask!.data
const length = Math.min(mask.length, alpha.length / 4)
for (let i = 0; i < length; ++i) {
alpha[i * 4 + 3] = 255 - mask[i]
}
}
/**
* Composite the segmentation mask over the output canvas: mask first, then
* the clear body, leaving the background to be filled by the caller.
* TODO: future improvement with WebGL.
*/
_compositeMaskAndBody() {
async blur() {
if (this.options.type !== 'blur') {
throw new Error('Blurring is only supported for blur background')
}
const mask = this.imageSegmenterResult!.categoryMask!.getAsUint8Array()
for (let i = 0; i < mask.length; ++i) {
this.segmentationMask!.data[i * 4 + 3] = 255 - mask[i]
}
this.segmentationMaskCanvasCtx!.putImageData(this.segmentationMask!, 0, 0)
this.outputCanvasCtx!.globalCompositeOperation = 'copy'
@@ -333,16 +240,6 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
this.outputCanvasCtx!.globalCompositeOperation = 'source-in'
this.outputCanvasCtx!.filter = 'none'
this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0)
}
/**
* TODO: future improvement with WebGL.
*/
async blur() {
if (this.options.type !== 'blur') {
throw new Error('Blurring is only supported for blur background')
}
this._compositeMaskAndBody()
// Draw blurry background.
this.outputCanvasCtx!.globalCompositeOperation = 'destination-over'
@@ -354,150 +251,87 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
* TODO: future improvement with WebGL.
*/
async drawVirtualBackground() {
this._compositeMaskAndBody()
this.outputCanvasCtx!.globalCompositeOperation = 'destination-over'
this.outputCanvasCtx!.filter = 'none'
if (this._isVirtualBackgroundImageReady()) {
// Draw virtual background.
this.outputCanvasCtx!.drawImage(
this.virtualBackgroundImage!,
0,
0,
this.outputCanvas!.width,
this.outputCanvas!.height
)
} else {
// Image not decoded (yet, or failed to load): fill the background with
// a heavy blur instead. Never fall back to the raw video here — the
// user selected this effect to conceal their surroundings, so the
// fallback must keep concealing them.
this.outputCanvasCtx!.filter = `blur(${CONCEALING_BLUR}px)`
this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0)
const mask = this.imageSegmenterResult!.categoryMask!.getAsUint8Array()
for (let i = 0; i < mask.length; ++i) {
this.segmentationMask!.data[i * 4 + 3] = 255 - mask[i]
}
}
/**
* Draw the whole frame heavily blurred (person included). Used when
* segmentation is broken: the outgoing video keeps flowing instead of
* freezing on a stale frame, while the surroundings the user chose to
* conceal stay concealed. Requires no segmenter, only one filtered draw.
*/
_drawDegradedFrame() {
this.segmentationMaskCanvasCtx!.putImageData(this.segmentationMask!, 0, 0)
this.outputCanvasCtx!.globalCompositeOperation = 'copy'
this.outputCanvasCtx!.filter = `blur(${CONCEALING_BLUR}px)`
this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0)
this.outputCanvasCtx!.filter = 'blur(8px)'
// Put opacity mask.
this.outputCanvasCtx!.drawImage(
this.segmentationMaskCanvas!,
0,
0,
PROCESSING_WIDTH,
PROCESSING_HEIGHT,
0,
0,
this.videoElement!.videoWidth,
this.videoElement!.videoHeight
)
// Draw clear body.
this.outputCanvasCtx!.globalCompositeOperation = 'source-in'
this.outputCanvasCtx!.filter = 'none'
this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0)
// Draw virtual background.
this.outputCanvasCtx!.globalCompositeOperation = 'destination-over'
this.outputCanvasCtx!.drawImage(
this.virtualBackgroundImage!,
0,
0,
this.outputCanvas!.width,
this.outputCanvas!.height
)
}
async process() {
if (this.destroyed) {
return
await this.sizeSource()
await this.segment()
if (this.options.type === 'blur') {
await this.blur()
} else {
await this.drawVirtualBackground()
}
try {
this._syncOutputCanvasSize()
// No decoded frame available (e.g. right after a device switch): skip
// this tick rather than processing a 0x0 source.
if (
!this.videoElement ||
this.videoElement.videoWidth === 0 ||
this.videoElement.videoHeight === 0
) {
this._scheduleNextFrame()
return
}
if (this.degraded) {
this._drawDegradedFrame()
this._scheduleNextFrame()
return
}
await this.sizeSource()
await this.segment()
if (this.destroyed) {
return
}
if (this.options.type === 'blur') {
await this.blur()
} else {
await this.drawVirtualBackground()
}
this.consecutiveErrors = 0
} catch (error) {
if (this.destroyed) {
return
}
this.consecutiveErrors += 1
if (this.consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) {
// Degrade to a fully blurred frame: a live camera beats a frozen
// one, and concealment must survive the failure.
this.degraded = true
reportError('effects_processor_failure', error, {
context:
'Background processing failed repeatedly, falling back to fully blurred video',
consecutive_errors: this.consecutiveErrors,
})
this.imageSegmenter?.close()
this.imageSegmenter = undefined
}
}
this._scheduleNextFrame()
}
_scheduleNextFrame() {
if (this.destroyed) {
return
}
this.timerWorker?.postMessage({
this.timerWorker!.postMessage({
id: SET_TIMEOUT,
timeMs: FRAME_INTERVAL_MS,
timeMs: 1000 / 30,
})
}
/**
* Keep the output canvas in sync with the actual decoded video dimensions.
* `MediaStreamTrack.getSettings()` can be incomplete or stale on Firefox,
* so the video element is the source of truth.
*/
_syncOutputCanvasSize() {
const width = this.videoElement?.videoWidth
const height = this.videoElement?.videoHeight
if (!width || !height || !this.outputCanvas) {
return
}
if (
this.outputCanvas.width !== width ||
this.outputCanvas.height !== height
) {
this.outputCanvas.width = width
this.outputCanvas.height = height
}
}
_createMainCanvas() {
const width =
this.sourceSettings?.width || this.videoElement?.videoWidth || 1280
const height =
this.sourceSettings?.height || this.videoElement?.videoHeight || 720
this.outputCanvas = this._createCanvas(BLUR_CANVAS_ID, width, height)
this.outputCanvas = document.querySelector(
'canvas#background-blur-local'
) as HTMLCanvasElement
if (!this.outputCanvas) {
this.outputCanvas = this._createCanvas(
BLUR_CANVAS_ID,
this.sourceSettings!.width!,
this.sourceSettings!.height!
)
}
this.outputCanvasCtx = this.outputCanvas.getContext('2d')!
}
_createMaskCanvas() {
this.segmentationMaskCanvas = this._createCanvas(
SEGMENTATION_MASK_CANVAS_ID,
PROCESSING_WIDTH,
PROCESSING_HEIGHT
)
// getImageData is called on this canvas 30 times per second: opt out of
// GPU backing to avoid a costly readback on every frame.
this.segmentationMaskCanvasCtx = this.segmentationMaskCanvas.getContext(
'2d',
{ willReadFrequently: true }
)!
this.segmentationMaskCanvas = document.querySelector(
`#${SEGMENTATION_MASK_CANVAS_ID}`
) as HTMLCanvasElement
if (!this.segmentationMaskCanvas) {
this.segmentationMaskCanvas = this._createCanvas(
SEGMENTATION_MASK_CANVAS_ID,
PROCESSING_WIDTH,
PROCESSING_HEIGHT
)
}
this.segmentationMaskCanvasCtx =
this.segmentationMaskCanvas.getContext('2d')!
}
_createCanvas(id: string, width: number, height: number) {
@@ -514,39 +348,11 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
}
async destroy() {
this.destroyed = true
this.timerWorker?.postMessage({
id: CLEAR_TIMEOUT,
})
// Let any in-flight frame finish before releasing the resources it uses,
// so segmentForVideo is never called on a closed segmenter.
try {
await this.processing
} catch {
// Failures are already handled inside process().
}
this.processing = undefined
if (this.onVideoLoaded && this.videoElement) {
this.videoElement.removeEventListener('loadeddata', this.onVideoLoaded)
}
this.onVideoLoaded = undefined
this.timerWorker?.terminate()
this.timerWorker = undefined
this.imageSegmenter?.close()
this.imageSegmenter = undefined
this.processedTrack?.stop()
this.processedTrack = undefined
this.outputCanvas = undefined
this.outputCanvasCtx = undefined
this.segmentationMaskCanvas = undefined
this.segmentationMaskCanvasCtx = undefined
this.sourceImageData = undefined
}
}
@@ -6,7 +6,6 @@ import type { Track, TrackProcessor } from 'livekit-client'
import { BackgroundCustomProcessor } from './BackgroundCustomProcessor'
import { UnifiedBackgroundTrackProcessor } from './UnifiedBackgroundTrackProcessor'
import { FaceLandmarksOptions } from './FaceLandmarksProcessor'
import { captureEvent } from '@/features/analytics/telemetry'
export const SELFIE_SEGMENTER_MODEL_PATH =
'/assets/mediapipe/models/selfie_segmenter_landscape.tflite'
@@ -32,29 +31,15 @@ export interface BackgroundProcessorInterface extends TrackProcessor<Track.Kind>
options: ProcessorConfig
}
let unsupportedReported = false
export class BackgroundProcessorFactory {
private static _isSupported?: boolean
static hasModernApiSupport() {
return ProcessorWrapper.hasModernApiSupport
}
static isSupported() {
if (this._isSupported === undefined) {
this._isSupported =
supportsBackgroundProcessors() || BackgroundCustomProcessor.isSupported
}
if (!this._isSupported && !unsupportedReported) {
unsupportedReported = true
captureEvent('background-processor-unsupported', {
path: 'isSupported',
})
}
return this._isSupported
return (
supportsBackgroundProcessors() || BackgroundCustomProcessor.isSupported
)
}
static getProcessor(
@@ -63,8 +48,6 @@ export class BackgroundProcessorFactory {
const isBlur = config.type === ProcessorType.BLUR
const isVirtual = config.type === ProcessorType.VIRTUAL
return new BackgroundCustomProcessor(config)
if (!isBlur && !isVirtual) return undefined
if (supportsBackgroundProcessors()) {
@@ -75,12 +58,6 @@ export class BackgroundProcessorFactory {
return new BackgroundCustomProcessor(config)
}
if (!unsupportedReported) {
captureEvent('background-processor-unsupported', {
path: 'getProcessor',
})
}
return undefined
}
@@ -2,7 +2,7 @@ import { useTranslation } from 'react-i18next'
import { useTrackToggle, UseTrackToggleProps } from '@livekit/components-react'
import { Button, Popover } from '@/primitives'
import { RiArrowUpSLine, RiImageCircleAiFill } from '@remixicon/react'
import { Track, type VideoCaptureOptions, VideoPresets } from 'livekit-client'
import { Track, type VideoCaptureOptions } from 'livekit-client'
import { ToggleDevice } from './ToggleDevice'
import { css } from '@/styled-system/css'
@@ -57,8 +57,7 @@ export const VideoDeviceControl = ({
}: VideoDeviceControlProps) => {
const { t } = useTranslation('rooms', { keyPrefix: 'selectDevice' })
const { videoDeviceId, processorConfig, videoPublishResolution } =
useSnapshot(userChoicesStore)
const { videoDeviceId, processorConfig } = useSnapshot(userChoicesStore)
const onChange = React.useCallback(
(enabled: boolean, isUserInitiated: boolean) =>
@@ -98,9 +97,6 @@ export const VideoDeviceControl = ({
await toggle(!trackProps.enabled, {
processor: processor,
...(videoPublishResolution && {
resolution: VideoPresets[videoPublishResolution].resolution,
}),
} as VideoCaptureOptions)
}
@@ -25,11 +25,7 @@ import {
} from '@/features/files/api/listFiles.ts'
import { useCreateFile } from '@/features/files/api/createFile.ts'
import { FileTrigger } from 'react-aria-components'
import {
RiDeleteBinLine,
RiImageAddFill,
RiProhibitedLine,
} from '@remixicon/react'
import { RiDeleteBinLine, RiImageAddFill } from '@remixicon/react'
import { useDeleteFile } from '@/features/files/api/deleteFile.ts'
import { useUser } from '@/features/auth/api/useUser'
import { ApiFileItem } from '@/features/files/api/types.ts'
@@ -201,23 +197,10 @@ export const EffectsConfiguration = ({
*
* We arrive in this condition when we enter the room with the camera already off.
*/
try {
const newProcessorTmp =
BackgroundProcessorFactory.getProcessor(config)!
await toggle(true, {
processor: newProcessorTmp,
})
} catch (error) {
reportError('effects_processor_failure', error, {
context: 'Error applying effect while enabling camera:',
})
saveProcessorConfig(undefined)
try {
await toggle(true)
} catch {
// Camera errors are handled by the toggle's own error path.
}
}
const newProcessorTmp = BackgroundProcessorFactory.getProcessor(config)!
await toggle(true, {
processor: newProcessorTmp,
})
setTimeout(() => setProcessorPending(false))
return
}
@@ -259,14 +242,6 @@ export const EffectsConfiguration = ({
reportError('effects_processor_failure', error, {
context: 'Error applying effect:',
})
try {
if (videoTrack.getProcessor()) {
await videoTrack.stopProcessor()
}
} catch {
// Best effort: the processor may already be broken.
}
saveProcessorConfig(undefined)
} finally {
// Without setTimeout the DOM is not refreshing when updating the options.
setTimeout(() => setProcessorPending(false))
@@ -275,24 +250,6 @@ export const EffectsConfiguration = ({
[enabled, selectedId, toggle, updateEffectStatusMessage, videoTrack]
)
const clearEffect = useCallback(async () => {
if (selectedId === 'none') return
setProcessorPending(true)
try {
if (videoTrack?.getProcessor()) {
await videoTrack.stopProcessor()
}
saveProcessorConfig(undefined)
announceEffectStatusMessage(t('blur.status.none'))
} catch (error) {
reportError('effects_processor_failure', error, {
context: 'Error clearing effect:',
})
} finally {
setTimeout(() => setProcessorPending(false))
}
}, [announceEffectStatusMessage, selectedId, t, videoTrack])
const { data: appConfig } = useConfig()
const { isLoggedIn } = useUser()
const canUploadBackground =
@@ -690,17 +647,6 @@ export const EffectsConfiguration = ({
gap: '1.25rem',
})}
>
<ToggleButton
variant="bigSquare"
aria-label={t('clear')}
tooltip={t('clear')}
isDisabled={processorOptions.isDisabled}
onChange={clearEffect}
isSelected={selectedId === 'none'}
data-attr="toggle-effect-none"
>
<RiProhibitedLine />
</ToggleButton>
{processorOptions.blurBased.map(({ Icon, ...option }) => (
<ToggleButton
key={option.id}
@@ -22,12 +22,10 @@ import { VOICE_AUDIO_CONSTRAINTS } from '../utils/constants'
import {
saveAudioInputDeviceId,
saveAudioInputEnabled,
saveProcessorConfig,
saveVideoInputDeviceId,
saveVideoInputEnabled,
userChoicesStore,
} from '@/stores/userChoices'
import { reportError } from '@/features/analytics/telemetry'
import { useSyncTrackDeviceId } from './useSyncTrackDeviceId'
// Module-level: effect dependencies, must be referentially stable.
@@ -223,32 +221,15 @@ export function useJoinTracks(): {
[audioDeviceId]
)
const createVideo = useCallback(async () => {
const processor =
BackgroundProcessorFactory.fromProcessorConfig(processorConfig)
if (!processor) {
return createLocalVideoTrack({ deviceId: videoDeviceId })
}
try {
return await createLocalVideoTrack({
const createVideo = useCallback(
() =>
createLocalVideoTrack({
deviceId: videoDeviceId,
processor,
})
} catch (error) {
// A camera problem (permission, device missing/busy) is not the
// effect's fault: let the normal media error handling deal with it
// without touching the user's saved effect.
const e = getMediaDeviceFailure(error as Error)
if (e !== MediaDeviceFailure.Other && !!e) {
throw error
}
reportError('effects_processor_failure', error, {
context: 'Restoring saved effect failed, retrying without it',
})
saveProcessorConfig(undefined)
return createLocalVideoTrack({ deviceId: videoDeviceId })
}
}, [videoDeviceId, processorConfig])
processor:
BackgroundProcessorFactory.fromProcessorConfig(processorConfig),
}),
[videoDeviceId, processorConfig]
)
const audioTrack = useLocalTrack({
ready: audioReady,
@@ -18,7 +18,6 @@ import {
saveVideoPublishResolution,
saveVideoSubscribeQuality,
userChoicesStore,
VIDEO_RESOLUTIONS,
VideoResolution,
} from '@/stores/userChoices'
import { RowWrapper } from './layout/RowWrapper'
@@ -33,8 +32,7 @@ const EMPTY_PROPS = {}
export const VideoTab = ({ id }: VideoTabProps) => {
const { t } = useTranslation('settings', { keyPrefix: 'video' })
const room = useRoomContext()
const { localParticipant, remoteParticipants } = room
const { localParticipant, remoteParticipants } = useRoomContext()
const {
videoDeviceId,
@@ -71,21 +69,20 @@ export const VideoTab = ({ id }: VideoTabProps) => {
isDisabled: true,
}
const handleVideoResolutionChange = async (key: VideoResolution) => {
saveVideoPublishResolution(key)
const videoTrack = localParticipant.getTrackPublication(
const handleVideoResolutionChange = async (key: 'h720' | 'h360' | 'h180') => {
const videoPublication = localParticipant.getTrackPublication(
Track.Source.Camera
)?.track
if (!videoTrack) {
return
)
const videoTrack = videoPublication?.track
if (videoTrack) {
saveVideoPublishResolution(key)
await videoTrack.restartTrack({
resolution: VideoPresets[key].resolution,
deviceId: { exact: videoDeviceId },
processor:
BackgroundProcessorFactory.fromProcessorConfig(processorConfig),
})
}
await videoTrack.restartTrack({
resolution: VideoPresets[key].resolution,
deviceId: { exact: videoDeviceId },
processor:
BackgroundProcessorFactory.fromProcessorConfig(processorConfig),
})
}
/**
@@ -125,13 +122,20 @@ export const VideoTab = ({ id }: VideoTabProps) => {
}, [videoDeviceId, videoElement])
const resolutionItems = useMemo(() => {
const labels: Record<VideoResolution, string> = {
h1080: `${t('resolution.publish.items.veryHigh')} (1080p)`,
h720: `${t('resolution.publish.items.high')} (720p)`,
h360: `${t('resolution.publish.items.medium')} (360p)`,
h180: `${t('resolution.publish.items.low')} (180p)`,
}
return VIDEO_RESOLUTIONS.map((value) => ({ value, label: labels[value] }))
return [
{
value: 'h720',
label: `${t('resolution.publish.items.high')} (720p)`,
},
{
value: 'h360',
label: `${t('resolution.publish.items.medium')} (360p)`,
},
{
value: 'h180',
label: `${t('resolution.publish.items.low')} (180p)`,
},
]
}, [t])
const videoQualityItems = useMemo(() => {
@@ -56,7 +56,6 @@
"publish": {
"label": "Wähle die maximale Auflösung beim Senden",
"items": {
"veryHigh": "Sehr hohe Auflösung",
"high": "Hohe Auflösung",
"medium": "Mittlere Auflösung",
"low": "Niedrige Auflösung"
@@ -56,7 +56,6 @@
"publish": {
"label": "Select your sending resolution (max.)",
"items": {
"veryHigh": "Very high definition",
"high": "High definition",
"medium": "Standard definition",
"low": "Low definition"
@@ -56,7 +56,6 @@
"publish": {
"label": "Selecciona tu resolución de envío (máx.)",
"items": {
"veryHigh": "Muy alta definición",
"high": "Alta definición",
"medium": "Definición estándar",
"low": "Baja definición"
@@ -56,7 +56,6 @@
"publish": {
"label": "Sélectionner votre résolution d'envoi (max.)",
"items": {
"veryHigh": "Très haute définition",
"high": "Haute définition",
"medium": "Définition standard",
"low": "Basse définition"
@@ -56,7 +56,6 @@
"publish": {
"label": "Selecteer uw verzendresolutie (max.)",
"items": {
"veryHigh": "Zeer hoge definitie",
"high": "Hoge definitie",
"medium": "Standaarddefinitie",
"low": "Lage definitie"
+2 -11
View File
@@ -10,12 +10,7 @@ import {
} from '@livekit/components-core'
import { VideoQuality } from 'livekit-client'
export const VIDEO_RESOLUTIONS = ['h1080', 'h720', 'h360', 'h180'] as const
export type VideoResolution = (typeof VIDEO_RESOLUTIONS)[number]
const isVideoResolution = (value: unknown): value is VideoResolution =>
VIDEO_RESOLUTIONS.includes(value as VideoResolution)
export type VideoResolution = 'h720' | 'h360' | 'h180'
export type LocalUserChoices = Omit<LocalUserChoicesLK, 'username'> & {
processorConfig?: ProcessorConfig
@@ -26,17 +21,13 @@ export type LocalUserChoices = Omit<LocalUserChoicesLK, 'username'> & {
}
function getUserChoicesState(): LocalUserChoices {
const stored: LocalUserChoices = {
return {
noiseReductionEnabled: false,
audioOutputDeviceId: 'default', // Use 'default' to match LiveKit's standard device selection behavior
videoPublishResolution: 'h720',
videoSubscribeQuality: VideoQuality.HIGH,
...loadUserChoices(),
}
if (!isVideoResolution(stored.videoPublishResolution)) {
stored.videoPublishResolution = 'h720'
}
return stored
}
export const userChoicesStore = proxy<LocalUserChoices>(getUserChoicesState())