From 2a71e5b836aa3031aacb791ecf22772f696270e2 Mon Sep 17 00:00:00 2001 From: lebaudantoine Date: Wed, 2 Sep 2026 18:53:33 +0200 Subject: [PATCH] wip --- .../blur/BackgroundCustomProcessor.ts | 394 +++++++++++++----- 1 file changed, 287 insertions(+), 107 deletions(-) diff --git a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts index 73896e7a..9cc6706b 100644 --- a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts +++ b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts @@ -17,7 +17,7 @@ import { type ProcessorType, MEDIAPIPE_PATH_WASM, } from '.' -import { captureEvent } from '@/features/analytics/telemetry.ts' +import { captureEvent, reportError } from '@/features/analytics/telemetry' const PROCESSING_WIDTH = 256 const PROCESSING_HEIGHT = 144 @@ -26,6 +26,39 @@ const SEGMENTATION_MASK_CANVAS_ID = 'background-blur-local-segmentation' const BLUR_CANVAS_ID = 'background-blur-local' const DEFAULT_BLUR = '10' +const FRAME_INTERVAL_MS = 1000 / 30 + +// After this many consecutive failed frames, stop segmenting and fall back to +// passing the raw video through, so the user keeps a live camera instead of a +// frozen frame. +const MAX_CONSECUTIVE_ERRORS = 5 + +let webgl2Supported: boolean | undefined + +/** + * MediaPipe's ImageSegmenter requires a WebGL2 context on the web even with + * `delegate: 'CPU'` (only inference runs on CPU; the mask post-processing in + * TensorsToSegmentationCalculator is GL-based). Without this check, machines + * with WebGL disabled or blocklisted fail at StartGraph with + * `emscripten_webgl_create_context() returned error 0`. + * + * The result is cached and the probe context is explicitly released so that + * repeated support checks do not count against the browser's limit on live + * WebGL contexts. + */ +const isWebGL2Supported = () => { + if (webgl2Supported === undefined) { + try { + const canvas = document.createElement('canvas') + const gl = canvas.getContext('webgl2') + webgl2Supported = !!gl + gl?.getExtension('WEBGL_lose_context')?.loseContext() + } catch { + webgl2Supported = false + } + } + return webgl2Supported +} /** * This implementation of video blurring is made to be run on CPU for browser that are @@ -42,14 +75,12 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface { source?: MediaStreamTrack sourceSettings?: MediaTrackSettings videoElement?: HTMLVideoElement - videoElementLoaded?: boolean // Canvas containing the video processing result, of which we extract as stream. outputCanvas?: HTMLCanvasElement outputCanvasCtx?: CanvasRenderingContext2D imageSegmenter?: ImageSegmenter - imageSegmenterResult?: ImageSegmenterResult // Canvas used for resizing video source and projecting mask. segmentationMaskCanvas?: HTMLCanvasElement @@ -66,6 +97,13 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface { type: ProcessorType virtualBackgroundImage?: HTMLImageElement + private virtualBackgroundImagePath?: string + private destroyed = false + private passthrough = false + private consecutiveErrors = 0 + private processing?: Promise + private onVideoLoaded?: () => void + constructor(opts: ProcessorConfig) { this.name = 'blur' this.options = opts @@ -73,7 +111,10 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface { } static get isSupported() { - return navigator.userAgent.toLowerCase().includes('firefox') + return ( + navigator.userAgent.toLowerCase().includes('firefox') && + isWebGL2Supported() + ) } async init(opts: ProcessorOptions) { @@ -81,6 +122,10 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface { throw new Error('Element is required for processing') } + this.destroyed = false + this.passthrough = false + this.consecutiveErrors = 0 + this.source = opts.track as MediaStreamTrack this.sourceSettings = this.source!.getSettings() this.videoElement = opts.element as HTMLVideoElement @@ -104,19 +149,38 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface { } _initVirtualBackgroundImage() { - if (this.options.type !== 'virtual') { + if (this.options.type !== 'virtual' || !this.options.imagePath) { return } - const needsUpdate = - this.options.imagePath && + if ( this.virtualBackgroundImage && - this.virtualBackgroundImage.src !== this.options.imagePath - if (this.options.imagePath || needsUpdate) { - this.virtualBackgroundImage = document.createElement('img') - this.virtualBackgroundImage.crossOrigin = 'anonymous' - this.virtualBackgroundImage.src = this.options.imagePath! + this.virtualBackgroundImagePath === this.options.imagePath + ) { + return } + + const image = document.createElement('img') + image.crossOrigin = 'anonymous' + image.src = this.options.imagePath + // Surface load failures once instead of letting drawImage throw on a + // broken image inside the processing loop. + image.decode().catch((error) => { + reportError('effects_processor_failure', error, { + context: 'Failed to load virtual background image', + image_path: this.options.type === 'virtual' ? this.options.imagePath : undefined, + }) + }) + this.virtualBackgroundImage = image + this.virtualBackgroundImagePath = this.options.imagePath + } + + _isVirtualBackgroundImageReady() { + return ( + !!this.virtualBackgroundImage && + this.virtualBackgroundImage.complete && + this.virtualBackgroundImage.naturalWidth > 0 + ) } async update(opts: ProcessorConfig): Promise { @@ -129,26 +193,29 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface { name: 'Blurring', }) this.timerWorker.onmessage = (data) => this.onTimerMessage(data) - // When hiding camera then showing it again, the onloadeddata callback is not fired again. - if (this.videoElementLoaded) { - this.timerWorker!.postMessage({ - id: SET_TIMEOUT, - timeMs: 1000 / 30, - }) + + const startLoop = () => { + this.onVideoLoaded = undefined + this._syncOutputCanvasSize() + this._scheduleNextFrame() + } + + // When re-initializing with an element that already has data (e.g. after + // hiding then showing the camera), 'loadeddata' will not fire again, so + // rely on readyState instead of a stale boolean flag. + if (this.videoElement!.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) { + startLoop() } else { - this.videoElement!.onloadeddata = () => { - this.videoElementLoaded = true - this.timerWorker!.postMessage({ - id: SET_TIMEOUT, - timeMs: 1000 / 30, - }) - } + this.onVideoLoaded = startLoop + this.videoElement!.addEventListener('loadeddata', this.onVideoLoaded, { + once: true, + }) } } onTimerMessage(response: { data: { id: number } }) { if (response.data.id === TIMEOUT_TICK) { - this.process() + this.processing = this.process() } } @@ -194,30 +261,47 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface { */ async segment() { const startTimeMs = performance.now() - return new Promise((resolve) => { - this.imageSegmenter!.segmentForVideo( - this.sourceImageData!, - startTimeMs, - (result: ImageSegmenterResult) => { - this.imageSegmenterResult = result - resolve() - } - ) + return new Promise((resolve, reject) => { + try { + this.imageSegmenter!.segmentForVideo( + this.sourceImageData!, + startTimeMs, + (result: ImageSegmenterResult) => { + try { + // The mask is only valid for the duration of this callback: + // MediaPipe frees the underlying WASM memory as soon as it + // returns, so the data must be copied out synchronously here. + this._applyMaskToAlphaChannel(result) + resolve() + } catch (error) { + reject(error) + } + } + ) + } catch (error) { + reject(error) + } }) } - /** - * TODO: future improvement with WebGL. - */ - async blur() { - if (this.options.type !== 'blur') { - throw new Error('Blurring is only supported for blur background') + _applyMaskToAlphaChannel(result: ImageSegmenterResult) { + const categoryMask = result.categoryMask + if (!categoryMask) { + return } - const mask = this.imageSegmenterResult!.categoryMask!.getAsUint8Array() - for (let i = 0; i < mask.length; ++i) { - this.segmentationMask!.data[i * 4 + 3] = 255 - mask[i] + const mask = categoryMask.getAsUint8Array() + const alpha = this.segmentationMask!.data + const length = Math.min(mask.length, alpha.length / 4) + for (let i = 0; i < length; ++i) { + alpha[i * 4 + 3] = 255 - mask[i] } + } + /** + * Composite the segmentation mask over the output canvas: mask first, then + * the clear body, leaving the background to be filled by the caller. + */ + _compositeMaskAndBody() { this.segmentationMaskCanvasCtx!.putImageData(this.segmentationMask!, 0, 0) this.outputCanvasCtx!.globalCompositeOperation = 'copy' @@ -240,6 +324,16 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface { this.outputCanvasCtx!.globalCompositeOperation = 'source-in' this.outputCanvasCtx!.filter = 'none' this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0) + } + + /** + * TODO: future improvement with WebGL. + */ + async blur() { + if (this.options.type !== 'blur') { + throw new Error('Blurring is only supported for blur background') + } + this._compositeMaskAndBody() // Draw blurry background. this.outputCanvasCtx!.globalCompositeOperation = 'destination-over' @@ -251,87 +345,145 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface { * TODO: future improvement with WebGL. */ async drawVirtualBackground() { - const mask = this.imageSegmenterResult!.categoryMask!.getAsUint8Array() - for (let i = 0; i < mask.length; ++i) { - this.segmentationMask!.data[i * 4 + 3] = 255 - mask[i] + this._compositeMaskAndBody() + + this.outputCanvasCtx!.globalCompositeOperation = 'destination-over' + this.outputCanvasCtx!.filter = 'none' + if (this._isVirtualBackgroundImageReady()) { + // Draw virtual background. + this.outputCanvasCtx!.drawImage( + this.virtualBackgroundImage!, + 0, + 0, + this.outputCanvas!.width, + this.outputCanvas!.height + ) + } else { + // Image not decoded (yet, or failed to load): fall back to the raw + // video so the participant never appears over a black background. + this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0) } + } - this.segmentationMaskCanvasCtx!.putImageData(this.segmentationMask!, 0, 0) - + /** + * Draw the raw video without any effect. Used when segmentation is broken, + * so the outgoing video keeps flowing instead of freezing on a stale frame. + */ + _drawPassthroughFrame() { this.outputCanvasCtx!.globalCompositeOperation = 'copy' - this.outputCanvasCtx!.filter = 'blur(8px)' - - // Put opacity mask. - this.outputCanvasCtx!.drawImage( - this.segmentationMaskCanvas!, - 0, - 0, - PROCESSING_WIDTH, - PROCESSING_HEIGHT, - 0, - 0, - this.videoElement!.videoWidth, - this.videoElement!.videoHeight - ) - - // Draw clear body. - this.outputCanvasCtx!.globalCompositeOperation = 'source-in' this.outputCanvasCtx!.filter = 'none' this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0) - - // Draw virtual background. - this.outputCanvasCtx!.globalCompositeOperation = 'destination-over' - this.outputCanvasCtx!.drawImage( - this.virtualBackgroundImage!, - 0, - 0, - this.outputCanvas!.width, - this.outputCanvas!.height - ) } async process() { - await this.sizeSource() - await this.segment() - - if (this.options.type === 'blur') { - await this.blur() - } else { - await this.drawVirtualBackground() + if (this.destroyed) { + return } - this.timerWorker!.postMessage({ + try { + this._syncOutputCanvasSize() + + // No decoded frame available (e.g. right after a device switch): skip + // this tick rather than processing a 0x0 source. + if ( + !this.videoElement || + this.videoElement.videoWidth === 0 || + this.videoElement.videoHeight === 0 + ) { + this._scheduleNextFrame() + return + } + + if (this.passthrough) { + this._drawPassthroughFrame() + this._scheduleNextFrame() + return + } + + await this.sizeSource() + await this.segment() + + if (this.destroyed) { + return + } + + if (this.options.type === 'blur') { + await this.blur() + } else { + await this.drawVirtualBackground() + } + this.consecutiveErrors = 0 + } catch (error) { + if (this.destroyed) { + return + } + this.consecutiveErrors += 1 + if (this.consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) { + // Degrade to passthrough: a working camera without the effect is + // better than a permanently frozen frame. + this.passthrough = true + reportError('effects_processor_failure', error, { + context: + 'Background processing failed repeatedly, falling back to unprocessed video', + consecutive_errors: this.consecutiveErrors, + }) + this.imageSegmenter?.close() + this.imageSegmenter = undefined + } + } + this._scheduleNextFrame() + } + + _scheduleNextFrame() { + if (this.destroyed) { + return + } + this.timerWorker?.postMessage({ id: SET_TIMEOUT, - timeMs: 1000 / 30, + timeMs: FRAME_INTERVAL_MS, }) } - _createMainCanvas() { - this.outputCanvas = document.querySelector( - 'canvas#background-blur-local' - ) as HTMLCanvasElement - if (!this.outputCanvas) { - this.outputCanvas = this._createCanvas( - BLUR_CANVAS_ID, - this.sourceSettings!.width!, - this.sourceSettings!.height! - ) + /** + * Keep the output canvas in sync with the actual decoded video dimensions. + * `MediaStreamTrack.getSettings()` can be incomplete or stale on Firefox, + * so the video element is the source of truth. + */ + _syncOutputCanvasSize() { + const width = this.videoElement?.videoWidth + const height = this.videoElement?.videoHeight + if (!width || !height || !this.outputCanvas) { + return } + if ( + this.outputCanvas.width !== width || + this.outputCanvas.height !== height + ) { + this.outputCanvas.width = width + this.outputCanvas.height = height + } + } + + _createMainCanvas() { + const width = + this.sourceSettings?.width || this.videoElement?.videoWidth || 1280 + const height = + this.sourceSettings?.height || this.videoElement?.videoHeight || 720 + this.outputCanvas = this._createCanvas(BLUR_CANVAS_ID, width, height) this.outputCanvasCtx = this.outputCanvas.getContext('2d')! } _createMaskCanvas() { - this.segmentationMaskCanvas = document.querySelector( - `#${SEGMENTATION_MASK_CANVAS_ID}` - ) as HTMLCanvasElement - if (!this.segmentationMaskCanvas) { - this.segmentationMaskCanvas = this._createCanvas( - SEGMENTATION_MASK_CANVAS_ID, - PROCESSING_WIDTH, - PROCESSING_HEIGHT - ) - } - this.segmentationMaskCanvasCtx = - this.segmentationMaskCanvas.getContext('2d')! + this.segmentationMaskCanvas = this._createCanvas( + SEGMENTATION_MASK_CANVAS_ID, + PROCESSING_WIDTH, + PROCESSING_HEIGHT + ) + // getImageData is called on this canvas 30 times per second: opt out of + // GPU backing to avoid a costly readback on every frame. + this.segmentationMaskCanvasCtx = this.segmentationMaskCanvas.getContext( + '2d', + { willReadFrequently: true } + )! } _createCanvas(id: string, width: number, height: number) { @@ -348,11 +500,39 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface { } async destroy() { + this.destroyed = true + this.timerWorker?.postMessage({ id: CLEAR_TIMEOUT, }) + // Let any in-flight frame finish before releasing the resources it uses, + // so segmentForVideo is never called on a closed segmenter. + try { + await this.processing + } catch { + // Failures are already handled inside process(). + } + this.processing = undefined + + if (this.onVideoLoaded && this.videoElement) { + this.videoElement.removeEventListener('loadeddata', this.onVideoLoaded) + } + this.onVideoLoaded = undefined + this.timerWorker?.terminate() + this.timerWorker = undefined + this.imageSegmenter?.close() + this.imageSegmenter = undefined + + this.processedTrack?.stop() + this.processedTrack = undefined + + this.outputCanvas = undefined + this.outputCanvasCtx = undefined + this.segmentationMaskCanvas = undefined + this.segmentationMaskCanvasCtx = undefined + this.sourceImageData = undefined } }