wip

src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts

@@ -17,7 +17,7 @@ import {
   type ProcessorType,
   MEDIAPIPE_PATH_WASM,
 } from '.'
-import { captureEvent } from '@/features/analytics/telemetry.ts'
+import { captureEvent, reportError } from '@/features/analytics/telemetry'
 
 const PROCESSING_WIDTH = 256
 const PROCESSING_HEIGHT = 144
@@ -26,6 +26,39 @@ const SEGMENTATION_MASK_CANVAS_ID = 'background-blur-local-segmentation'
 const BLUR_CANVAS_ID = 'background-blur-local'
 
 const DEFAULT_BLUR = '10'
+const FRAME_INTERVAL_MS = 1000 / 30
+
+// After this many consecutive failed frames, stop segmenting and fall back to
+// passing the raw video through, so the user keeps a live camera instead of a
+// frozen frame.
+const MAX_CONSECUTIVE_ERRORS = 5
+
+let webgl2Supported: boolean | undefined
+
+/**
+ * MediaPipe's ImageSegmenter requires a WebGL2 context on the web even with
+ * `delegate: 'CPU'` (only inference runs on CPU; the mask post-processing in
+ * TensorsToSegmentationCalculator is GL-based). Without this check, machines
+ * with WebGL disabled or blocklisted fail at StartGraph with
+ * `emscripten_webgl_create_context() returned error 0`.
+ *
+ * The result is cached and the probe context is explicitly released so that
+ * repeated support checks do not count against the browser's limit on live
+ * WebGL contexts.
+ */
+const isWebGL2Supported = () => {
+  if (webgl2Supported === undefined) {
+    try {
+      const canvas = document.createElement('canvas')
+      const gl = canvas.getContext('webgl2')
+      webgl2Supported = !!gl
+      gl?.getExtension('WEBGL_lose_context')?.loseContext()
+    } catch {
+      webgl2Supported = false
+    }
+  }
+  return webgl2Supported
+}
 
 /**
  * This implementation of video blurring is made to be run on CPU for browser that are
@@ -42,14 +75,12 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
   source?: MediaStreamTrack
   sourceSettings?: MediaTrackSettings
   videoElement?: HTMLVideoElement
-  videoElementLoaded?: boolean
 
   // Canvas containing the video processing result, of which we extract as stream.
   outputCanvas?: HTMLCanvasElement
   outputCanvasCtx?: CanvasRenderingContext2D
 
   imageSegmenter?: ImageSegmenter
-  imageSegmenterResult?: ImageSegmenterResult
 
   // Canvas used for resizing video source and projecting mask.
   segmentationMaskCanvas?: HTMLCanvasElement
@@ -66,6 +97,13 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
   type: ProcessorType
   virtualBackgroundImage?: HTMLImageElement
 
+  private virtualBackgroundImagePath?: string
+  private destroyed = false
+  private passthrough = false
+  private consecutiveErrors = 0
+  private processing?: Promise<void>
+  private onVideoLoaded?: () => void
+
   constructor(opts: ProcessorConfig) {
     this.name = 'blur'
     this.options = opts
@@ -73,7 +111,10 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
   }
 
   static get isSupported() {
-    return navigator.userAgent.toLowerCase().includes('firefox')
+    return (
+      navigator.userAgent.toLowerCase().includes('firefox') &&
+      isWebGL2Supported()
+    )
   }
 
   async init(opts: ProcessorOptions<Track.Kind>) {
@@ -81,6 +122,10 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
       throw new Error('Element is required for processing')
     }
 
+    this.destroyed = false
+    this.passthrough = false
+    this.consecutiveErrors = 0
+
     this.source = opts.track as MediaStreamTrack
     this.sourceSettings = this.source!.getSettings()
     this.videoElement = opts.element as HTMLVideoElement
@@ -104,19 +149,38 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
   }
 
   _initVirtualBackgroundImage() {
-    if (this.options.type !== 'virtual') {
+    if (this.options.type !== 'virtual' || !this.options.imagePath) {
       return
     }
 
-    const needsUpdate =
+    if (
-      this.options.imagePath &&
       this.virtualBackgroundImage &&
-      this.virtualBackgroundImage.src !== this.options.imagePath
+      this.virtualBackgroundImagePath === this.options.imagePath
-    if (this.options.imagePath || needsUpdate) {
+    ) {
-      this.virtualBackgroundImage = document.createElement('img')
+      return
-      this.virtualBackgroundImage.crossOrigin = 'anonymous'
-      this.virtualBackgroundImage.src = this.options.imagePath!
     }
+
+    const image = document.createElement('img')
+    image.crossOrigin = 'anonymous'
+    image.src = this.options.imagePath
+    // Surface load failures once instead of letting drawImage throw on a
+    // broken image inside the processing loop.
+    image.decode().catch((error) => {
+      reportError('effects_processor_failure', error, {
+        context: 'Failed to load virtual background image',
+        image_path: this.options.type === 'virtual' ? this.options.imagePath : undefined,
+      })
+    })
+    this.virtualBackgroundImage = image
+    this.virtualBackgroundImagePath = this.options.imagePath
+  }
+
+  _isVirtualBackgroundImageReady() {
+    return (
+      !!this.virtualBackgroundImage &&
+      this.virtualBackgroundImage.complete &&
+      this.virtualBackgroundImage.naturalWidth > 0
+    )
   }
 
   async update(opts: ProcessorConfig): Promise<void> {
@@ -129,26 +193,29 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
       name: 'Blurring',
     })
     this.timerWorker.onmessage = (data) => this.onTimerMessage(data)
-    // When hiding camera then showing it again, the onloadeddata callback is not fired again.
+
-    if (this.videoElementLoaded) {
+    const startLoop = () => {
-      this.timerWorker!.postMessage({
+      this.onVideoLoaded = undefined
-        id: SET_TIMEOUT,
+      this._syncOutputCanvasSize()
-        timeMs: 1000 / 30,
+      this._scheduleNextFrame()
-      })
+    }
+
+    // When re-initializing with an element that already has data (e.g. after
+    // hiding then showing the camera), 'loadeddata' will not fire again, so
+    // rely on readyState instead of a stale boolean flag.
+    if (this.videoElement!.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) {
+      startLoop()
     } else {
-      this.videoElement!.onloadeddata = () => {
+      this.onVideoLoaded = startLoop
-        this.videoElementLoaded = true
+      this.videoElement!.addEventListener('loadeddata', this.onVideoLoaded, {
-        this.timerWorker!.postMessage({
+        once: true,
-          id: SET_TIMEOUT,
+      })
-          timeMs: 1000 / 30,
-        })
-      }
     }
   }
 
   onTimerMessage(response: { data: { id: number } }) {
     if (response.data.id === TIMEOUT_TICK) {
-      this.process()
+      this.processing = this.process()
     }
   }
 
@@ -194,30 +261,47 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
    */
   async segment() {
     const startTimeMs = performance.now()
-    return new Promise<void>((resolve) => {
+    return new Promise<void>((resolve, reject) => {
-      this.imageSegmenter!.segmentForVideo(
+      try {
-        this.sourceImageData!,
+        this.imageSegmenter!.segmentForVideo(
-        startTimeMs,
+          this.sourceImageData!,
-        (result: ImageSegmenterResult) => {
+          startTimeMs,
-          this.imageSegmenterResult = result
+          (result: ImageSegmenterResult) => {
-          resolve()
+            try {
-        }
+              // The mask is only valid for the duration of this callback:
-      )
+              // MediaPipe frees the underlying WASM memory as soon as it
+              // returns, so the data must be copied out synchronously here.
+              this._applyMaskToAlphaChannel(result)
+              resolve()
+            } catch (error) {
+              reject(error)
+            }
+          }
+        )
+      } catch (error) {
+        reject(error)
+      }
     })
   }
 
-  /**
+  _applyMaskToAlphaChannel(result: ImageSegmenterResult) {
-   * TODO: future improvement with WebGL.
+    const categoryMask = result.categoryMask
-   */
+    if (!categoryMask) {
-  async blur() {
+      return
-    if (this.options.type !== 'blur') {
-      throw new Error('Blurring is only supported for blur background')
     }
-    const mask = this.imageSegmenterResult!.categoryMask!.getAsUint8Array()
+    const mask = categoryMask.getAsUint8Array()
-    for (let i = 0; i < mask.length; ++i) {
+    const alpha = this.segmentationMask!.data
-      this.segmentationMask!.data[i * 4 + 3] = 255 - mask[i]
+    const length = Math.min(mask.length, alpha.length / 4)
+    for (let i = 0; i < length; ++i) {
+      alpha[i * 4 + 3] = 255 - mask[i]
     }
+  }
 
+  /**
+   * Composite the segmentation mask over the output canvas: mask first, then
+   * the clear body, leaving the background to be filled by the caller.
+   */
+  _compositeMaskAndBody() {
     this.segmentationMaskCanvasCtx!.putImageData(this.segmentationMask!, 0, 0)
 
     this.outputCanvasCtx!.globalCompositeOperation = 'copy'
@@ -240,6 +324,16 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
     this.outputCanvasCtx!.globalCompositeOperation = 'source-in'
     this.outputCanvasCtx!.filter = 'none'
     this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0)
+  }
+
+  /**
+   * TODO: future improvement with WebGL.
+   */
+  async blur() {
+    if (this.options.type !== 'blur') {
+      throw new Error('Blurring is only supported for blur background')
+    }
+    this._compositeMaskAndBody()
 
     // Draw blurry background.
     this.outputCanvasCtx!.globalCompositeOperation = 'destination-over'
@@ -251,87 +345,145 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
    * TODO: future improvement with WebGL.
    */
   async drawVirtualBackground() {
-    const mask = this.imageSegmenterResult!.categoryMask!.getAsUint8Array()
+    this._compositeMaskAndBody()
-    for (let i = 0; i < mask.length; ++i) {
+
-      this.segmentationMask!.data[i * 4 + 3] = 255 - mask[i]
+    this.outputCanvasCtx!.globalCompositeOperation = 'destination-over'
+    this.outputCanvasCtx!.filter = 'none'
+    if (this._isVirtualBackgroundImageReady()) {
+      // Draw virtual background.
+      this.outputCanvasCtx!.drawImage(
+        this.virtualBackgroundImage!,
+        0,
+        0,
+        this.outputCanvas!.width,
+        this.outputCanvas!.height
+      )
+    } else {
+      // Image not decoded (yet, or failed to load): fall back to the raw
+      // video so the participant never appears over a black background.
+      this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0)
     }
+  }
 
-    this.segmentationMaskCanvasCtx!.putImageData(this.segmentationMask!, 0, 0)
+  /**
-
+   * Draw the raw video without any effect. Used when segmentation is broken,
+   * so the outgoing video keeps flowing instead of freezing on a stale frame.
+   */
+  _drawPassthroughFrame() {
     this.outputCanvasCtx!.globalCompositeOperation = 'copy'
-    this.outputCanvasCtx!.filter = 'blur(8px)'
-
-    // Put opacity mask.
-    this.outputCanvasCtx!.drawImage(
-      this.segmentationMaskCanvas!,
-      0,
-      0,
-      PROCESSING_WIDTH,
-      PROCESSING_HEIGHT,
-      0,
-      0,
-      this.videoElement!.videoWidth,
-      this.videoElement!.videoHeight
-    )
-
-    // Draw clear body.
-    this.outputCanvasCtx!.globalCompositeOperation = 'source-in'
     this.outputCanvasCtx!.filter = 'none'
     this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0)
-
-    // Draw virtual background.
-    this.outputCanvasCtx!.globalCompositeOperation = 'destination-over'
-    this.outputCanvasCtx!.drawImage(
-      this.virtualBackgroundImage!,
-      0,
-      0,
-      this.outputCanvas!.width,
-      this.outputCanvas!.height
-    )
   }
 
   async process() {
-    await this.sizeSource()
+    if (this.destroyed) {
-    await this.segment()
+      return
-
-    if (this.options.type === 'blur') {
-      await this.blur()
-    } else {
-      await this.drawVirtualBackground()
     }
-    this.timerWorker!.postMessage({
+    try {
+      this._syncOutputCanvasSize()
+
+      // No decoded frame available (e.g. right after a device switch): skip
+      // this tick rather than processing a 0x0 source.
+      if (
+        !this.videoElement ||
+        this.videoElement.videoWidth === 0 ||
+        this.videoElement.videoHeight === 0
+      ) {
+        this._scheduleNextFrame()
+        return
+      }
+
+      if (this.passthrough) {
+        this._drawPassthroughFrame()
+        this._scheduleNextFrame()
+        return
+      }
+
+      await this.sizeSource()
+      await this.segment()
+
+      if (this.destroyed) {
+        return
+      }
+
+      if (this.options.type === 'blur') {
+        await this.blur()
+      } else {
+        await this.drawVirtualBackground()
+      }
+      this.consecutiveErrors = 0
+    } catch (error) {
+      if (this.destroyed) {
+        return
+      }
+      this.consecutiveErrors += 1
+      if (this.consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) {
+        // Degrade to passthrough: a working camera without the effect is
+        // better than a permanently frozen frame.
+        this.passthrough = true
+        reportError('effects_processor_failure', error, {
+          context:
+            'Background processing failed repeatedly, falling back to unprocessed video',
+          consecutive_errors: this.consecutiveErrors,
+        })
+        this.imageSegmenter?.close()
+        this.imageSegmenter = undefined
+      }
+    }
+    this._scheduleNextFrame()
+  }
+
+  _scheduleNextFrame() {
+    if (this.destroyed) {
+      return
+    }
+    this.timerWorker?.postMessage({
       id: SET_TIMEOUT,
-      timeMs: 1000 / 30,
+      timeMs: FRAME_INTERVAL_MS,
     })
   }
 
-  _createMainCanvas() {
+  /**
-    this.outputCanvas = document.querySelector(
+   * Keep the output canvas in sync with the actual decoded video dimensions.
-      'canvas#background-blur-local'
+   * `MediaStreamTrack.getSettings()` can be incomplete or stale on Firefox,
-    ) as HTMLCanvasElement
+   * so the video element is the source of truth.
-    if (!this.outputCanvas) {
+   */
-      this.outputCanvas = this._createCanvas(
+  _syncOutputCanvasSize() {
-        BLUR_CANVAS_ID,
+    const width = this.videoElement?.videoWidth
-        this.sourceSettings!.width!,
+    const height = this.videoElement?.videoHeight
-        this.sourceSettings!.height!
+    if (!width || !height || !this.outputCanvas) {
-      )
+      return
     }
+    if (
+      this.outputCanvas.width !== width ||
+      this.outputCanvas.height !== height
+    ) {
+      this.outputCanvas.width = width
+      this.outputCanvas.height = height
+    }
+  }
+
+  _createMainCanvas() {
+    const width =
+      this.sourceSettings?.width || this.videoElement?.videoWidth || 1280
+    const height =
+      this.sourceSettings?.height || this.videoElement?.videoHeight || 720
+    this.outputCanvas = this._createCanvas(BLUR_CANVAS_ID, width, height)
     this.outputCanvasCtx = this.outputCanvas.getContext('2d')!
   }
 
   _createMaskCanvas() {
-    this.segmentationMaskCanvas = document.querySelector(
+    this.segmentationMaskCanvas = this._createCanvas(
-      `#${SEGMENTATION_MASK_CANVAS_ID}`
+      SEGMENTATION_MASK_CANVAS_ID,
-    ) as HTMLCanvasElement
+      PROCESSING_WIDTH,
-    if (!this.segmentationMaskCanvas) {
+      PROCESSING_HEIGHT
-      this.segmentationMaskCanvas = this._createCanvas(
+    )
-        SEGMENTATION_MASK_CANVAS_ID,
+    // getImageData is called on this canvas 30 times per second: opt out of
-        PROCESSING_WIDTH,
+    // GPU backing to avoid a costly readback on every frame.
-        PROCESSING_HEIGHT
+    this.segmentationMaskCanvasCtx = this.segmentationMaskCanvas.getContext(
-      )
+      '2d',
-    }
+      { willReadFrequently: true }
-    this.segmentationMaskCanvasCtx =
+    )!
-      this.segmentationMaskCanvas.getContext('2d')!
   }
 
   _createCanvas(id: string, width: number, height: number) {
@@ -348,11 +500,39 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
   }
 
   async destroy() {
+    this.destroyed = true
+
     this.timerWorker?.postMessage({
       id: CLEAR_TIMEOUT,
     })
 
+    // Let any in-flight frame finish before releasing the resources it uses,
+    // so segmentForVideo is never called on a closed segmenter.
+    try {
+      await this.processing
+    } catch {
+      // Failures are already handled inside process().
+    }
+    this.processing = undefined
+
+    if (this.onVideoLoaded && this.videoElement) {
+      this.videoElement.removeEventListener('loadeddata', this.onVideoLoaded)
+    }
+    this.onVideoLoaded = undefined
+
     this.timerWorker?.terminate()
+    this.timerWorker = undefined
+
     this.imageSegmenter?.close()
+    this.imageSegmenter = undefined
+
+    this.processedTrack?.stop()
+    this.processedTrack = undefined
+
+    this.outputCanvas = undefined
+    this.outputCanvasCtx = undefined
+    this.segmentationMaskCanvas = undefined
+    this.segmentationMaskCanvasCtx = undefined
+    this.sourceImageData = undefined
   }
 }