mirror of
https://github.com/suitenumerique/meet.git
synced 2026-09-03 14:17:59 +00:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 6265953858 | |||
| 0edac91793 |
@@ -11,7 +11,6 @@ and this project adheres to
|
|||||||
### Added
|
### Added
|
||||||
|
|
||||||
- ✨(frontend) add 1080p sending resolution option #1660
|
- ✨(frontend) add 1080p sending resolution option #1660
|
||||||
- ✨(backend) add Traefik support via configurable media-auth url header #1649
|
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
|
|
||||||
|
|||||||
@@ -1076,10 +1076,9 @@ class RecordingViewSet(
|
|||||||
|
|
||||||
def _auth_get_original_url(self, request):
|
def _auth_get_original_url(self, request):
|
||||||
"""
|
"""
|
||||||
Extracts and parses the original URL from the configured header.
|
Extracts and parses the original URL from the "HTTP_X_ORIGINAL_URL" header.
|
||||||
Raises PermissionDenied if the header is missing.
|
Raises PermissionDenied if the header is missing.
|
||||||
The original url is passed by the reverse proxy in the header named by the
|
The original url is passed by nginx in the "HTTP_X_ORIGINAL_URL" header.
|
||||||
MEDIA_AUTH_ORIGINAL_URL_HEADER setting, which defaults to "HTTP_X_ORIGINAL_URL".
|
|
||||||
See corresponding ingress configuration in Helm chart and read about the
|
See corresponding ingress configuration in Helm chart and read about the
|
||||||
nginx.ingress.kubernetes.io/auth-url annotation to understand how the Nginx ingress
|
nginx.ingress.kubernetes.io/auth-url annotation to understand how the Nginx ingress
|
||||||
is configured to do this.
|
is configured to do this.
|
||||||
@@ -1089,13 +1088,9 @@ class RecordingViewSet(
|
|||||||
reasons.
|
reasons.
|
||||||
"""
|
"""
|
||||||
# Extract the original URL from the request header
|
# Extract the original URL from the request header
|
||||||
original_url = request.META.get(settings.MEDIA_AUTH_ORIGINAL_URL_HEADER)
|
original_url = request.META.get("HTTP_X_ORIGINAL_URL")
|
||||||
if not original_url:
|
if not original_url:
|
||||||
logger.warning(
|
logger.warning("Missing HTTP_X_ORIGINAL_URL header in subrequest")
|
||||||
"Missing %s header in subrequest. Set MEDIA_AUTH_ORIGINAL_URL_HEADER "
|
|
||||||
"to the header your reverse proxy sends.",
|
|
||||||
settings.MEDIA_AUTH_ORIGINAL_URL_HEADER,
|
|
||||||
)
|
|
||||||
raise drf_exceptions.PermissionDenied()
|
raise drf_exceptions.PermissionDenied()
|
||||||
|
|
||||||
logger.debug("Original url: '%s'", original_url)
|
logger.debug("Original url: '%s'", original_url)
|
||||||
@@ -1420,8 +1415,7 @@ class FileViewSet(
|
|||||||
Authorize access based on the original URL of an Nginx subrequest
|
Authorize access based on the original URL of an Nginx subrequest
|
||||||
and user permissions. Returns a dictionary of URL parameters if authorized.
|
and user permissions. Returns a dictionary of URL parameters if authorized.
|
||||||
|
|
||||||
The original url is passed by the reverse proxy in the header named by the
|
The original url is passed by nginx in the "HTTP_X_ORIGINAL_URL" header.
|
||||||
MEDIA_AUTH_ORIGINAL_URL_HEADER setting, which defaults to "HTTP_X_ORIGINAL_URL".
|
|
||||||
See corresponding ingress configuration in Helm chart and read about the
|
See corresponding ingress configuration in Helm chart and read about the
|
||||||
nginx.ingress.kubernetes.io/auth-url annotation to understand how the Nginx ingress
|
nginx.ingress.kubernetes.io/auth-url annotation to understand how the Nginx ingress
|
||||||
is configured to do this.
|
is configured to do this.
|
||||||
@@ -1440,13 +1434,9 @@ class FileViewSet(
|
|||||||
- PermissionDenied if authorization fails.
|
- PermissionDenied if authorization fails.
|
||||||
"""
|
"""
|
||||||
# Extract the original URL from the request header
|
# Extract the original URL from the request header
|
||||||
original_url = request.META.get(settings.MEDIA_AUTH_ORIGINAL_URL_HEADER)
|
original_url = request.META.get("HTTP_X_ORIGINAL_URL")
|
||||||
if not original_url:
|
if not original_url:
|
||||||
logger.warning(
|
logger.warning("Missing HTTP_X_ORIGINAL_URL header in subrequest")
|
||||||
"Missing %s header in subrequest. Set MEDIA_AUTH_ORIGINAL_URL_HEADER "
|
|
||||||
"to the header your reverse proxy sends.",
|
|
||||||
settings.MEDIA_AUTH_ORIGINAL_URL_HEADER,
|
|
||||||
)
|
|
||||||
raise drf_exceptions.PermissionDenied()
|
raise drf_exceptions.PermissionDenied()
|
||||||
|
|
||||||
parsed_url = urlparse(original_url)
|
parsed_url = urlparse(original_url)
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ from urllib.parse import quote, urlparse
|
|||||||
|
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.core.files.storage import default_storage
|
from django.core.files.storage import default_storage
|
||||||
from django.test import override_settings
|
|
||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
@@ -144,59 +143,3 @@ def test_api_files_media_auth_own_file_deleted():
|
|||||||
)
|
)
|
||||||
|
|
||||||
assert response.status_code == 403
|
assert response.status_code == 403
|
||||||
|
|
||||||
|
|
||||||
@override_settings(MEDIA_AUTH_ORIGINAL_URL_HEADER="HTTP_X_FORWARDED_URI")
|
|
||||||
def test_api_files_media_auth_custom_original_url_header():
|
|
||||||
"""
|
|
||||||
Authorization should honour the configured original-url header.
|
|
||||||
|
|
||||||
Covers the attachment subrequest path, which resolves the header separately
|
|
||||||
from the recording one. Reverse proxies other than nginx-ingress use
|
|
||||||
different headers: Traefik's ForwardAuth sends X-Forwarded-Uri and cannot
|
|
||||||
emit X-Original-URL at all.
|
|
||||||
"""
|
|
||||||
user = factories.UserFactory()
|
|
||||||
|
|
||||||
file = factories.FileFactory(
|
|
||||||
type=models.FileTypeChoices.BACKGROUND_IMAGE,
|
|
||||||
update_upload_state=models.FileUploadStateChoices.READY,
|
|
||||||
creator=user,
|
|
||||||
)
|
|
||||||
|
|
||||||
client = APIClient()
|
|
||||||
client.force_login(user)
|
|
||||||
|
|
||||||
default_storage.save(file.file_key, BytesIO(b"my prose"))
|
|
||||||
|
|
||||||
original_url = f"http://localhost/media/{file.file_key:s}"
|
|
||||||
response = client.get(
|
|
||||||
"/api/v1.0/files/media-auth/", HTTP_X_FORWARDED_URI=original_url
|
|
||||||
)
|
|
||||||
|
|
||||||
assert response.status_code == 200
|
|
||||||
assert "AWS4-HMAC-SHA256 Credential=" in response["Authorization"]
|
|
||||||
|
|
||||||
|
|
||||||
@override_settings(MEDIA_AUTH_ORIGINAL_URL_HEADER="HTTP_X_FORWARDED_URI")
|
|
||||||
def test_api_files_media_auth_default_header_ignored_when_reconfigured():
|
|
||||||
"""
|
|
||||||
Only the configured header should be honoured, never a hardcoded fallback.
|
|
||||||
"""
|
|
||||||
user = factories.UserFactory()
|
|
||||||
|
|
||||||
file = factories.FileFactory(
|
|
||||||
type=models.FileTypeChoices.BACKGROUND_IMAGE,
|
|
||||||
update_upload_state=models.FileUploadStateChoices.READY,
|
|
||||||
creator=user,
|
|
||||||
)
|
|
||||||
|
|
||||||
client = APIClient()
|
|
||||||
client.force_login(user)
|
|
||||||
|
|
||||||
original_url = f"http://localhost/media/{file.file_key:s}"
|
|
||||||
response = client.get(
|
|
||||||
"/api/v1.0/files/media-auth/", HTTP_X_ORIGINAL_URL=original_url
|
|
||||||
)
|
|
||||||
|
|
||||||
assert response.status_code == 403
|
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ from uuid import uuid4
|
|||||||
|
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.core.files.storage import default_storage
|
from django.core.files.storage import default_storage
|
||||||
from django.test import override_settings
|
|
||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
@@ -283,63 +282,3 @@ def test_api_recordings_media_auth_success_administrator(mode):
|
|||||||
timeout=1,
|
timeout=1,
|
||||||
)
|
)
|
||||||
assert response.content.decode("utf-8") == "my prose"
|
assert response.content.decode("utf-8") == "my prose"
|
||||||
|
|
||||||
|
|
||||||
def test_api_recordings_media_auth_missing_header():
|
|
||||||
"""
|
|
||||||
Test that a subrequest without the configured original-url header is rejected.
|
|
||||||
"""
|
|
||||||
user = UserFactory()
|
|
||||||
|
|
||||||
client = APIClient()
|
|
||||||
client.force_login(user)
|
|
||||||
|
|
||||||
response = client.get("/api/v1.0/recordings/media-auth/")
|
|
||||||
|
|
||||||
assert response.status_code == 403
|
|
||||||
|
|
||||||
|
|
||||||
@override_settings(MEDIA_AUTH_ORIGINAL_URL_HEADER="HTTP_X_FORWARDED_URI")
|
|
||||||
def test_api_recordings_media_auth_custom_original_url_header():
|
|
||||||
"""
|
|
||||||
Test that the header carrying the original URL can be configured.
|
|
||||||
|
|
||||||
Reverse proxies other than nginx-ingress use different headers: Traefik's
|
|
||||||
ForwardAuth sends X-Forwarded-Uri and cannot emit X-Original-URL at all.
|
|
||||||
"""
|
|
||||||
user = UserFactory()
|
|
||||||
|
|
||||||
client = APIClient()
|
|
||||||
client.force_login(user)
|
|
||||||
|
|
||||||
original_url = f"http://localhost/media/recordings/{uuid4()!s}.mp4"
|
|
||||||
|
|
||||||
response = client.get(
|
|
||||||
"/api/v1.0/recordings/media-auth/", HTTP_X_FORWARDED_URI=original_url
|
|
||||||
)
|
|
||||||
|
|
||||||
# The header was read and parsed: we get as far as looking the recording up,
|
|
||||||
# rather than being rejected for a missing header.
|
|
||||||
assert response.status_code == 404
|
|
||||||
|
|
||||||
|
|
||||||
@override_settings(MEDIA_AUTH_ORIGINAL_URL_HEADER="HTTP_X_FORWARDED_URI")
|
|
||||||
def test_api_recordings_media_auth_default_header_ignored_when_reconfigured():
|
|
||||||
"""
|
|
||||||
Test that only the configured header is honoured.
|
|
||||||
|
|
||||||
Guards against the header being read from a hardcoded name in parallel with
|
|
||||||
the setting.
|
|
||||||
"""
|
|
||||||
user = UserFactory()
|
|
||||||
|
|
||||||
client = APIClient()
|
|
||||||
client.force_login(user)
|
|
||||||
|
|
||||||
original_url = f"http://localhost/media/recordings/{uuid4()!s}.mp4"
|
|
||||||
|
|
||||||
response = client.get(
|
|
||||||
"/api/v1.0/recordings/media-auth/", HTTP_X_ORIGINAL_URL=original_url
|
|
||||||
)
|
|
||||||
|
|
||||||
assert response.status_code == 403
|
|
||||||
|
|||||||
@@ -129,15 +129,6 @@ class Base(Configuration):
|
|||||||
MEDIA_BASE_URL = values.Value(
|
MEDIA_BASE_URL = values.Value(
|
||||||
"", environ_name="MEDIA_BASE_URL", environ_prefix=None
|
"", environ_name="MEDIA_BASE_URL", environ_prefix=None
|
||||||
)
|
)
|
||||||
# Header the reverse proxy uses to pass the original request URL to the
|
|
||||||
# media-auth subrequest views. nginx-ingress sends X-Original-URL, which is
|
|
||||||
# the default. Other proxies use different headers -- Traefik's ForwardAuth,
|
|
||||||
# for instance, sends X-Forwarded-Uri and cannot emit X-Original-URL at all.
|
|
||||||
MEDIA_AUTH_ORIGINAL_URL_HEADER = values.Value(
|
|
||||||
default="HTTP_X_ORIGINAL_URL",
|
|
||||||
environ_name="MEDIA_AUTH_ORIGINAL_URL_HEADER",
|
|
||||||
environ_prefix=None,
|
|
||||||
)
|
|
||||||
|
|
||||||
SITE_ID = 1
|
SITE_ID = 1
|
||||||
|
|
||||||
|
|||||||
@@ -8,28 +8,6 @@ const IGNORED_EXCEPTION_PATTERNS = [
|
|||||||
// the close reason is already logged by the SDK.
|
// the close reason is already logged by the SDK.
|
||||||
// See: https://github.com/livekit/client-sdk-js/issues/2062
|
// See: https://github.com/livekit/client-sdk-js/issues/2062
|
||||||
/^Event captured as exception with keys: isTrusted$/,
|
/^Event captured as exception with keys: isTrusted$/,
|
||||||
// MediaPipe's WASM writes its native logs to stderr, which Emscripten
|
|
||||||
// routes to console.error, which PostHog's console capture then promotes
|
|
||||||
// to an $exception — even though nothing was thrown. Two flavors:
|
|
||||||
//
|
|
||||||
// 1. "INFO: ..." lines are purely informational. In particular
|
|
||||||
// "INFO: Created TensorFlow Lite XNNPACK delegate for CPU." is a
|
|
||||||
// SUCCESS message: TFLite prints it when it lazily initializes CPU
|
|
||||||
// inference on the first segmented frame. It fires on every effects
|
|
||||||
// init, on every browser and delegate (the GPU delegate still
|
|
||||||
// instantiates the CPU/XNNPACK delegate for non-delegated ops), so it
|
|
||||||
// was our single noisiest "error" while carrying zero signal.
|
|
||||||
/^INFO: /,
|
|
||||||
//
|
|
||||||
// 2. absl-formatted log lines, e.g.
|
|
||||||
// "E0901 19:21:45.443000 1880752 gl_graph_runner_internal.cc:260]
|
|
||||||
// StartGraph failed: ..."
|
|
||||||
// (severity letter E/W/I/F + MMDD + timestamp). These are the stderr
|
|
||||||
// *copies* of failures that MediaPipe also raises as real JS
|
|
||||||
// exceptions, which we already capture via reportError / thrown
|
|
||||||
// errors. Dropping them de-duplicates each incident (previously
|
|
||||||
// counted 2-3x) without losing the actual error report.
|
|
||||||
/^[EWIF]\d{4} \d{2}:\d{2}:\d{2}\./,
|
|
||||||
]
|
]
|
||||||
|
|
||||||
const shouldIgnoreException = (value: unknown): boolean =>
|
const shouldIgnoreException = (value: unknown): boolean =>
|
||||||
|
|||||||
+110
-304
@@ -17,7 +17,7 @@ import {
|
|||||||
type ProcessorType,
|
type ProcessorType,
|
||||||
MEDIAPIPE_PATH_WASM,
|
MEDIAPIPE_PATH_WASM,
|
||||||
} from '.'
|
} from '.'
|
||||||
import { captureEvent, reportError } from '@/features/analytics/telemetry'
|
import { captureEvent } from '@/features/analytics/telemetry.ts'
|
||||||
|
|
||||||
const PROCESSING_WIDTH = 256
|
const PROCESSING_WIDTH = 256
|
||||||
const PROCESSING_HEIGHT = 144
|
const PROCESSING_HEIGHT = 144
|
||||||
@@ -26,40 +26,6 @@ const SEGMENTATION_MASK_CANVAS_ID = 'background-blur-local-segmentation'
|
|||||||
const BLUR_CANVAS_ID = 'background-blur-local'
|
const BLUR_CANVAS_ID = 'background-blur-local'
|
||||||
|
|
||||||
const DEFAULT_BLUR = '10'
|
const DEFAULT_BLUR = '10'
|
||||||
const CONCEALING_BLUR = '25'
|
|
||||||
const FRAME_INTERVAL_MS = 1000 / 30
|
|
||||||
|
|
||||||
// After this many consecutive failed frames, stop segmenting and fall back to
|
|
||||||
// publishing a fully blurred frame: the user keeps a live camera instead of a
|
|
||||||
// frozen one, without ever exposing the surroundings they chose to conceal.
|
|
||||||
const MAX_CONSECUTIVE_ERRORS = 5
|
|
||||||
|
|
||||||
let webgl2Supported: boolean | undefined
|
|
||||||
|
|
||||||
/**
|
|
||||||
* MediaPipe's ImageSegmenter requires a WebGL2 context on the web even with
|
|
||||||
* `delegate: 'CPU'` (only inference runs on CPU; the mask post-processing in
|
|
||||||
* TensorsToSegmentationCalculator is GL-based). Without this check, machines
|
|
||||||
* with WebGL disabled or blocklisted fail at StartGraph with
|
|
||||||
* `emscripten_webgl_create_context() returned error 0`.
|
|
||||||
*
|
|
||||||
* The result is cached and the probe context is explicitly released so that
|
|
||||||
* repeated support checks do not count against the browser's limit on live
|
|
||||||
* WebGL contexts.
|
|
||||||
*/
|
|
||||||
const isWebGL2Supported = () => {
|
|
||||||
if (webgl2Supported === undefined) {
|
|
||||||
try {
|
|
||||||
const canvas = document.createElement('canvas')
|
|
||||||
const gl = canvas.getContext('webgl2')
|
|
||||||
webgl2Supported = !!gl
|
|
||||||
gl?.getExtension('WEBGL_lose_context')?.loseContext()
|
|
||||||
} catch {
|
|
||||||
webgl2Supported = false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return webgl2Supported
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This implementation of video blurring is made to be run on CPU for browser that are
|
* This implementation of video blurring is made to be run on CPU for browser that are
|
||||||
@@ -76,12 +42,14 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
|
|||||||
source?: MediaStreamTrack
|
source?: MediaStreamTrack
|
||||||
sourceSettings?: MediaTrackSettings
|
sourceSettings?: MediaTrackSettings
|
||||||
videoElement?: HTMLVideoElement
|
videoElement?: HTMLVideoElement
|
||||||
|
videoElementLoaded?: boolean
|
||||||
|
|
||||||
// Canvas containing the video processing result, of which we extract as stream.
|
// Canvas containing the video processing result, of which we extract as stream.
|
||||||
outputCanvas?: HTMLCanvasElement
|
outputCanvas?: HTMLCanvasElement
|
||||||
outputCanvasCtx?: CanvasRenderingContext2D
|
outputCanvasCtx?: CanvasRenderingContext2D
|
||||||
|
|
||||||
imageSegmenter?: ImageSegmenter
|
imageSegmenter?: ImageSegmenter
|
||||||
|
imageSegmenterResult?: ImageSegmenterResult
|
||||||
|
|
||||||
// Canvas used for resizing video source and projecting mask.
|
// Canvas used for resizing video source and projecting mask.
|
||||||
segmentationMaskCanvas?: HTMLCanvasElement
|
segmentationMaskCanvas?: HTMLCanvasElement
|
||||||
@@ -98,13 +66,6 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
|
|||||||
type: ProcessorType
|
type: ProcessorType
|
||||||
virtualBackgroundImage?: HTMLImageElement
|
virtualBackgroundImage?: HTMLImageElement
|
||||||
|
|
||||||
private virtualBackgroundImagePath?: string
|
|
||||||
private destroyed = false
|
|
||||||
private degraded = false
|
|
||||||
private consecutiveErrors = 0
|
|
||||||
private processing?: Promise<void>
|
|
||||||
private onVideoLoaded?: () => void
|
|
||||||
|
|
||||||
constructor(opts: ProcessorConfig) {
|
constructor(opts: ProcessorConfig) {
|
||||||
this.name = 'blur'
|
this.name = 'blur'
|
||||||
this.options = opts
|
this.options = opts
|
||||||
@@ -112,10 +73,7 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
static get isSupported() {
|
static get isSupported() {
|
||||||
return (
|
return navigator.userAgent.toLowerCase().includes('firefox')
|
||||||
navigator.userAgent.toLowerCase().includes('firefox') &&
|
|
||||||
isWebGL2Supported()
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async init(opts: ProcessorOptions<Track.Kind>) {
|
async init(opts: ProcessorOptions<Track.Kind>) {
|
||||||
@@ -123,10 +81,6 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
|
|||||||
throw new Error('Element is required for processing')
|
throw new Error('Element is required for processing')
|
||||||
}
|
}
|
||||||
|
|
||||||
this.destroyed = false
|
|
||||||
this.degraded = false
|
|
||||||
this.consecutiveErrors = 0
|
|
||||||
|
|
||||||
this.source = opts.track as MediaStreamTrack
|
this.source = opts.track as MediaStreamTrack
|
||||||
this.sourceSettings = this.source!.getSettings()
|
this.sourceSettings = this.source!.getSettings()
|
||||||
this.videoElement = opts.element as HTMLVideoElement
|
this.videoElement = opts.element as HTMLVideoElement
|
||||||
@@ -143,56 +97,26 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
|
|||||||
this.processedTrack = tracks[0]
|
this.processedTrack = tracks[0]
|
||||||
|
|
||||||
this.segmentationMask = new ImageData(PROCESSING_WIDTH, PROCESSING_HEIGHT)
|
this.segmentationMask = new ImageData(PROCESSING_WIDTH, PROCESSING_HEIGHT)
|
||||||
|
|
||||||
const t0 = performance.now()
|
|
||||||
await this.initSegmenter()
|
await this.initSegmenter()
|
||||||
const segmenterInitMs = Math.round(performance.now() - t0)
|
|
||||||
|
|
||||||
this._initWorker()
|
this._initWorker()
|
||||||
|
|
||||||
captureEvent('legacy-background-processor', {
|
captureEvent('firefox-blurring-init', {})
|
||||||
effect_type: this.options.type,
|
|
||||||
hw_concurrency: navigator.hardwareConcurrency,
|
|
||||||
video_width: this.videoElement?.videoWidth,
|
|
||||||
video_height: this.videoElement?.videoHeight,
|
|
||||||
segmenter_init_ms: segmenterInitMs,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
_initVirtualBackgroundImage() {
|
_initVirtualBackgroundImage() {
|
||||||
if (this.options.type !== 'virtual' || !this.options.imagePath) {
|
if (this.options.type !== 'virtual') {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if (
|
const needsUpdate =
|
||||||
|
this.options.imagePath &&
|
||||||
this.virtualBackgroundImage &&
|
this.virtualBackgroundImage &&
|
||||||
this.virtualBackgroundImagePath === this.options.imagePath
|
this.virtualBackgroundImage.src !== this.options.imagePath
|
||||||
) {
|
if (this.options.imagePath || needsUpdate) {
|
||||||
return
|
this.virtualBackgroundImage = document.createElement('img')
|
||||||
|
this.virtualBackgroundImage.crossOrigin = 'anonymous'
|
||||||
|
this.virtualBackgroundImage.src = this.options.imagePath!
|
||||||
}
|
}
|
||||||
|
|
||||||
const image = document.createElement('img')
|
|
||||||
image.crossOrigin = 'anonymous'
|
|
||||||
image.src = this.options.imagePath
|
|
||||||
// Surface load failures once instead of letting drawImage throw on a
|
|
||||||
// broken image inside the processing loop.
|
|
||||||
image.decode().catch((error) => {
|
|
||||||
reportError('effects_processor_failure', error, {
|
|
||||||
context: 'Failed to load virtual background image',
|
|
||||||
image_path:
|
|
||||||
this.options.type === 'virtual' ? this.options.imagePath : undefined,
|
|
||||||
})
|
|
||||||
})
|
|
||||||
this.virtualBackgroundImage = image
|
|
||||||
this.virtualBackgroundImagePath = this.options.imagePath
|
|
||||||
}
|
|
||||||
|
|
||||||
_isVirtualBackgroundImageReady() {
|
|
||||||
return (
|
|
||||||
!!this.virtualBackgroundImage &&
|
|
||||||
this.virtualBackgroundImage.complete &&
|
|
||||||
this.virtualBackgroundImage.naturalWidth > 0
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async update(opts: ProcessorConfig): Promise<void> {
|
async update(opts: ProcessorConfig): Promise<void> {
|
||||||
@@ -205,26 +129,26 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
|
|||||||
name: 'Blurring',
|
name: 'Blurring',
|
||||||
})
|
})
|
||||||
this.timerWorker.onmessage = (data) => this.onTimerMessage(data)
|
this.timerWorker.onmessage = (data) => this.onTimerMessage(data)
|
||||||
|
// When hiding camera then showing it again, the onloadeddata callback is not fired again.
|
||||||
const startLoop = () => {
|
if (this.videoElementLoaded) {
|
||||||
this.onVideoLoaded = undefined
|
this.timerWorker!.postMessage({
|
||||||
this._syncOutputCanvasSize()
|
id: SET_TIMEOUT,
|
||||||
this._scheduleNextFrame()
|
timeMs: 1000 / 30,
|
||||||
}
|
|
||||||
|
|
||||||
if (this.videoElement!.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) {
|
|
||||||
startLoop()
|
|
||||||
} else {
|
|
||||||
this.onVideoLoaded = startLoop
|
|
||||||
this.videoElement!.addEventListener('loadeddata', this.onVideoLoaded, {
|
|
||||||
once: true,
|
|
||||||
})
|
})
|
||||||
|
} else {
|
||||||
|
this.videoElement!.onloadeddata = () => {
|
||||||
|
this.videoElementLoaded = true
|
||||||
|
this.timerWorker!.postMessage({
|
||||||
|
id: SET_TIMEOUT,
|
||||||
|
timeMs: 1000 / 30,
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
onTimerMessage(response: { data: { id: number } }) {
|
onTimerMessage(response: { data: { id: number } }) {
|
||||||
if (response.data.id === TIMEOUT_TICK) {
|
if (response.data.id === TIMEOUT_TICK) {
|
||||||
this.processing = this.process()
|
this.process()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -270,47 +194,30 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
|
|||||||
*/
|
*/
|
||||||
async segment() {
|
async segment() {
|
||||||
const startTimeMs = performance.now()
|
const startTimeMs = performance.now()
|
||||||
return new Promise<void>((resolve, reject) => {
|
return new Promise<void>((resolve) => {
|
||||||
try {
|
this.imageSegmenter!.segmentForVideo(
|
||||||
this.imageSegmenter!.segmentForVideo(
|
this.sourceImageData!,
|
||||||
this.sourceImageData!,
|
startTimeMs,
|
||||||
startTimeMs,
|
(result: ImageSegmenterResult) => {
|
||||||
(result: ImageSegmenterResult) => {
|
this.imageSegmenterResult = result
|
||||||
try {
|
resolve()
|
||||||
// The mask is only valid for the duration of this callback:
|
}
|
||||||
// MediaPipe frees the underlying WASM memory as soon as it
|
)
|
||||||
// returns, so the data must be copied out synchronously here.
|
|
||||||
this._applyMaskToAlphaChannel(result)
|
|
||||||
resolve()
|
|
||||||
} catch (error) {
|
|
||||||
reject(error)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
)
|
|
||||||
} catch (error) {
|
|
||||||
reject(error)
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
_applyMaskToAlphaChannel(result: ImageSegmenterResult) {
|
|
||||||
const categoryMask = result.categoryMask
|
|
||||||
if (!categoryMask) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
const mask = categoryMask.getAsUint8Array()
|
|
||||||
const alpha = this.segmentationMask!.data
|
|
||||||
const length = Math.min(mask.length, alpha.length / 4)
|
|
||||||
for (let i = 0; i < length; ++i) {
|
|
||||||
alpha[i * 4 + 3] = 255 - mask[i]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Composite the segmentation mask over the output canvas: mask first, then
|
* TODO: future improvement with WebGL.
|
||||||
* the clear body, leaving the background to be filled by the caller.
|
|
||||||
*/
|
*/
|
||||||
_compositeMaskAndBody() {
|
async blur() {
|
||||||
|
if (this.options.type !== 'blur') {
|
||||||
|
throw new Error('Blurring is only supported for blur background')
|
||||||
|
}
|
||||||
|
const mask = this.imageSegmenterResult!.categoryMask!.getAsUint8Array()
|
||||||
|
for (let i = 0; i < mask.length; ++i) {
|
||||||
|
this.segmentationMask!.data[i * 4 + 3] = 255 - mask[i]
|
||||||
|
}
|
||||||
|
|
||||||
this.segmentationMaskCanvasCtx!.putImageData(this.segmentationMask!, 0, 0)
|
this.segmentationMaskCanvasCtx!.putImageData(this.segmentationMask!, 0, 0)
|
||||||
|
|
||||||
this.outputCanvasCtx!.globalCompositeOperation = 'copy'
|
this.outputCanvasCtx!.globalCompositeOperation = 'copy'
|
||||||
@@ -333,16 +240,6 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
|
|||||||
this.outputCanvasCtx!.globalCompositeOperation = 'source-in'
|
this.outputCanvasCtx!.globalCompositeOperation = 'source-in'
|
||||||
this.outputCanvasCtx!.filter = 'none'
|
this.outputCanvasCtx!.filter = 'none'
|
||||||
this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0)
|
this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0)
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* TODO: future improvement with WebGL.
|
|
||||||
*/
|
|
||||||
async blur() {
|
|
||||||
if (this.options.type !== 'blur') {
|
|
||||||
throw new Error('Blurring is only supported for blur background')
|
|
||||||
}
|
|
||||||
this._compositeMaskAndBody()
|
|
||||||
|
|
||||||
// Draw blurry background.
|
// Draw blurry background.
|
||||||
this.outputCanvasCtx!.globalCompositeOperation = 'destination-over'
|
this.outputCanvasCtx!.globalCompositeOperation = 'destination-over'
|
||||||
@@ -354,150 +251,87 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
|
|||||||
* TODO: future improvement with WebGL.
|
* TODO: future improvement with WebGL.
|
||||||
*/
|
*/
|
||||||
async drawVirtualBackground() {
|
async drawVirtualBackground() {
|
||||||
this._compositeMaskAndBody()
|
const mask = this.imageSegmenterResult!.categoryMask!.getAsUint8Array()
|
||||||
|
for (let i = 0; i < mask.length; ++i) {
|
||||||
this.outputCanvasCtx!.globalCompositeOperation = 'destination-over'
|
this.segmentationMask!.data[i * 4 + 3] = 255 - mask[i]
|
||||||
this.outputCanvasCtx!.filter = 'none'
|
|
||||||
if (this._isVirtualBackgroundImageReady()) {
|
|
||||||
// Draw virtual background.
|
|
||||||
this.outputCanvasCtx!.drawImage(
|
|
||||||
this.virtualBackgroundImage!,
|
|
||||||
0,
|
|
||||||
0,
|
|
||||||
this.outputCanvas!.width,
|
|
||||||
this.outputCanvas!.height
|
|
||||||
)
|
|
||||||
} else {
|
|
||||||
// Image not decoded (yet, or failed to load): fill the background with
|
|
||||||
// a heavy blur instead. Never fall back to the raw video here — the
|
|
||||||
// user selected this effect to conceal their surroundings, so the
|
|
||||||
// fallback must keep concealing them.
|
|
||||||
this.outputCanvasCtx!.filter = `blur(${CONCEALING_BLUR}px)`
|
|
||||||
this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0)
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
this.segmentationMaskCanvasCtx!.putImageData(this.segmentationMask!, 0, 0)
|
||||||
* Draw the whole frame heavily blurred (person included). Used when
|
|
||||||
* segmentation is broken: the outgoing video keeps flowing instead of
|
|
||||||
* freezing on a stale frame, while the surroundings the user chose to
|
|
||||||
* conceal stay concealed. Requires no segmenter, only one filtered draw.
|
|
||||||
*/
|
|
||||||
_drawDegradedFrame() {
|
|
||||||
this.outputCanvasCtx!.globalCompositeOperation = 'copy'
|
this.outputCanvasCtx!.globalCompositeOperation = 'copy'
|
||||||
this.outputCanvasCtx!.filter = `blur(${CONCEALING_BLUR}px)`
|
this.outputCanvasCtx!.filter = 'blur(8px)'
|
||||||
this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0)
|
|
||||||
|
// Put opacity mask.
|
||||||
|
this.outputCanvasCtx!.drawImage(
|
||||||
|
this.segmentationMaskCanvas!,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
PROCESSING_WIDTH,
|
||||||
|
PROCESSING_HEIGHT,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
this.videoElement!.videoWidth,
|
||||||
|
this.videoElement!.videoHeight
|
||||||
|
)
|
||||||
|
|
||||||
|
// Draw clear body.
|
||||||
|
this.outputCanvasCtx!.globalCompositeOperation = 'source-in'
|
||||||
this.outputCanvasCtx!.filter = 'none'
|
this.outputCanvasCtx!.filter = 'none'
|
||||||
|
this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0)
|
||||||
|
|
||||||
|
// Draw virtual background.
|
||||||
|
this.outputCanvasCtx!.globalCompositeOperation = 'destination-over'
|
||||||
|
this.outputCanvasCtx!.drawImage(
|
||||||
|
this.virtualBackgroundImage!,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
this.outputCanvas!.width,
|
||||||
|
this.outputCanvas!.height
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
async process() {
|
async process() {
|
||||||
if (this.destroyed) {
|
await this.sizeSource()
|
||||||
return
|
await this.segment()
|
||||||
|
|
||||||
|
if (this.options.type === 'blur') {
|
||||||
|
await this.blur()
|
||||||
|
} else {
|
||||||
|
await this.drawVirtualBackground()
|
||||||
}
|
}
|
||||||
try {
|
this.timerWorker!.postMessage({
|
||||||
this._syncOutputCanvasSize()
|
|
||||||
// No decoded frame available (e.g. right after a device switch): skip
|
|
||||||
// this tick rather than processing a 0x0 source.
|
|
||||||
if (
|
|
||||||
!this.videoElement ||
|
|
||||||
this.videoElement.videoWidth === 0 ||
|
|
||||||
this.videoElement.videoHeight === 0
|
|
||||||
) {
|
|
||||||
this._scheduleNextFrame()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if (this.degraded) {
|
|
||||||
this._drawDegradedFrame()
|
|
||||||
this._scheduleNextFrame()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
await this.sizeSource()
|
|
||||||
await this.segment()
|
|
||||||
|
|
||||||
if (this.destroyed) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if (this.options.type === 'blur') {
|
|
||||||
await this.blur()
|
|
||||||
} else {
|
|
||||||
await this.drawVirtualBackground()
|
|
||||||
}
|
|
||||||
this.consecutiveErrors = 0
|
|
||||||
} catch (error) {
|
|
||||||
if (this.destroyed) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
this.consecutiveErrors += 1
|
|
||||||
if (this.consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) {
|
|
||||||
// Degrade to a fully blurred frame: a live camera beats a frozen
|
|
||||||
// one, and concealment must survive the failure.
|
|
||||||
this.degraded = true
|
|
||||||
reportError('effects_processor_failure', error, {
|
|
||||||
context:
|
|
||||||
'Background processing failed repeatedly, falling back to fully blurred video',
|
|
||||||
consecutive_errors: this.consecutiveErrors,
|
|
||||||
})
|
|
||||||
this.imageSegmenter?.close()
|
|
||||||
this.imageSegmenter = undefined
|
|
||||||
}
|
|
||||||
}
|
|
||||||
this._scheduleNextFrame()
|
|
||||||
}
|
|
||||||
|
|
||||||
_scheduleNextFrame() {
|
|
||||||
if (this.destroyed) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
this.timerWorker?.postMessage({
|
|
||||||
id: SET_TIMEOUT,
|
id: SET_TIMEOUT,
|
||||||
timeMs: FRAME_INTERVAL_MS,
|
timeMs: 1000 / 30,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Keep the output canvas in sync with the actual decoded video dimensions.
|
|
||||||
* `MediaStreamTrack.getSettings()` can be incomplete or stale on Firefox,
|
|
||||||
* so the video element is the source of truth.
|
|
||||||
*/
|
|
||||||
_syncOutputCanvasSize() {
|
|
||||||
const width = this.videoElement?.videoWidth
|
|
||||||
const height = this.videoElement?.videoHeight
|
|
||||||
if (!width || !height || !this.outputCanvas) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if (
|
|
||||||
this.outputCanvas.width !== width ||
|
|
||||||
this.outputCanvas.height !== height
|
|
||||||
) {
|
|
||||||
this.outputCanvas.width = width
|
|
||||||
this.outputCanvas.height = height
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
_createMainCanvas() {
|
_createMainCanvas() {
|
||||||
const width =
|
this.outputCanvas = document.querySelector(
|
||||||
this.sourceSettings?.width || this.videoElement?.videoWidth || 1280
|
'canvas#background-blur-local'
|
||||||
const height =
|
) as HTMLCanvasElement
|
||||||
this.sourceSettings?.height || this.videoElement?.videoHeight || 720
|
if (!this.outputCanvas) {
|
||||||
this.outputCanvas = this._createCanvas(BLUR_CANVAS_ID, width, height)
|
this.outputCanvas = this._createCanvas(
|
||||||
|
BLUR_CANVAS_ID,
|
||||||
|
this.sourceSettings!.width!,
|
||||||
|
this.sourceSettings!.height!
|
||||||
|
)
|
||||||
|
}
|
||||||
this.outputCanvasCtx = this.outputCanvas.getContext('2d')!
|
this.outputCanvasCtx = this.outputCanvas.getContext('2d')!
|
||||||
}
|
}
|
||||||
|
|
||||||
_createMaskCanvas() {
|
_createMaskCanvas() {
|
||||||
this.segmentationMaskCanvas = this._createCanvas(
|
this.segmentationMaskCanvas = document.querySelector(
|
||||||
SEGMENTATION_MASK_CANVAS_ID,
|
`#${SEGMENTATION_MASK_CANVAS_ID}`
|
||||||
PROCESSING_WIDTH,
|
) as HTMLCanvasElement
|
||||||
PROCESSING_HEIGHT
|
if (!this.segmentationMaskCanvas) {
|
||||||
)
|
this.segmentationMaskCanvas = this._createCanvas(
|
||||||
// getImageData is called on this canvas 30 times per second: opt out of
|
SEGMENTATION_MASK_CANVAS_ID,
|
||||||
// GPU backing to avoid a costly readback on every frame.
|
PROCESSING_WIDTH,
|
||||||
this.segmentationMaskCanvasCtx = this.segmentationMaskCanvas.getContext(
|
PROCESSING_HEIGHT
|
||||||
'2d',
|
)
|
||||||
{ willReadFrequently: true }
|
}
|
||||||
)!
|
this.segmentationMaskCanvasCtx =
|
||||||
|
this.segmentationMaskCanvas.getContext('2d')!
|
||||||
}
|
}
|
||||||
|
|
||||||
_createCanvas(id: string, width: number, height: number) {
|
_createCanvas(id: string, width: number, height: number) {
|
||||||
@@ -514,39 +348,11 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async destroy() {
|
async destroy() {
|
||||||
this.destroyed = true
|
|
||||||
|
|
||||||
this.timerWorker?.postMessage({
|
this.timerWorker?.postMessage({
|
||||||
id: CLEAR_TIMEOUT,
|
id: CLEAR_TIMEOUT,
|
||||||
})
|
})
|
||||||
|
|
||||||
// Let any in-flight frame finish before releasing the resources it uses,
|
|
||||||
// so segmentForVideo is never called on a closed segmenter.
|
|
||||||
try {
|
|
||||||
await this.processing
|
|
||||||
} catch {
|
|
||||||
// Failures are already handled inside process().
|
|
||||||
}
|
|
||||||
this.processing = undefined
|
|
||||||
|
|
||||||
if (this.onVideoLoaded && this.videoElement) {
|
|
||||||
this.videoElement.removeEventListener('loadeddata', this.onVideoLoaded)
|
|
||||||
}
|
|
||||||
this.onVideoLoaded = undefined
|
|
||||||
|
|
||||||
this.timerWorker?.terminate()
|
this.timerWorker?.terminate()
|
||||||
this.timerWorker = undefined
|
|
||||||
|
|
||||||
this.imageSegmenter?.close()
|
this.imageSegmenter?.close()
|
||||||
this.imageSegmenter = undefined
|
|
||||||
|
|
||||||
this.processedTrack?.stop()
|
|
||||||
this.processedTrack = undefined
|
|
||||||
|
|
||||||
this.outputCanvas = undefined
|
|
||||||
this.outputCanvasCtx = undefined
|
|
||||||
this.segmentationMaskCanvas = undefined
|
|
||||||
this.segmentationMaskCanvasCtx = undefined
|
|
||||||
this.sourceImageData = undefined
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,7 +6,6 @@ import type { Track, TrackProcessor } from 'livekit-client'
|
|||||||
import { BackgroundCustomProcessor } from './BackgroundCustomProcessor'
|
import { BackgroundCustomProcessor } from './BackgroundCustomProcessor'
|
||||||
import { UnifiedBackgroundTrackProcessor } from './UnifiedBackgroundTrackProcessor'
|
import { UnifiedBackgroundTrackProcessor } from './UnifiedBackgroundTrackProcessor'
|
||||||
import { FaceLandmarksOptions } from './FaceLandmarksProcessor'
|
import { FaceLandmarksOptions } from './FaceLandmarksProcessor'
|
||||||
import { captureEvent } from '@/features/analytics/telemetry'
|
|
||||||
|
|
||||||
export const SELFIE_SEGMENTER_MODEL_PATH =
|
export const SELFIE_SEGMENTER_MODEL_PATH =
|
||||||
'/assets/mediapipe/models/selfie_segmenter_landscape.tflite'
|
'/assets/mediapipe/models/selfie_segmenter_landscape.tflite'
|
||||||
@@ -32,29 +31,15 @@ export interface BackgroundProcessorInterface extends TrackProcessor<Track.Kind>
|
|||||||
options: ProcessorConfig
|
options: ProcessorConfig
|
||||||
}
|
}
|
||||||
|
|
||||||
let unsupportedReported = false
|
|
||||||
|
|
||||||
export class BackgroundProcessorFactory {
|
export class BackgroundProcessorFactory {
|
||||||
private static _isSupported?: boolean
|
|
||||||
|
|
||||||
static hasModernApiSupport() {
|
static hasModernApiSupport() {
|
||||||
return ProcessorWrapper.hasModernApiSupport
|
return ProcessorWrapper.hasModernApiSupport
|
||||||
}
|
}
|
||||||
|
|
||||||
static isSupported() {
|
static isSupported() {
|
||||||
if (this._isSupported === undefined) {
|
return (
|
||||||
this._isSupported =
|
supportsBackgroundProcessors() || BackgroundCustomProcessor.isSupported
|
||||||
supportsBackgroundProcessors() || BackgroundCustomProcessor.isSupported
|
)
|
||||||
}
|
|
||||||
|
|
||||||
if (!this._isSupported && !unsupportedReported) {
|
|
||||||
unsupportedReported = true
|
|
||||||
captureEvent('background-processor-unsupported', {
|
|
||||||
path: 'isSupported',
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return this._isSupported
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static getProcessor(
|
static getProcessor(
|
||||||
@@ -63,8 +48,6 @@ export class BackgroundProcessorFactory {
|
|||||||
const isBlur = config.type === ProcessorType.BLUR
|
const isBlur = config.type === ProcessorType.BLUR
|
||||||
const isVirtual = config.type === ProcessorType.VIRTUAL
|
const isVirtual = config.type === ProcessorType.VIRTUAL
|
||||||
|
|
||||||
return new BackgroundCustomProcessor(config)
|
|
||||||
|
|
||||||
if (!isBlur && !isVirtual) return undefined
|
if (!isBlur && !isVirtual) return undefined
|
||||||
|
|
||||||
if (supportsBackgroundProcessors()) {
|
if (supportsBackgroundProcessors()) {
|
||||||
@@ -75,12 +58,6 @@ export class BackgroundProcessorFactory {
|
|||||||
return new BackgroundCustomProcessor(config)
|
return new BackgroundCustomProcessor(config)
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!unsupportedReported) {
|
|
||||||
captureEvent('background-processor-unsupported', {
|
|
||||||
path: 'getProcessor',
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return undefined
|
return undefined
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+5
-59
@@ -25,11 +25,7 @@ import {
|
|||||||
} from '@/features/files/api/listFiles.ts'
|
} from '@/features/files/api/listFiles.ts'
|
||||||
import { useCreateFile } from '@/features/files/api/createFile.ts'
|
import { useCreateFile } from '@/features/files/api/createFile.ts'
|
||||||
import { FileTrigger } from 'react-aria-components'
|
import { FileTrigger } from 'react-aria-components'
|
||||||
import {
|
import { RiDeleteBinLine, RiImageAddFill } from '@remixicon/react'
|
||||||
RiDeleteBinLine,
|
|
||||||
RiImageAddFill,
|
|
||||||
RiProhibitedLine,
|
|
||||||
} from '@remixicon/react'
|
|
||||||
import { useDeleteFile } from '@/features/files/api/deleteFile.ts'
|
import { useDeleteFile } from '@/features/files/api/deleteFile.ts'
|
||||||
import { useUser } from '@/features/auth/api/useUser'
|
import { useUser } from '@/features/auth/api/useUser'
|
||||||
import { ApiFileItem } from '@/features/files/api/types.ts'
|
import { ApiFileItem } from '@/features/files/api/types.ts'
|
||||||
@@ -201,23 +197,10 @@ export const EffectsConfiguration = ({
|
|||||||
*
|
*
|
||||||
* We arrive in this condition when we enter the room with the camera already off.
|
* We arrive in this condition when we enter the room with the camera already off.
|
||||||
*/
|
*/
|
||||||
try {
|
const newProcessorTmp = BackgroundProcessorFactory.getProcessor(config)!
|
||||||
const newProcessorTmp =
|
await toggle(true, {
|
||||||
BackgroundProcessorFactory.getProcessor(config)!
|
processor: newProcessorTmp,
|
||||||
await toggle(true, {
|
})
|
||||||
processor: newProcessorTmp,
|
|
||||||
})
|
|
||||||
} catch (error) {
|
|
||||||
reportError('effects_processor_failure', error, {
|
|
||||||
context: 'Error applying effect while enabling camera:',
|
|
||||||
})
|
|
||||||
saveProcessorConfig(undefined)
|
|
||||||
try {
|
|
||||||
await toggle(true)
|
|
||||||
} catch {
|
|
||||||
// Camera errors are handled by the toggle's own error path.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
setTimeout(() => setProcessorPending(false))
|
setTimeout(() => setProcessorPending(false))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -259,14 +242,6 @@ export const EffectsConfiguration = ({
|
|||||||
reportError('effects_processor_failure', error, {
|
reportError('effects_processor_failure', error, {
|
||||||
context: 'Error applying effect:',
|
context: 'Error applying effect:',
|
||||||
})
|
})
|
||||||
try {
|
|
||||||
if (videoTrack.getProcessor()) {
|
|
||||||
await videoTrack.stopProcessor()
|
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
// Best effort: the processor may already be broken.
|
|
||||||
}
|
|
||||||
saveProcessorConfig(undefined)
|
|
||||||
} finally {
|
} finally {
|
||||||
// Without setTimeout the DOM is not refreshing when updating the options.
|
// Without setTimeout the DOM is not refreshing when updating the options.
|
||||||
setTimeout(() => setProcessorPending(false))
|
setTimeout(() => setProcessorPending(false))
|
||||||
@@ -275,24 +250,6 @@ export const EffectsConfiguration = ({
|
|||||||
[enabled, selectedId, toggle, updateEffectStatusMessage, videoTrack]
|
[enabled, selectedId, toggle, updateEffectStatusMessage, videoTrack]
|
||||||
)
|
)
|
||||||
|
|
||||||
const clearEffect = useCallback(async () => {
|
|
||||||
if (selectedId === 'none') return
|
|
||||||
setProcessorPending(true)
|
|
||||||
try {
|
|
||||||
if (videoTrack?.getProcessor()) {
|
|
||||||
await videoTrack.stopProcessor()
|
|
||||||
}
|
|
||||||
saveProcessorConfig(undefined)
|
|
||||||
announceEffectStatusMessage(t('blur.status.none'))
|
|
||||||
} catch (error) {
|
|
||||||
reportError('effects_processor_failure', error, {
|
|
||||||
context: 'Error clearing effect:',
|
|
||||||
})
|
|
||||||
} finally {
|
|
||||||
setTimeout(() => setProcessorPending(false))
|
|
||||||
}
|
|
||||||
}, [announceEffectStatusMessage, selectedId, t, videoTrack])
|
|
||||||
|
|
||||||
const { data: appConfig } = useConfig()
|
const { data: appConfig } = useConfig()
|
||||||
const { isLoggedIn } = useUser()
|
const { isLoggedIn } = useUser()
|
||||||
const canUploadBackground =
|
const canUploadBackground =
|
||||||
@@ -690,17 +647,6 @@ export const EffectsConfiguration = ({
|
|||||||
gap: '1.25rem',
|
gap: '1.25rem',
|
||||||
})}
|
})}
|
||||||
>
|
>
|
||||||
<ToggleButton
|
|
||||||
variant="bigSquare"
|
|
||||||
aria-label={t('clear')}
|
|
||||||
tooltip={t('clear')}
|
|
||||||
isDisabled={processorOptions.isDisabled}
|
|
||||||
onChange={clearEffect}
|
|
||||||
isSelected={selectedId === 'none'}
|
|
||||||
data-attr="toggle-effect-none"
|
|
||||||
>
|
|
||||||
<RiProhibitedLine />
|
|
||||||
</ToggleButton>
|
|
||||||
{processorOptions.blurBased.map(({ Icon, ...option }) => (
|
{processorOptions.blurBased.map(({ Icon, ...option }) => (
|
||||||
<ToggleButton
|
<ToggleButton
|
||||||
key={option.id}
|
key={option.id}
|
||||||
|
|||||||
@@ -22,12 +22,10 @@ import { VOICE_AUDIO_CONSTRAINTS } from '../utils/constants'
|
|||||||
import {
|
import {
|
||||||
saveAudioInputDeviceId,
|
saveAudioInputDeviceId,
|
||||||
saveAudioInputEnabled,
|
saveAudioInputEnabled,
|
||||||
saveProcessorConfig,
|
|
||||||
saveVideoInputDeviceId,
|
saveVideoInputDeviceId,
|
||||||
saveVideoInputEnabled,
|
saveVideoInputEnabled,
|
||||||
userChoicesStore,
|
userChoicesStore,
|
||||||
} from '@/stores/userChoices'
|
} from '@/stores/userChoices'
|
||||||
import { reportError } from '@/features/analytics/telemetry'
|
|
||||||
import { useSyncTrackDeviceId } from './useSyncTrackDeviceId'
|
import { useSyncTrackDeviceId } from './useSyncTrackDeviceId'
|
||||||
|
|
||||||
// Module-level: effect dependencies, must be referentially stable.
|
// Module-level: effect dependencies, must be referentially stable.
|
||||||
@@ -223,32 +221,15 @@ export function useJoinTracks(): {
|
|||||||
[audioDeviceId]
|
[audioDeviceId]
|
||||||
)
|
)
|
||||||
|
|
||||||
const createVideo = useCallback(async () => {
|
const createVideo = useCallback(
|
||||||
const processor =
|
() =>
|
||||||
BackgroundProcessorFactory.fromProcessorConfig(processorConfig)
|
createLocalVideoTrack({
|
||||||
if (!processor) {
|
|
||||||
return createLocalVideoTrack({ deviceId: videoDeviceId })
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
return await createLocalVideoTrack({
|
|
||||||
deviceId: videoDeviceId,
|
deviceId: videoDeviceId,
|
||||||
processor,
|
processor:
|
||||||
})
|
BackgroundProcessorFactory.fromProcessorConfig(processorConfig),
|
||||||
} catch (error) {
|
}),
|
||||||
// A camera problem (permission, device missing/busy) is not the
|
[videoDeviceId, processorConfig]
|
||||||
// effect's fault: let the normal media error handling deal with it
|
)
|
||||||
// without touching the user's saved effect.
|
|
||||||
const e = getMediaDeviceFailure(error as Error)
|
|
||||||
if (e !== MediaDeviceFailure.Other && !!e) {
|
|
||||||
throw error
|
|
||||||
}
|
|
||||||
reportError('effects_processor_failure', error, {
|
|
||||||
context: 'Restoring saved effect failed, retrying without it',
|
|
||||||
})
|
|
||||||
saveProcessorConfig(undefined)
|
|
||||||
return createLocalVideoTrack({ deviceId: videoDeviceId })
|
|
||||||
}
|
|
||||||
}, [videoDeviceId, processorConfig])
|
|
||||||
|
|
||||||
const audioTrack = useLocalTrack({
|
const audioTrack = useLocalTrack({
|
||||||
ready: audioReady,
|
ready: audioReady,
|
||||||
|
|||||||
Reference in New Issue
Block a user