From ed3784e1ca4abf29f3facf42abdc3672efcfd733 Mon Sep 17 00:00:00 2001 From: lebaudantoine Date: Sun, 2 Aug 2026 23:18:51 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=A9=B9(frontend)=20unblock=20virtual=20ba?= =?UTF-8?q?ckground=20loading=20under=20bearer=20auth?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moving off cookie-based authentication surfaced several hard issues, especially around loading virtual backgrounds: requests used to be sent with cookies automatically, which trivially authenticated those loads. With bearer tokens, those requests need to be authenticated explicitly. The situation is made harder by the fact that, when the custom virtual background was introduced, some of the loading was done as module-level, blocking imports that are not handled by React and therefore live outside the normal auth flow. Ship a functional patch to unblock third parties currently waiting on this integration. The virtual background loading path should definitely be refactored and simplified in a follow-up. --- .../files/hooks/useResolvedMediaUrls.ts | 56 +++++++++++++++++++ .../features/files/utils/resolveMediaUrl.ts | 47 ++++++++++++++++ .../blur/BackgroundCustomProcessor.ts | 13 +++-- .../blur/UnifiedBackgroundTrackProcessor.ts | 15 ++++- .../effects/EffectsConfiguration.tsx | 11 +++- src/frontend/src/stores/userChoices.ts | 11 ++++ 6 files changed, 147 insertions(+), 6 deletions(-) create mode 100644 src/frontend/src/features/files/hooks/useResolvedMediaUrls.ts create mode 100644 src/frontend/src/features/files/utils/resolveMediaUrl.ts diff --git a/src/frontend/src/features/files/hooks/useResolvedMediaUrls.ts b/src/frontend/src/features/files/hooks/useResolvedMediaUrls.ts new file mode 100644 index 00000000..538635f6 --- /dev/null +++ b/src/frontend/src/features/files/hooks/useResolvedMediaUrls.ts @@ -0,0 +1,56 @@ +import { useCallback, useEffect, useState } from 'react' +import { useSnapshot } from 'valtio' +import { accessTokenStore } from '@/stores/accessToken' +import { resolveMediaUrl } from '../utils/resolveMediaUrl' + +/** + * Reactive companion of resolveMediaUrl for browser-native consumers + * (CSS url(), img src attributes): resolves a list of /media/ URLs and + * returns a stable lookup, identity in regular mode. + * + * Object URLs come from the shared session-lifetime cache and are never + * revoked here: they may be used concurrently by the background + * processors. + */ +export const useResolvedMediaUrls = ( + urls: (string | null | undefined)[] +): ((url: string) => string) => { + const [resolved, setResolved] = useState>({}) + const { accessToken } = useSnapshot(accessTokenStore) + + // Stable dependency for the effect, insensitive to array identity + const urlsKey = urls.filter(Boolean).sort().join('\n') + + useEffect(() => { + if (!accessToken || !urlsKey) { + return + } + + let isMounted = true + + const resolveAll = async () => { + const entries = await Promise.all( + urlsKey.split('\n').map(async (url) => { + try { + return [url, await resolveMediaUrl(url)] as const + } catch (error) { + console.warn(error) + return [url, url] as const + } + }) + ) + if (isMounted) { + setResolved(Object.fromEntries(entries)) + } + } + resolveAll() + + return () => { + isMounted = false + } + }, [accessToken, urlsKey]) + + // Stable identity so that consumers can safely list the resolver in + // their memo dependencies: it only changes when resolutions land. + return useCallback((url: string) => resolved[url] ?? url, [resolved]) +} diff --git a/src/frontend/src/features/files/utils/resolveMediaUrl.ts b/src/frontend/src/features/files/utils/resolveMediaUrl.ts new file mode 100644 index 00000000..8af828b2 --- /dev/null +++ b/src/frontend/src/features/files/utils/resolveMediaUrl.ts @@ -0,0 +1,47 @@ +import { getAccessToken } from '@/stores/accessToken' + +// Session-lifetime cache: object URLs are shared between every consumer +// of a given media (background processors, thumbnails) and are therefore +// never revoked - their number is bounded by the user's custom +// backgrounds, and they die with the page like the access token does. +const objectUrlCache = new Map() + +/** + * Resolve an authenticated /media/ URL for the embedded (token) mode. + * + * Media files are served behind an nginx auth_request subrequest that + * authenticates the original request. In regular mode the session cookie + * rides along browser-native loads (img.src, CSS url()) and the URL is + * returned unchanged, without any fetch. In embedded mode the + * third-party cookie is blocked and native loads cannot carry the + * Authorization header, so the media is fetched here with the Bearer + * header - which the media-auth endpoint accepts, as it sits behind the + * default authentication stack - and exposed as a blob object URL. + */ +export const resolveMediaUrl = async (url: string): Promise => { + const accessToken = getAccessToken() + + if (!accessToken) { + return url + } + + const cached = objectUrlCache.get(url) + if (cached) { + return cached + } + + const response = await fetch(url, { + headers: { Authorization: `Bearer ${accessToken}` }, + }) + + if (!response.ok) { + throw new Error( + `Failed to resolve media url ${url}: HTTP ${response.status}` + ) + } + + const objectUrl = URL.createObjectURL(await response.blob()) + objectUrlCache.set(url, objectUrl) + + return objectUrl +} diff --git a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts index 24cc8a77..b24a1e58 100644 --- a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts +++ b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts @@ -1,4 +1,5 @@ import type { ProcessorOptions, Track } from 'livekit-client' +import { resolveMediaUrl } from '@/features/files/utils/resolveMediaUrl' import { FilesetResolver, ImageSegmenter, @@ -85,7 +86,7 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface { this.sourceSettings = this.source!.getSettings() this.videoElement = opts.element as HTMLVideoElement - this._initVirtualBackgroundImage() + await this._initVirtualBackgroundImage() this._createMainCanvas() this._createMaskCanvas() @@ -103,7 +104,7 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface { captureEvent('firefox-blurring-init', {}) } - _initVirtualBackgroundImage() { + async _initVirtualBackgroundImage() { if (this.options.type !== 'virtual') { throw new Error( 'Virtual background is only supported for virtual background' @@ -115,15 +116,19 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface { this.virtualBackgroundImage && this.virtualBackgroundImage.src !== this.options.imagePath if (this.options.imagePath || needsUpdate) { + // Embedded (token) mode: img.src cannot carry the Authorization + // header, resolve the media to a blob object URL first. Identity + // in regular mode. + const imagePath = await resolveMediaUrl(this.options.imagePath!) this.virtualBackgroundImage = document.createElement('img') this.virtualBackgroundImage.crossOrigin = 'anonymous' - this.virtualBackgroundImage.src = this.options.imagePath! + this.virtualBackgroundImage.src = imagePath } } async update(opts: ProcessorConfig): Promise { this.options = opts - this._initVirtualBackgroundImage() + await this._initVirtualBackgroundImage() } _initWorker() { diff --git a/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts index 5087215f..2cdb11ff 100644 --- a/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts +++ b/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts @@ -1,4 +1,5 @@ import type { ProcessorOptions, Track } from 'livekit-client' +import { resolveMediaUrl } from '@/features/files/utils/resolveMediaUrl' import { ProcessorWrapper, BackgroundProcessor, @@ -47,7 +48,16 @@ export class UnifiedBackgroundTrackProcessor implements BackgroundProcessorInter } async init(opts: ProcessorOptions) { - return this.processor.init(opts) + await this.processor.init(opts) + // Embedded (token) mode: the constructor passed the raw imagePath, + // whose native load cannot carry the Authorization header. Swap it + // for a resolved blob object URL. No-op in regular mode. + if (this.opts.type === 'virtual') { + const imagePath = await resolveMediaUrl(this.opts.imagePath) + if (imagePath !== this.opts.imagePath) { + await this.processor.updateTransformerOptions({ imagePath }) + } + } } async restart(opts: ProcessorOptions) { @@ -59,6 +69,9 @@ export class UnifiedBackgroundTrackProcessor implements BackgroundProcessorInter } async update(opts: ProcessorConfig): Promise { + if (opts.type === 'virtual') { + opts = { ...opts, imagePath: await resolveMediaUrl(opts.imagePath) } + } this.opts = opts const newProcessorType = diff --git a/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx b/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx index 8c3fb335..24b745db 100644 --- a/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx +++ b/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx @@ -8,6 +8,7 @@ import { ProcessorType, } from '../blur' import { css } from '@/styled-system/css' +import { useResolvedMediaUrls } from '@/features/files/hooks/useResolvedMediaUrls' import { Button, Dialog, H, P, Text, ToggleButton } from '@/primitives' import { VisualOnlyTooltip } from '@/primitives/VisualOnlyTooltip' import { HStack, styled } from '@/styled-system/jsx' @@ -280,6 +281,14 @@ export const EffectsConfiguration = ({ filesQ.data.count >= appConfig.background_image.max_count_by_user) ?? false + // Thumbnails are browser-native loads (CSS url()) which cannot carry + // the Authorization header in embedded (token) mode: resolve them. The + // processor configs keep the stable raw URLs - they are persisted in + // the user choices - and the processors resolve them internally. + const resolveMediaUrl = useResolvedMediaUrls( + (filesQ.data?.results ?? []).map((file) => file.url) + ) + const getHandleSelectChangeFile = useCallback( (file: ApiFileItem) => { return async () => { @@ -757,7 +766,7 @@ export const EffectsConfiguration = ({ bgSize: 'cover', })} style={{ - backgroundImage: `url(${option.file.url!})`, + backgroundImage: `url(${resolveMediaUrl(option.file.url!)})`, }} data-attr={`toggle-virtual-${option.file.id}`} /> diff --git a/src/frontend/src/stores/userChoices.ts b/src/frontend/src/stores/userChoices.ts index de64a07d..b1a6c0f6 100644 --- a/src/frontend/src/stores/userChoices.ts +++ b/src/frontend/src/stores/userChoices.ts @@ -1,4 +1,6 @@ import { proxy, subscribe } from 'valtio' +import { initializeAccessTokenFromFragment } from '@/features/auth/api/exchangeAccessToken' +import { getAccessToken } from '@/stores/accessToken' import { ProcessorConfig, ProcessorType, @@ -48,10 +50,19 @@ if (userChoicesStore.processorConfig?.type === ProcessorType.VIRTUAL) { // we restore clear the processor config to avoid displaying a black screen. userChoicesStore.processorConfig = undefined } else if (userChoicesStore.processorConfig.fileId) { + // Embedded (token) mode: this module loads before the transit code + // exchange has settled - wait for it, and carry the Bearer header, + // otherwise the check below would wrongly clear the config. + await initializeAccessTokenFromFragment() + const accessToken = getAccessToken() + // Checking if the image is still available / accessible await fetch(userChoicesStore.processorConfig.imagePath, { // We bypass the cache to ensure we have access cache: 'reload', + ...(accessToken && { + headers: { Authorization: `Bearer ${accessToken}` }, + }), }) .then((response) => { // if we cannot fetch the image (likely a 401 from the backend because