From 0a879a96fdb547eeb6cea679f4605675bfcbe4bb Mon Sep 17 00:00:00 2001 From: lebaudantoine Date: Sun, 2 Aug 2026 23:18:51 +0200 Subject: [PATCH] wip handle virtual background loading in an iframe context --- .../files/hooks/useResolvedMediaUrls.ts | 56 +++++++++++++++++++ .../features/files/utils/resolveMediaUrl.ts | 47 ++++++++++++++++ .../blur/BackgroundCustomProcessor.ts | 13 +++-- .../blur/UnifiedBackgroundTrackProcessor.ts | 15 ++++- .../effects/EffectsConfiguration.tsx | 11 +++- src/frontend/src/stores/userChoices.ts | 11 ++++ 6 files changed, 147 insertions(+), 6 deletions(-) create mode 100644 src/frontend/src/features/files/hooks/useResolvedMediaUrls.ts create mode 100644 src/frontend/src/features/files/utils/resolveMediaUrl.ts diff --git a/src/frontend/src/features/files/hooks/useResolvedMediaUrls.ts b/src/frontend/src/features/files/hooks/useResolvedMediaUrls.ts new file mode 100644 index 00000000..538635f6 --- /dev/null +++ b/src/frontend/src/features/files/hooks/useResolvedMediaUrls.ts @@ -0,0 +1,56 @@ +import { useCallback, useEffect, useState } from 'react' +import { useSnapshot } from 'valtio' +import { accessTokenStore } from '@/stores/accessToken' +import { resolveMediaUrl } from '../utils/resolveMediaUrl' + +/** + * Reactive companion of resolveMediaUrl for browser-native consumers + * (CSS url(), img src attributes): resolves a list of /media/ URLs and + * returns a stable lookup, identity in regular mode. + * + * Object URLs come from the shared session-lifetime cache and are never + * revoked here: they may be used concurrently by the background + * processors. + */ +export const useResolvedMediaUrls = ( + urls: (string | null | undefined)[] +): ((url: string) => string) => { + const [resolved, setResolved] = useState>({}) + const { accessToken } = useSnapshot(accessTokenStore) + + // Stable dependency for the effect, insensitive to array identity + const urlsKey = urls.filter(Boolean).sort().join('\n') + + useEffect(() => { + if (!accessToken || !urlsKey) { + return + } + + let isMounted = true + + const resolveAll = async () => { + const entries = await Promise.all( + urlsKey.split('\n').map(async (url) => { + try { + return [url, await resolveMediaUrl(url)] as const + } catch (error) { + console.warn(error) + return [url, url] as const + } + }) + ) + if (isMounted) { + setResolved(Object.fromEntries(entries)) + } + } + resolveAll() + + return () => { + isMounted = false + } + }, [accessToken, urlsKey]) + + // Stable identity so that consumers can safely list the resolver in + // their memo dependencies: it only changes when resolutions land. + return useCallback((url: string) => resolved[url] ?? url, [resolved]) +} diff --git a/src/frontend/src/features/files/utils/resolveMediaUrl.ts b/src/frontend/src/features/files/utils/resolveMediaUrl.ts new file mode 100644 index 00000000..8af828b2 --- /dev/null +++ b/src/frontend/src/features/files/utils/resolveMediaUrl.ts @@ -0,0 +1,47 @@ +import { getAccessToken } from '@/stores/accessToken' + +// Session-lifetime cache: object URLs are shared between every consumer +// of a given media (background processors, thumbnails) and are therefore +// never revoked - their number is bounded by the user's custom +// backgrounds, and they die with the page like the access token does. +const objectUrlCache = new Map() + +/** + * Resolve an authenticated /media/ URL for the embedded (token) mode. + * + * Media files are served behind an nginx auth_request subrequest that + * authenticates the original request. In regular mode the session cookie + * rides along browser-native loads (img.src, CSS url()) and the URL is + * returned unchanged, without any fetch. In embedded mode the + * third-party cookie is blocked and native loads cannot carry the + * Authorization header, so the media is fetched here with the Bearer + * header - which the media-auth endpoint accepts, as it sits behind the + * default authentication stack - and exposed as a blob object URL. + */ +export const resolveMediaUrl = async (url: string): Promise => { + const accessToken = getAccessToken() + + if (!accessToken) { + return url + } + + const cached = objectUrlCache.get(url) + if (cached) { + return cached + } + + const response = await fetch(url, { + headers: { Authorization: `Bearer ${accessToken}` }, + }) + + if (!response.ok) { + throw new Error( + `Failed to resolve media url ${url}: HTTP ${response.status}` + ) + } + + const objectUrl = URL.createObjectURL(await response.blob()) + objectUrlCache.set(url, objectUrl) + + return objectUrl +} diff --git a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts index d2ce7c28..d35bab20 100644 --- a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts +++ b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts @@ -1,4 +1,5 @@ import type { ProcessorOptions, Track } from 'livekit-client' +import { resolveMediaUrl } from '@/features/files/utils/resolveMediaUrl' import posthog from 'posthog-js' import { FilesetResolver, @@ -85,7 +86,7 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface { this.sourceSettings = this.source!.getSettings() this.videoElement = opts.element as HTMLVideoElement - this._initVirtualBackgroundImage() + await this._initVirtualBackgroundImage() this._createMainCanvas() this._createMaskCanvas() @@ -103,7 +104,7 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface { posthog.capture('firefox-blurring-init') } - _initVirtualBackgroundImage() { + async _initVirtualBackgroundImage() { if (this.options.type !== 'virtual') { throw new Error( 'Virtual background is only supported for virtual background' @@ -115,15 +116,19 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface { this.virtualBackgroundImage && this.virtualBackgroundImage.src !== this.options.imagePath if (this.options.imagePath || needsUpdate) { + // Embedded (token) mode: img.src cannot carry the Authorization + // header, resolve the media to a blob object URL first. Identity + // in regular mode. + const imagePath = await resolveMediaUrl(this.options.imagePath!) this.virtualBackgroundImage = document.createElement('img') this.virtualBackgroundImage.crossOrigin = 'anonymous' - this.virtualBackgroundImage.src = this.options.imagePath! + this.virtualBackgroundImage.src = imagePath } } async update(opts: ProcessorConfig): Promise { this.options = opts - this._initVirtualBackgroundImage() + await this._initVirtualBackgroundImage() } _initWorker() { diff --git a/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts index 5087215f..2cdb11ff 100644 --- a/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts +++ b/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts @@ -1,4 +1,5 @@ import type { ProcessorOptions, Track } from 'livekit-client' +import { resolveMediaUrl } from '@/features/files/utils/resolveMediaUrl' import { ProcessorWrapper, BackgroundProcessor, @@ -47,7 +48,16 @@ export class UnifiedBackgroundTrackProcessor implements BackgroundProcessorInter } async init(opts: ProcessorOptions) { - return this.processor.init(opts) + await this.processor.init(opts) + // Embedded (token) mode: the constructor passed the raw imagePath, + // whose native load cannot carry the Authorization header. Swap it + // for a resolved blob object URL. No-op in regular mode. + if (this.opts.type === 'virtual') { + const imagePath = await resolveMediaUrl(this.opts.imagePath) + if (imagePath !== this.opts.imagePath) { + await this.processor.updateTransformerOptions({ imagePath }) + } + } } async restart(opts: ProcessorOptions) { @@ -59,6 +69,9 @@ export class UnifiedBackgroundTrackProcessor implements BackgroundProcessorInter } async update(opts: ProcessorConfig): Promise { + if (opts.type === 'virtual') { + opts = { ...opts, imagePath: await resolveMediaUrl(opts.imagePath) } + } this.opts = opts const newProcessorType = diff --git a/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx b/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx index 44c761d6..ad584cb7 100644 --- a/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx +++ b/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx @@ -8,6 +8,7 @@ import { ProcessorType, } from '../blur' import { css } from '@/styled-system/css' +import { useResolvedMediaUrls } from '@/features/files/hooks/useResolvedMediaUrls' import { Button, Dialog, H, P, Text, ToggleButton } from '@/primitives' import { VisualOnlyTooltip } from '@/primitives/VisualOnlyTooltip' import { HStack, styled } from '@/styled-system/jsx' @@ -277,6 +278,14 @@ export const EffectsConfiguration = ({ filesQ.data.count >= appConfig.background_image.max_count_by_user) ?? false + // Thumbnails are browser-native loads (CSS url()) which cannot carry + // the Authorization header in embedded (token) mode: resolve them. The + // processor configs keep the stable raw URLs - they are persisted in + // the user choices - and the processors resolve them internally. + const resolveMediaUrl = useResolvedMediaUrls( + (filesQ.data?.results ?? []).map((file) => file.url) + ) + const getHandleSelectChangeFile = useCallback( (file: ApiFileItem) => { return async () => { @@ -754,7 +763,7 @@ export const EffectsConfiguration = ({ bgSize: 'cover', })} style={{ - backgroundImage: `url(${option.file.url!})`, + backgroundImage: `url(${resolveMediaUrl(option.file.url!)})`, }} data-attr={`toggle-virtual-${option.file.id}`} /> diff --git a/src/frontend/src/stores/userChoices.ts b/src/frontend/src/stores/userChoices.ts index de64a07d..b1a6c0f6 100644 --- a/src/frontend/src/stores/userChoices.ts +++ b/src/frontend/src/stores/userChoices.ts @@ -1,4 +1,6 @@ import { proxy, subscribe } from 'valtio' +import { initializeAccessTokenFromFragment } from '@/features/auth/api/exchangeAccessToken' +import { getAccessToken } from '@/stores/accessToken' import { ProcessorConfig, ProcessorType, @@ -48,10 +50,19 @@ if (userChoicesStore.processorConfig?.type === ProcessorType.VIRTUAL) { // we restore clear the processor config to avoid displaying a black screen. userChoicesStore.processorConfig = undefined } else if (userChoicesStore.processorConfig.fileId) { + // Embedded (token) mode: this module loads before the transit code + // exchange has settled - wait for it, and carry the Bearer header, + // otherwise the check below would wrongly clear the config. + await initializeAccessTokenFromFragment() + const accessToken = getAccessToken() + // Checking if the image is still available / accessible await fetch(userChoicesStore.processorConfig.imagePath, { // We bypass the cache to ensure we have access cache: 'reload', + ...(accessToken && { + headers: { Authorization: `Bearer ${accessToken}` }, + }), }) .then((response) => { // if we cannot fetch the image (likely a 401 from the backend because