From 581e115c03e1be31f64d5f2881cec5238df3a68e Mon Sep 17 00:00:00 2001 From: lebaudantoine Date: Sat, 11 Jul 2026 15:42:41 +0200 Subject: [PATCH] =?UTF-8?q?=E2=99=BB=EF=B8=8F(frontend)=20inline=20MediaPi?= =?UTF-8?q?pe=20WASM=20modules=20to=20avoid=20loading=20from=20remote?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Copy the MediaPipe WASM modules into the frontend build output and serve them locally, instead of loading them from the Google CDN at runtime. Mediapipe was a transitive dependency, add it explicitly. Requested by some members of the community. It closes #1168 --- CHANGELOG.md | 1 + src/frontend/package-lock.json | 1 + src/frontend/package.json | 1 + .../components/blur/BackgroundCustomProcessor.ts | 5 ++--- .../livekit/components/blur/FaceLandmarksProcessor.ts | 10 ++++++---- .../components/blur/UnifiedBackgroundTrackProcessor.ts | 3 +++ .../features/rooms/livekit/components/blur/index.ts | 2 ++ src/frontend/vite.config.ts | 10 ++++++++++ 8 files changed, 26 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68436d9b..d0793c03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ and this project adheres to - 🚸(frontend) initialize the join input name with the persisted full name - ♻️(frontend) refactor background processors to use the new API - ♻️(frontend) inline model weights to avoid loading them from remote +- ♻️(frontend) inline MediaPipe WASM modules to avoid loading from remote ### Fixed diff --git a/src/frontend/package-lock.json b/src/frontend/package-lock.json index f5cf0734..c25d89e1 100644 --- a/src/frontend/package-lock.json +++ b/src/frontend/package-lock.json @@ -15,6 +15,7 @@ "@livekit/components-react": "2.9.21", "@livekit/components-styles": "1.2.0", "@livekit/track-processors": "0.7.2", + "@mediapipe/tasks-vision": "0.10.14", "@pandacss/preset-panda": "1.11.3", "@react-types/overlays": "3.10.0", "@remixicon/react": "4.9.0", diff --git a/src/frontend/package.json b/src/frontend/package.json index 93cafdf1..f81aa632 100644 --- a/src/frontend/package.json +++ b/src/frontend/package.json @@ -22,6 +22,7 @@ "@livekit/components-react": "2.9.21", "@livekit/components-styles": "1.2.0", "@livekit/track-processors": "0.7.2", + "@mediapipe/tasks-vision": "0.10.14", "@pandacss/preset-panda": "1.11.3", "@react-types/overlays": "3.10.0", "@remixicon/react": "4.9.0", diff --git a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts index 95b5ed32..d2ce7c28 100644 --- a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts +++ b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts @@ -16,6 +16,7 @@ import { SELFIE_SEGMENTER_MODEL_PATH, type ProcessorConfig, type ProcessorType, + MEDIAPIPE_PATH_WASM, } from '.' const PROCESSING_WIDTH = 256 @@ -154,9 +155,7 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface { } async initSegmenter() { - const vision = await FilesetResolver.forVisionTasks( - 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision/wasm' - ) + const vision = await FilesetResolver.forVisionTasks(MEDIAPIPE_PATH_WASM) this.imageSegmenter = await ImageSegmenter.createFromOptions(vision, { baseOptions: { modelAssetPath: SELFIE_SEGMENTER_MODEL_PATH, diff --git a/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts index b6f6e3f2..5b4873cf 100644 --- a/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts +++ b/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts @@ -11,7 +11,11 @@ import { TIMEOUT_TICK, timerWorkerScript, } from './TimerWorker' -import { FACE_LANDMARKS_MODEL_PATH, ProcessorType } from '.' +import { + FACE_LANDMARKS_MODEL_PATH, + ProcessorType, + MEDIAPIPE_PATH_WASM, +} from '.' const PROCESSING_WIDTH = 256 * 3 const PROCESSING_HEIGHT = 144 * 3 @@ -128,9 +132,7 @@ export class FaceLandmarksProcessor implements TrackProcessor { } async initFaceLandmarker() { - const vision = await FilesetResolver.forVisionTasks( - 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision/wasm' - ) + const vision = await FilesetResolver.forVisionTasks(MEDIAPIPE_PATH_WASM) this.faceLandmarker = await FaceLandmarker.createFromOptions(vision, { baseOptions: { modelAssetPath: FACE_LANDMARKS_MODEL_PATH, diff --git a/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts index 237e790c..5087215f 100644 --- a/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts +++ b/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts @@ -7,6 +7,7 @@ import { type ProcessorConfig, BackgroundProcessorInterface, ProcessorType, + MEDIAPIPE_PATH_WASM, SELFIE_SEGMENTER_MODEL_PATH, } from '.' @@ -24,6 +25,7 @@ export class UnifiedBackgroundTrackProcessor implements BackgroundProcessorInter mode: 'virtual-background', imagePath: opts.imagePath, assetPaths: { + tasksVisionFileSet: MEDIAPIPE_PATH_WASM, modelAssetPath: SELFIE_SEGMENTER_MODEL_PATH, }, }) @@ -33,6 +35,7 @@ export class UnifiedBackgroundTrackProcessor implements BackgroundProcessorInter mode: 'background-blur', blurRadius: opts.blurRadius, assetPaths: { + tasksVisionFileSet: MEDIAPIPE_PATH_WASM, modelAssetPath: SELFIE_SEGMENTER_MODEL_PATH, }, }) diff --git a/src/frontend/src/features/rooms/livekit/components/blur/index.ts b/src/frontend/src/features/rooms/livekit/components/blur/index.ts index 892af9bd..e6399f83 100644 --- a/src/frontend/src/features/rooms/livekit/components/blur/index.ts +++ b/src/frontend/src/features/rooms/livekit/components/blur/index.ts @@ -10,6 +10,8 @@ export const SELFIE_SEGMENTER_MODEL_PATH = export const FACE_LANDMARKS_MODEL_PATH = '/assets/mediapipe/models/face_landmarker.task' +export const MEDIAPIPE_PATH_WASM = '/assets/mediapipe/wasm' + export enum ProcessorType { BLUR = 'blur', VIRTUAL = 'virtual', diff --git a/src/frontend/vite.config.ts b/src/frontend/vite.config.ts index 44fda9ea..fd12d6fb 100644 --- a/src/frontend/vite.config.ts +++ b/src/frontend/vite.config.ts @@ -2,6 +2,7 @@ import { defineConfig, loadEnv } from 'vite' import react from '@vitejs/plugin-react' import { visualizer } from 'rollup-plugin-visualizer' import svgr from 'vite-plugin-svgr' +import { viteStaticCopy } from 'vite-plugin-static-copy' // https://vitejs.dev/config/ export default defineConfig(({ mode }) => { @@ -18,6 +19,15 @@ export default defineConfig(({ mode }) => { }, }, }), + viteStaticCopy({ + targets: [ + { + src: 'node_modules/@mediapipe/tasks-vision/wasm/*', + dest: 'assets/mediapipe/wasm', + rename: { stripBase: 4 }, + }, + ], + }), env.VITE_ANALYZE === 'true' && visualizer({ open: true,