♻️(frontend) inline MediaPipe WASM modules to avoid loading from remote

Copy the MediaPipe WASM modules into the frontend build output and
serve them locally, instead of loading them from the Google CDN at
runtime.

Mediapipe was a transitive dependency, add it explicitly.

Requested by some members of the community.
It closes #1168

CHANGELOG.md

@@ -22,6 +22,7 @@ and this project adheres to
 - 🚸(frontend) initialize the join input name with the persisted full name
 - ♻️(frontend) refactor background processors to use the new API
 - ♻️(frontend) inline model weights to avoid loading them from remote
+- ♻️(frontend) inline MediaPipe WASM modules to avoid loading from remote
 
 ### Fixed
 

src/frontend/package-lock.json

@@ -15,6 +15,7 @@
         "@livekit/components-react": "2.9.21",
         "@livekit/components-styles": "1.2.0",
         "@livekit/track-processors": "0.7.2",
+        "@mediapipe/tasks-vision": "0.10.14",
         "@pandacss/preset-panda": "1.11.3",
         "@react-types/overlays": "3.10.0",
         "@remixicon/react": "4.9.0",

src/frontend/package.json

@@ -22,6 +22,7 @@
     "@livekit/components-react": "2.9.21",
     "@livekit/components-styles": "1.2.0",
     "@livekit/track-processors": "0.7.2",
+    "@mediapipe/tasks-vision": "0.10.14",
     "@pandacss/preset-panda": "1.11.3",
     "@react-types/overlays": "3.10.0",
     "@remixicon/react": "4.9.0",

src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts

@@ -16,6 +16,7 @@ import {
   SELFIE_SEGMENTER_MODEL_PATH,
   type ProcessorConfig,
   type ProcessorType,
+  MEDIAPIPE_PATH_WASM,
 } from '.'
 
 const PROCESSING_WIDTH = 256
@@ -154,9 +155,7 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
   }
 
   async initSegmenter() {
-    const vision = await FilesetResolver.forVisionTasks(
-      'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision/wasm'
-    )
+    const vision = await FilesetResolver.forVisionTasks(MEDIAPIPE_PATH_WASM)
     this.imageSegmenter = await ImageSegmenter.createFromOptions(vision, {
       baseOptions: {
         modelAssetPath: SELFIE_SEGMENTER_MODEL_PATH,

src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts

@@ -11,7 +11,11 @@ import {
   TIMEOUT_TICK,
   timerWorkerScript,
 } from './TimerWorker'
-import { FACE_LANDMARKS_MODEL_PATH, ProcessorType } from '.'
+import {
+  FACE_LANDMARKS_MODEL_PATH,
+  ProcessorType,
+  MEDIAPIPE_PATH_WASM,
+} from '.'
 
 const PROCESSING_WIDTH = 256 * 3
 const PROCESSING_HEIGHT = 144 * 3
@@ -128,9 +132,7 @@ export class FaceLandmarksProcessor implements TrackProcessor<Track.Kind> {
   }
 
   async initFaceLandmarker() {
-    const vision = await FilesetResolver.forVisionTasks(
-      'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision/wasm'
-    )
+    const vision = await FilesetResolver.forVisionTasks(MEDIAPIPE_PATH_WASM)
     this.faceLandmarker = await FaceLandmarker.createFromOptions(vision, {
       baseOptions: {
         modelAssetPath: FACE_LANDMARKS_MODEL_PATH,

src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts

@@ -7,6 +7,7 @@ import {
   type ProcessorConfig,
   BackgroundProcessorInterface,
   ProcessorType,
+  MEDIAPIPE_PATH_WASM,
   SELFIE_SEGMENTER_MODEL_PATH,
 } from '.'
 
@@ -24,6 +25,7 @@ export class UnifiedBackgroundTrackProcessor implements BackgroundProcessorInter
         mode: 'virtual-background',
         imagePath: opts.imagePath,
         assetPaths: {
+          tasksVisionFileSet: MEDIAPIPE_PATH_WASM,
           modelAssetPath: SELFIE_SEGMENTER_MODEL_PATH,
         },
       })
@@ -33,6 +35,7 @@ export class UnifiedBackgroundTrackProcessor implements BackgroundProcessorInter
         mode: 'background-blur',
         blurRadius: opts.blurRadius,
         assetPaths: {
+          tasksVisionFileSet: MEDIAPIPE_PATH_WASM,
           modelAssetPath: SELFIE_SEGMENTER_MODEL_PATH,
         },
       })

src/frontend/src/features/rooms/livekit/components/blur/index.ts

@@ -10,6 +10,8 @@ export const SELFIE_SEGMENTER_MODEL_PATH =
 export const FACE_LANDMARKS_MODEL_PATH =
   '/assets/mediapipe/models/face_landmarker.task'
 
+export const MEDIAPIPE_PATH_WASM = '/assets/mediapipe/wasm'
+
 export enum ProcessorType {
   BLUR = 'blur',
   VIRTUAL = 'virtual',

src/frontend/vite.config.ts

@@ -2,6 +2,7 @@ import { defineConfig, loadEnv } from 'vite'
 import react from '@vitejs/plugin-react'
 import { visualizer } from 'rollup-plugin-visualizer'
 import svgr from 'vite-plugin-svgr'
+import { viteStaticCopy } from 'vite-plugin-static-copy'
 
 // https://vitejs.dev/config/
 export default defineConfig(({ mode }) => {
@@ -18,6 +19,15 @@ export default defineConfig(({ mode }) => {
           },
         },
       }),
+      viteStaticCopy({
+        targets: [
+          {
+            src: 'node_modules/@mediapipe/tasks-vision/wasm/*',
+            dest: 'assets/mediapipe/wasm',
+            rename: { stripBase: 4 },
+          },
+        ],
+      }),
       env.VITE_ANALYZE === 'true' &&
         visualizer({
           open: true,