♻️(frontend) inline MediaPipe WASM modules to avoid loading from remote

Copy the MediaPipe WASM modules into the frontend build output and
serve them locally, instead of loading them from the Google CDN at
runtime.

Mediapipe was a transitive dependency, add it explicitly.

Requested by some members of the community.
It closes #1168

CHANGELOG.md

@@ -13,12 +13,16 @@ and this project adheres to
 - ✨(backend) allow searching the recording admin table by owner email
 - ✨(frontend) add participant color gradient when camera is off #1490
 - ✨(all) allow forcing SSO display name for authenticated users
+- ➕(frontend) install vite-plugin-static-copy for MediaPipe WASM assets
 
 ### Changed
 
 - 🗑️(settings) deprecate SUMMARY_SERVICE_VERSION=1
 - ⬆️(mail) update mjml to v5 and @html-to/text-cli
 - 🚸(frontend) initialize the join input name with the persisted full name
+- ♻️(frontend) refactor background processors to use the new API
+- ♻️(frontend) inline model weights to avoid loading them from remote
+- ♻️(frontend) inline MediaPipe WASM modules to avoid loading from remote
 
 ### Fixed
 

src/frontend/package-lock.json

@@ -15,6 +15,7 @@
         "@livekit/components-react": "2.9.21",
         "@livekit/components-styles": "1.2.0",
         "@livekit/track-processors": "0.7.2",
+        "@mediapipe/tasks-vision": "0.10.14",
         "@pandacss/preset-panda": "1.11.3",
         "@react-types/overlays": "3.10.0",
         "@remixicon/react": "4.9.0",
@@ -63,6 +64,7 @@
         "typescript": "6.0.3",
         "typescript-eslint": "8.60.1",
         "vite": "8.0.14",
+        "vite-plugin-static-copy": "4.1.1",
         "vite-plugin-svgr": "5.2.0"
       }
     },
@@ -3192,6 +3194,19 @@
         "node": ">=6.0.0"
       }
     },
+    "node_modules/binary-extensions": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz",
+      "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/bl": {
       "version": "5.1.0",
       "resolved": "https://registry.npmjs.org/bl/-/bl-5.1.0.tgz",
@@ -7094,6 +7109,19 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/is-binary-path": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz",
+      "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "binary-extensions": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/is-boolean-object": {
       "version": "1.2.2",
       "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.2.2.tgz",
@@ -8675,6 +8703,19 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/p-map": {
+      "version": "7.0.5",
+      "resolved": "https://registry.npmjs.org/p-map/-/p-map-7.0.5.tgz",
+      "integrity": "sha512-e8vJF4XdVkzqqSHguEMz41mQO1wKwxKm5ENrUJQUu9kLDCtn83cxbyHZcszr4QC5zEA7WffRRC4gsTecC7J9oA==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=18"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/package-manager-detector": {
       "version": "1.6.0",
       "resolved": "https://registry.npmjs.org/package-manager-detector/-/package-manager-detector-1.6.0.tgz",
@@ -11137,6 +11178,80 @@
         }
       }
     },
+    "node_modules/vite-plugin-static-copy": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/vite-plugin-static-copy/-/vite-plugin-static-copy-4.1.1.tgz",
+      "integrity": "sha512-GrlA8YklrAfSyxJ4M3fdQLOo9oNkp56IM9FYgX/WtEgeIFkPwhu4wzpufBCIuNKCa6Fn77FkRdYxkHqV0FwjAw==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "chokidar": "^3.6.0",
+        "p-map": "^7.0.4",
+        "picocolors": "^1.1.1",
+        "tinyglobby": "^0.2.17"
+      },
+      "engines": {
+        "node": "^22.0.0 || >=24.0.0"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/sapphi-red"
+      },
+      "peerDependencies": {
+        "vite": "^6.0.0 || ^7.0.0 || ^8.0.0"
+      }
+    },
+    "node_modules/vite-plugin-static-copy/node_modules/chokidar": {
+      "version": "3.6.0",
+      "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz",
+      "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "anymatch": "~3.1.2",
+        "braces": "~3.0.2",
+        "glob-parent": "~5.1.2",
+        "is-binary-path": "~2.1.0",
+        "is-glob": "~4.0.1",
+        "normalize-path": "~3.0.0",
+        "readdirp": "~3.6.0"
+      },
+      "engines": {
+        "node": ">= 8.10.0"
+      },
+      "funding": {
+        "url": "https://paulmillr.com/funding/"
+      },
+      "optionalDependencies": {
+        "fsevents": "~2.3.2"
+      }
+    },
+    "node_modules/vite-plugin-static-copy/node_modules/glob-parent": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
+      "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==",
+      "dev": true,
+      "license": "ISC",
+      "dependencies": {
+        "is-glob": "^4.0.1"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
+    "node_modules/vite-plugin-static-copy/node_modules/readdirp": {
+      "version": "3.6.0",
+      "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz",
+      "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "picomatch": "^2.2.1"
+      },
+      "engines": {
+        "node": ">=8.10.0"
+      }
+    },
     "node_modules/vite-plugin-svgr": {
       "version": "5.2.0",
       "resolved": "https://registry.npmjs.org/vite-plugin-svgr/-/vite-plugin-svgr-5.2.0.tgz",

src/frontend/package.json

@@ -17,11 +17,12 @@
   "dependencies": {
     "@fontsource-variable/atkinson-hyperlegible-next": "5.2.6",
     "@fontsource-variable/lexend": "5.2.11",
-    "@libreaudio/la-call": "0.1.4",
     "@fontsource/opendyslexic": "5.2.5",
+    "@libreaudio/la-call": "0.1.4",
     "@livekit/components-react": "2.9.21",
     "@livekit/components-styles": "1.2.0",
     "@livekit/track-processors": "0.7.2",
+    "@mediapipe/tasks-vision": "0.10.14",
     "@pandacss/preset-panda": "1.11.3",
     "@react-types/overlays": "3.10.0",
     "@remixicon/react": "4.9.0",
@@ -70,6 +71,7 @@
     "typescript": "6.0.3",
     "typescript-eslint": "8.60.1",
     "vite": "8.0.14",
+    "vite-plugin-static-copy": "4.1.1",
     "vite-plugin-svgr": "5.2.0"
   }
 }

src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts

@@ -13,8 +13,10 @@ import {
 } from './TimerWorker'
 import {
   BackgroundProcessorInterface,
+  SELFIE_SEGMENTER_MODEL_PATH,
   type ProcessorConfig,
   type ProcessorType,
+  MEDIAPIPE_PATH_WASM,
 } from '.'
 
 const PROCESSING_WIDTH = 256
@@ -153,13 +155,10 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
   }
 
   async initSegmenter() {
-    const vision = await FilesetResolver.forVisionTasks(
-      'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision/wasm'
-    )
+    const vision = await FilesetResolver.forVisionTasks(MEDIAPIPE_PATH_WASM)
     this.imageSegmenter = await ImageSegmenter.createFromOptions(vision, {
       baseOptions: {
-        modelAssetPath:
-          'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter_landscape/float16/latest/selfie_segmenter_landscape.tflite',
+        modelAssetPath: SELFIE_SEGMENTER_MODEL_PATH,
         delegate: 'CPU', // Use CPU for Firefox.
       },
       runningMode: 'VIDEO',

src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts

@@ -11,7 +11,11 @@ import {
   TIMEOUT_TICK,
   timerWorkerScript,
 } from './TimerWorker'
-import { ProcessorType } from '.'
+import {
+  FACE_LANDMARKS_MODEL_PATH,
+  ProcessorType,
+  MEDIAPIPE_PATH_WASM,
+} from '.'
 
 const PROCESSING_WIDTH = 256 * 3
 const PROCESSING_HEIGHT = 144 * 3
@@ -128,13 +132,10 @@ export class FaceLandmarksProcessor implements TrackProcessor<Track.Kind> {
   }
 
   async initFaceLandmarker() {
-    const vision = await FilesetResolver.forVisionTasks(
-      'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision/wasm'
-    )
+    const vision = await FilesetResolver.forVisionTasks(MEDIAPIPE_PATH_WASM)
     this.faceLandmarker = await FaceLandmarker.createFromOptions(vision, {
       baseOptions: {
-        modelAssetPath:
-          'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/latest/face_landmarker.task',
+        modelAssetPath: FACE_LANDMARKS_MODEL_PATH,
         delegate: 'GPU',
       },
       runningMode: 'VIDEO',

src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts

@@ -1,13 +1,14 @@
 import type { ProcessorOptions, Track } from 'livekit-client'
 import {
-  BackgroundBlur,
   ProcessorWrapper,
-  VirtualBackground,
+  BackgroundProcessor,
 } from '@livekit/track-processors'
 import {
   type ProcessorConfig,
   BackgroundProcessorInterface,
   ProcessorType,
+  MEDIAPIPE_PATH_WASM,
+  SELFIE_SEGMENTER_MODEL_PATH,
 } from '.'
 
 export class UnifiedBackgroundTrackProcessor implements BackgroundProcessorInterface {
@@ -20,10 +21,24 @@ export class UnifiedBackgroundTrackProcessor implements BackgroundProcessorInter
 
     if (opts.type === 'virtual') {
       this.processorType = ProcessorType.VIRTUAL
-      this.processor = VirtualBackground(opts.imagePath)
+      this.processor = BackgroundProcessor({
+        mode: 'virtual-background',
+        imagePath: opts.imagePath,
+        assetPaths: {
+          tasksVisionFileSet: MEDIAPIPE_PATH_WASM,
+          modelAssetPath: SELFIE_SEGMENTER_MODEL_PATH,
+        },
+      })
     } else if (opts.type === 'blur') {
       this.processorType = ProcessorType.BLUR
-      this.processor = BackgroundBlur(opts.blurRadius)
+      this.processor = BackgroundProcessor({
+        mode: 'background-blur',
+        blurRadius: opts.blurRadius,
+        assetPaths: {
+          tasksVisionFileSet: MEDIAPIPE_PATH_WASM,
+          modelAssetPath: SELFIE_SEGMENTER_MODEL_PATH,
+        },
+      })
     } else {
       throw new Error(
         'Must provide either imagePath for virtual background or blurRadius for blur'

src/frontend/src/features/rooms/livekit/components/blur/index.ts

@@ -4,6 +4,14 @@ import { BackgroundCustomProcessor } from './BackgroundCustomProcessor'
 import { UnifiedBackgroundTrackProcessor } from './UnifiedBackgroundTrackProcessor'
 import { FaceLandmarksOptions } from './FaceLandmarksProcessor'
 
+export const SELFIE_SEGMENTER_MODEL_PATH =
+  '/assets/mediapipe/models/selfie_segmenter_landscape.tflite'
+
+export const FACE_LANDMARKS_MODEL_PATH =
+  '/assets/mediapipe/models/face_landmarker.task'
+
+export const MEDIAPIPE_PATH_WASM = '/assets/mediapipe/wasm'
+
 export enum ProcessorType {
   BLUR = 'blur',
   VIRTUAL = 'virtual',

src/frontend/vite.config.ts

@@ -2,6 +2,7 @@ import { defineConfig, loadEnv } from 'vite'
 import react from '@vitejs/plugin-react'
 import { visualizer } from 'rollup-plugin-visualizer'
 import svgr from 'vite-plugin-svgr'
+import { viteStaticCopy } from 'vite-plugin-static-copy'
 
 // https://vitejs.dev/config/
 export default defineConfig(({ mode }) => {
@@ -11,18 +12,32 @@ export default defineConfig(({ mode }) => {
       react(),
       svgr({
         svgrOptions: {
-          replaceAttrValues: { '#000': 'currentColor', '#000000': 'currentColor', '#1f1f1f': 'currentColor' },
+          replaceAttrValues: {
+            '#000': 'currentColor',
+            '#000000': 'currentColor',
+            '#1f1f1f': 'currentColor',
+          },
         },
       }),
-      env.VITE_ANALYZE === 'true' && visualizer({
-        open: true,
-        filename: 'rollup-plugin-visualizer/stats.html',
-        gzipSize: true,
-        brotliSize: true,
+      viteStaticCopy({
+        targets: [
+          {
+            src: 'node_modules/@mediapipe/tasks-vision/wasm/*',
+            dest: 'assets/mediapipe/wasm',
+            rename: { stripBase: 4 },
+          },
+        ],
       }),
+      env.VITE_ANALYZE === 'true' &&
+        visualizer({
+          open: true,
+          filename: 'rollup-plugin-visualizer/stats.html',
+          gzipSize: true,
+          brotliSize: true,
+        }),
     ],
     resolve: {
-      tsconfigPaths: true
+      tsconfigPaths: true,
     },
     build: {
       sourcemap: env.VITE_BUILD_SOURCEMAP === 'true',