diff --git a/CHANGELOG.md b/CHANGELOG.md index 16120c3e..51848936 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ and this project adheres to - ⬆️(mail) update mjml to v5 and @html-to/text-cli - 🚸(frontend) initialize the join input name with the persisted full name - ♻️(frontend) refactor background processors to use the new API +- ♻️(frontend) inline model weights to avoid loading them from remote ### Fixed diff --git a/src/frontend/public/assets/mediapipe/models/face_landmarker.task b/src/frontend/public/assets/mediapipe/models/face_landmarker.task new file mode 100644 index 00000000..c50c845d Binary files /dev/null and b/src/frontend/public/assets/mediapipe/models/face_landmarker.task differ diff --git a/src/frontend/public/assets/mediapipe/models/selfie_segmenter_landscape.tflite b/src/frontend/public/assets/mediapipe/models/selfie_segmenter_landscape.tflite new file mode 100644 index 00000000..3a508f8d Binary files /dev/null and b/src/frontend/public/assets/mediapipe/models/selfie_segmenter_landscape.tflite differ diff --git a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts index d13b047b..95b5ed32 100644 --- a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts +++ b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts @@ -13,6 +13,7 @@ import { } from './TimerWorker' import { BackgroundProcessorInterface, + SELFIE_SEGMENTER_MODEL_PATH, type ProcessorConfig, type ProcessorType, } from '.' @@ -158,8 +159,7 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface { ) this.imageSegmenter = await ImageSegmenter.createFromOptions(vision, { baseOptions: { - modelAssetPath: - 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter_landscape/float16/latest/selfie_segmenter_landscape.tflite', + modelAssetPath: SELFIE_SEGMENTER_MODEL_PATH, delegate: 'CPU', // Use CPU for Firefox. }, runningMode: 'VIDEO', diff --git a/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts index 7299ae44..b6f6e3f2 100644 --- a/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts +++ b/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts @@ -11,7 +11,7 @@ import { TIMEOUT_TICK, timerWorkerScript, } from './TimerWorker' -import { ProcessorType } from '.' +import { FACE_LANDMARKS_MODEL_PATH, ProcessorType } from '.' const PROCESSING_WIDTH = 256 * 3 const PROCESSING_HEIGHT = 144 * 3 @@ -133,8 +133,7 @@ export class FaceLandmarksProcessor implements TrackProcessor { ) this.faceLandmarker = await FaceLandmarker.createFromOptions(vision, { baseOptions: { - modelAssetPath: - 'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/latest/face_landmarker.task', + modelAssetPath: FACE_LANDMARKS_MODEL_PATH, delegate: 'GPU', }, runningMode: 'VIDEO', diff --git a/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts index 0bf9c4bb..237e790c 100644 --- a/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts +++ b/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts @@ -7,6 +7,7 @@ import { type ProcessorConfig, BackgroundProcessorInterface, ProcessorType, + SELFIE_SEGMENTER_MODEL_PATH, } from '.' export class UnifiedBackgroundTrackProcessor implements BackgroundProcessorInterface { @@ -22,12 +23,18 @@ export class UnifiedBackgroundTrackProcessor implements BackgroundProcessorInter this.processor = BackgroundProcessor({ mode: 'virtual-background', imagePath: opts.imagePath, + assetPaths: { + modelAssetPath: SELFIE_SEGMENTER_MODEL_PATH, + }, }) } else if (opts.type === 'blur') { this.processorType = ProcessorType.BLUR this.processor = BackgroundProcessor({ mode: 'background-blur', blurRadius: opts.blurRadius, + assetPaths: { + modelAssetPath: SELFIE_SEGMENTER_MODEL_PATH, + }, }) } else { throw new Error( diff --git a/src/frontend/src/features/rooms/livekit/components/blur/index.ts b/src/frontend/src/features/rooms/livekit/components/blur/index.ts index f9e9c6d4..892af9bd 100644 --- a/src/frontend/src/features/rooms/livekit/components/blur/index.ts +++ b/src/frontend/src/features/rooms/livekit/components/blur/index.ts @@ -4,6 +4,12 @@ import { BackgroundCustomProcessor } from './BackgroundCustomProcessor' import { UnifiedBackgroundTrackProcessor } from './UnifiedBackgroundTrackProcessor' import { FaceLandmarksOptions } from './FaceLandmarksProcessor' +export const SELFIE_SEGMENTER_MODEL_PATH = + '/assets/mediapipe/models/selfie_segmenter_landscape.tflite' + +export const FACE_LANDMARKS_MODEL_PATH = + '/assets/mediapipe/models/face_landmarker.task' + export enum ProcessorType { BLUR = 'blur', VIRTUAL = 'virtual',