wip

compose.yml

@@ -267,6 +267,7 @@ services:
     build:
       context: ./src/agents
       target: development
+    command: ["python", "multi_user_transcriber.py", "dev"]
     env_file:
       - env.d/development/multi_user_transcriber
     volumes:

env.d/development/multi_user_transcriber.dist

@@ -1,14 +1,23 @@
+AWS_S3_ENDPOINT_URL=minio:9000
+AWS_S3_ACCESS_KEY_ID=meet
+AWS_S3_SECRET_ACCESS_KEY=password
+
 LIVEKIT_URL=ws://livekit:7880
 LIVEKIT_API_KEY=devkey
 LIVEKIT_API_SECRET=secret
 
-STT_PROVIDER=kyutai # kyutai, deepgram
+STT_PROVIDER=voxtral-vllm # voxtral-vllm, kyutai, deepgram
 ENABLE_SILERO_VAD=False
 
-DEEPGRAM_API_KEY=
+DEEPGRAM_API_KEY=your-deepgram-api-key
 
-KYUTAI_STT_BASE_URL=
-KYUTAI_API_KEY=
+KYUTAI_STT_BASE_URL=url
+KYUTAI_API_KEY=your-kyutai-api-key
+
+VOXTRAL_VLLM_BASE_URL=wss://<host>/v1/realtime
+VOXTRAL_VLLM_MODEL=voxtral-mini-4b-realtime-2602
+VOXTRAL_VLLM_API_KEY=your-vllm-api-key
+VOXTRAL_VLLM_TARGET_STREAMING_DELAY_MS=480
 
 SENTRY_DSN=
 SENTRY_ENVIRONMENT=

src/agents/multi_user_transcriber.py

@@ -25,6 +25,8 @@ from livekit.agents import (
 )
 from livekit.plugins import deepgram, silero
 
+import voxtral_vllm_stt
+
 from observability import configure_sentry, set_job_context
 from tasks import done_callback
 
@@ -49,6 +51,9 @@ def create_stt_provider():
         )
     elif STT_PROVIDER == "kyutai":
         _stt_instance = kyutai.STT(base_url=os.getenv("KYUTAI_STT_BASE_URL"))
+    elif STT_PROVIDER == "voxtral-vllm":
+        # The plugin resolves base_url / model / api_key from the environment.
+        _stt_instance = voxtral_vllm_stt.STT()
     else:
         raise ValueError(f"Unknown STT_PROVIDER: {STT_PROVIDER}")
 

src/agents/voxtral_vllm_stt.py

@@ -0,0 +1,474 @@
+"""LiveKit STT plugin for Voxtral Realtime served via vLLM (/v1/realtime).
+
+vLLM exposes Voxtral Realtime over a WebSocket that follows the OpenAI Realtime
+API protocol (not Mistral's proprietary realtime protocol).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import json
+import logging
+import os
+import weakref
+from collections import deque
+from dataclasses import dataclass, field
+
+import websockets
+from livekit.agents import (
+    DEFAULT_API_CONNECT_OPTIONS,
+    APIConnectionError,
+    APIConnectOptions,
+    APIStatusError,
+    stt,
+    utils,
+)
+from livekit.agents import (
+    vad as vad_module,
+)
+from livekit.agents.types import NOT_GIVEN, NotGivenOr
+from livekit.agents.utils import is_given
+
+logger = logging.getLogger("voxtral-vllm-stt")
+
+SAMPLE_RATE = 16000
+NUM_CHANNELS = 1
+CHUNK_SAMPLES = 1600  # 100 ms @ 16 kHz mono
+PREROLL_CHUNKS = 5  # keep 500 ms of audio before start of speech as detected by VAD
+
+# Reconnect policy: exponential backoff capped at MAX, give up after MAX_ATTEMPTS
+# consecutive failures (a successful handshake resets the counter).
+RECONNECT_BACKOFF_BASE_S = 0.5
+RECONNECT_BACKOFF_MAX_S = 8.0
+RECONNECT_MAX_ATTEMPTS = 5
+
+
+@dataclass
+class _STTOptions:
+    base_url: str
+    model: str
+    api_key: str | None
+    target_streaming_delay_ms: int | None
+
+
+@dataclass
+class _PendingUtterance:
+    """An utterance in flight on the shared websocket used for reconnect.
+
+    `sent_chunks` holds every chunk we have already enqueued for send on this
+    or a prior connection; on reconnect we replay them before resuming reads
+    from `queue`. vLLM concatenates `input_audio_buffer.append` events into a
+    single audio buffer per generation, so duplicates from a partial prior send
+    are harmless.
+    """
+
+    queue: asyncio.Queue[bytes | None]
+    sent_chunks: list[bytes] = field(default_factory=list)
+    ended: bool = False
+
+
+class STT(stt.STT):
+    """LiveKit STT speaking the OpenAI Realtime protocol served by vLLM."""
+
+    def __init__(
+        self,
+        *,
+        base_url: NotGivenOr[str] = NOT_GIVEN,
+        model: NotGivenOr[str] = NOT_GIVEN,
+        api_key: NotGivenOr[str] = NOT_GIVEN,
+        target_streaming_delay_ms: NotGivenOr[int] = NOT_GIVEN,
+        vad: vad_module.VAD | None = None,
+    ) -> None:
+        """Build the STT.
+
+        Args:
+            base_url: WebSocket URL of the vLLM realtime endpoint, e.g.
+                ws://example:8000/v1/realtime. Falls back to $VOXTRAL_VLLM_BASE_URL.
+            model: Model name exposed by vLLM, default
+                mistralai/Voxtral-Mini-4B-Realtime-2602.
+            api_key: Optional bearer token. Falls back to $VOXTRAL_VLLM_API_KEY.
+            target_streaming_delay_ms: Target streaming delay in ms forwarded to
+                vLLM via session.update. Falls back to
+                $VOXTRAL_VLLM_TARGET_STREAMING_DELAY_MS, else server default.
+            vad: Voice Activity Detector. If omitted, Silero VAD is loaded.
+        """
+        super().__init__(
+            capabilities=stt.STTCapabilities(streaming=True, interim_results=True)
+        )
+
+        resolved_url = (
+            base_url
+            if is_given(base_url)
+            else os.environ.get(
+                "VOXTRAL_VLLM_BASE_URL", "ws://127.0.0.1:8000/v1/realtime"
+            )
+        )
+        resolved_model = (
+            model
+            if is_given(model)
+            else os.environ.get(
+                "VOXTRAL_VLLM_MODEL", "mistralai/Voxtral-Mini-4B-Realtime-2602"
+            )
+        )
+        resolved_key = (
+            api_key if is_given(api_key) else os.environ.get("VOXTRAL_VLLM_API_KEY")
+        )
+        resolved_delay = (
+            target_streaming_delay_ms
+            if is_given(target_streaming_delay_ms)
+            else (
+                int(os.environ["VOXTRAL_VLLM_TARGET_STREAMING_DELAY_MS"])
+                if os.environ.get("VOXTRAL_VLLM_TARGET_STREAMING_DELAY_MS")
+                else None
+            )
+        )
+
+        if vad is None:
+            try:
+                from livekit.plugins.silero import VAD as SileroVAD  # noqa: PLC0415
+            except ImportError as exc:
+                raise ImportError(
+                    "livekit-plugins-silero is required for vLLM Voxtral realtime "
+                    "(no server-side endpointing)."
+                ) from exc
+            vad = SileroVAD.load()
+        self._vad = vad
+
+        self._opts = _STTOptions(
+            base_url=resolved_url,
+            model=resolved_model,
+            api_key=resolved_key,
+            target_streaming_delay_ms=resolved_delay,
+        )
+        self._streams: weakref.WeakSet[SpeechStream] = weakref.WeakSet()
+
+    @property
+    def model(self) -> str:
+        """Return the configured vLLM model name."""
+        return self._opts.model
+
+    @property
+    def provider(self) -> str:
+        """Return the provider identifier."""
+        return "vllm-voxtral-realtime"
+
+    async def _recognize_impl(self, *_args, **_kwargs) -> stt.SpeechEvent:
+        raise NotImplementedError(
+            "vLLM Voxtral Realtime STT only supports streaming recognition."
+        )
+
+    def stream(
+        self,
+        *,
+        conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
+    ) -> SpeechStream:
+        """Open a new streaming recognition stream."""
+        s = SpeechStream(
+            stt=self,
+            opts=self._opts,
+            vad_instance=self._vad,
+            conn_options=conn_options,
+        )
+        self._streams.add(s)
+        return s
+
+
+class SpeechStream(stt.RecognizeStream):
+    def __init__(
+        self,
+        *,
+        stt: STT,
+        opts: _STTOptions,
+        vad_instance: vad_module.VAD,
+        conn_options: APIConnectOptions,
+    ) -> None:
+        """Init the speech stream."""
+        super().__init__(stt=stt, conn_options=conn_options, sample_rate=SAMPLE_RATE)
+        self._opts = opts
+        self._vad = vad_instance
+        self._utterance_q: asyncio.Queue[bytes | None] | None = None
+        self._speaking = False
+        self._preroll: deque[bytes] = deque(maxlen=PREROLL_CHUNKS)
+        # Voxtral realtime is strictly sequential: only one generation runs at a
+        # time, and a new `commit` is ignored while the previous one is still
+        # producing. We queue per-utterance audio buffers here and let the
+        # pipeline process them one by one on the shared websocket.
+        self._utterance_chan: asyncio.Queue[asyncio.Queue[bytes | None] | None] = (
+            asyncio.Queue()
+        )
+
+    @utils.log_exceptions(logger=logger)
+    async def _run(self) -> None:
+        vad_stream = self._vad.stream()
+
+        bstream = utils.audio.AudioByteStream(
+            sample_rate=SAMPLE_RATE,
+            num_channels=NUM_CHANNELS,
+            samples_per_channel=CHUNK_SAMPLES,
+        )
+
+        async def input_task() -> None:
+            async for data in self._input_ch:
+                if isinstance(data, self._FlushSentinel):
+                    for frame in bstream.flush():
+                        self._handle_chunk(frame.data.tobytes())
+                    continue
+
+                vad_stream.push_frame(data)
+                for frame in bstream.write(data.data.tobytes()):
+                    self._handle_chunk(frame.data.tobytes())
+
+            vad_stream.end_input()
+
+        async def vad_task() -> None:
+            async for ev in vad_stream:
+                if ev.type == vad_module.VADEventType.START_OF_SPEECH:
+                    self._on_start_of_speech()
+                elif ev.type == vad_module.VADEventType.END_OF_SPEECH:
+                    self._on_end_of_speech()
+
+        pipeline_t = asyncio.create_task(self._utterance_pipeline())
+        try:
+            await asyncio.gather(input_task(), vad_task())
+            # signal end-of-stream; pipeline finishes pending utterances first
+            self._utterance_chan.put_nowait(None)
+            await pipeline_t
+        except (APIStatusError, APIConnectionError, asyncio.CancelledError):
+            raise
+        except Exception as exc:
+            logger.exception("vLLM realtime stream failed")
+            raise APIConnectionError() from exc
+        finally:
+            if not pipeline_t.done():
+                pipeline_t.cancel()
+            try:
+                await pipeline_t
+            except asyncio.CancelledError:
+                # CancelledError is the expected flow on cancel()
+                pass
+            except Exception:
+                logger.exception("utterance pipeline failed during finalize")
+            await vad_stream.aclose()
+
+    def _handle_chunk(self, chunk: bytes) -> None:
+        self._preroll.append(chunk)
+        if self._speaking and self._utterance_q is not None:
+            self._utterance_q.put_nowait(chunk)
+
+    def _on_start_of_speech(self) -> None:
+        if self._speaking:
+            return
+        self._speaking = True
+        q: asyncio.Queue[bytes | None] = asyncio.Queue()
+        for chunk in self._preroll:
+            q.put_nowait(chunk)
+        self._utterance_q = q
+        self._utterance_chan.put_nowait(q)
+        self._event_ch.send_nowait(
+            stt.SpeechEvent(type=stt.SpeechEventType.START_OF_SPEECH)
+        )
+
+    def _on_end_of_speech(self) -> None:
+        if not self._speaking:
+            return
+        self._speaking = False
+        if self._utterance_q is not None:
+            self._utterance_q.put_nowait(None)
+        self._utterance_q = None
+        self._event_ch.send_nowait(
+            stt.SpeechEvent(type=stt.SpeechEventType.END_OF_SPEECH)
+        )
+
+    async def _handshake(self, ws: websockets.ClientConnection) -> str:
+        created = json.loads(await ws.recv())
+        if created.get("type") != "session.created":
+            raise APIStatusError(
+                f"expected session.created, got {created}",
+                status_code=500,
+                body=created,
+            )
+        session_update: dict = {"type": "session.update", "model": self._opts.model}
+        if self._opts.target_streaming_delay_ms is not None:
+            session_update["target_streaming_delay_ms"] = (
+                self._opts.target_streaming_delay_ms
+            )
+        await ws.send(json.dumps(session_update))
+        return created.get("id", "")
+
+    def _auth_headers(self) -> dict[str, str]:
+        if self._opts.api_key:
+            return {"Authorization": f"Bearer {self._opts.api_key}"}
+        return {}
+
+    async def _utterance_pipeline(self) -> None:
+        # Owns the websocket lifecycle. On drop, reopens and resumes the
+        # in-flight utterance (if any) by replaying its already-sent chunks.
+        pending: _PendingUtterance | None = None
+        attempt = 0
+        while True:
+            try:
+                async with websockets.connect(
+                    self._opts.base_url,
+                    additional_headers=self._auth_headers(),
+                    open_timeout=self._conn_options.timeout,
+                ) as ws:
+                    request_id = await self._handshake(ws)
+                    attempt = 0
+                    while True:
+                        if pending is None:
+                            q = await self._utterance_chan.get()
+                            if q is None:
+                                return
+                            pending = _PendingUtterance(queue=q)
+                        await self._process_utterance(ws, pending, request_id)
+                        pending = None
+            except (websockets.WebSocketException, OSError, TimeoutError) as exc:
+                attempt += 1
+                if attempt > RECONNECT_MAX_ATTEMPTS:
+                    logger.exception(
+                        "vLLM realtime: giving up after %d reconnect attempts",
+                        RECONNECT_MAX_ATTEMPTS,
+                    )
+                    raise APIConnectionError() from exc
+                backoff = min(
+                    RECONNECT_BACKOFF_BASE_S * (2 ** (attempt - 1)),
+                    RECONNECT_BACKOFF_MAX_S,
+                )
+                if pending is None:
+                    logger.warning(
+                        "vLLM WS connection lost between utterances "
+                        "(attempt %d/%d): %s; retrying in %.1fs",
+                        attempt,
+                        RECONNECT_MAX_ATTEMPTS,
+                        exc,
+                        backoff,
+                    )
+                else:
+                    logger.warning(
+                        "vLLM WS dropped mid-utterance (%d chunks buffered, "
+                        "ended=%s, attempt %d/%d): %s; retrying in %.1fs",
+                        len(pending.sent_chunks),
+                        pending.ended,
+                        attempt,
+                        RECONNECT_MAX_ATTEMPTS,
+                        exc,
+                        backoff,
+                    )
+                await asyncio.sleep(backoff)
+
+    async def _process_utterance(
+        self,
+        ws: websockets.ClientConnection,
+        pending: _PendingUtterance,
+        request_id: str,
+    ) -> None:
+        # Start a fresh generation. Safe to send here: the previous utterance's
+        # transcription.done has already been received (we await it below), so
+        # the server-side generation_task is done and won't ignore this commit.
+        await ws.send(json.dumps({"type": "input_audio_buffer.commit"}))
+        send_t = asyncio.create_task(self._send_audio(ws, pending))
+        try:
+            await self._receive_one_transcription(ws, request_id)
+        finally:
+            if not send_t.done():
+                send_t.cancel()
+            try:
+                await send_t
+            except (asyncio.CancelledError, websockets.WebSocketException):
+                pass
+            except Exception:
+                logger.exception("send-audio task failed during finalize")
+
+    @staticmethod
+    async def _send_audio(
+        ws: websockets.ClientConnection, pending: _PendingUtterance
+    ) -> None:
+        # Replay anything already sent on a previous (now-dead) connection.
+        # sent_chunks is appended before send, so a chunk that failed to send
+        # last time is still present and gets retried here.
+        for chunk in pending.sent_chunks:
+            await ws.send(
+                json.dumps(
+                    {
+                        "type": "input_audio_buffer.append",
+                        "audio": base64.b64encode(chunk).decode("ascii"),
+                    }
+                )
+            )
+        if pending.ended:
+            await ws.send(
+                json.dumps({"type": "input_audio_buffer.commit", "final": True})
+            )
+            return
+        while True:
+            chunk = await pending.queue.get()
+            if chunk is None:
+                pending.ended = True
+                await ws.send(
+                    json.dumps({"type": "input_audio_buffer.commit", "final": True})
+                )
+                return
+            pending.sent_chunks.append(chunk)
+            await ws.send(
+                json.dumps(
+                    {
+                        "type": "input_audio_buffer.append",
+                        "audio": base64.b64encode(chunk).decode("ascii"),
+                    }
+                )
+            )
+
+    async def _receive_one_transcription(
+        self, ws: websockets.ClientConnection, request_id: str
+    ) -> None:
+        # Use recv() rather than `async for`: the latter swallows
+        # ConnectionClosed on close-mid-iteration, which would let a dropped
+        # WS look like a clean "no transcription" return.
+        current_text = ""
+        while True:
+            raw = await ws.recv()
+            data = json.loads(raw)
+            event_type = data.get("type")
+
+            if event_type == "transcription.delta":
+                delta = data.get("delta", "")
+                if not delta:
+                    continue
+                current_text += delta
+                self._event_ch.send_nowait(
+                    stt.SpeechEvent(
+                        type=stt.SpeechEventType.INTERIM_TRANSCRIPT,
+                        request_id=request_id,
+                        alternatives=[stt.SpeechData(text=current_text, language="")],
+                    )
+                )
+            elif event_type == "transcription.done":
+                final_text = data.get("text") or current_text
+                self._event_ch.send_nowait(
+                    stt.SpeechEvent(
+                        type=stt.SpeechEventType.FINAL_TRANSCRIPT,
+                        request_id=request_id,
+                        alternatives=[stt.SpeechData(text=final_text, language="")],
+                    )
+                )
+                usage = data.get("usage") or {}
+                self._event_ch.send_nowait(
+                    stt.SpeechEvent(
+                        type=stt.SpeechEventType.RECOGNITION_USAGE,
+                        request_id=request_id,
+                        recognition_usage=stt.RecognitionUsage(
+                            audio_duration=float(
+                                usage.get("audio_seconds")
+                                or usage.get("prompt_audio_seconds")
+                                or 0
+                            ),
+                            input_tokens=int(usage.get("prompt_tokens") or 0),
+                            output_tokens=int(usage.get("completion_tokens") or 0),
+                        ),
+                    )
+                )
+                return
+            elif event_type == "error":
+                err = data.get("error")
+                raise APIStatusError(str(err), status_code=500, body=data)

src/frontend/src/features/subtitle/component/Subtitles.tsx

@@ -75,15 +75,15 @@ const useTranscriptionState = () => {
     const segment = segments[0]
 
     setTranscriptionSegments((prevSegments) => {
-      const existingSegmentIds = new Set(prevSegments.map((s) => s.id))
-      if (existingSegmentIds.has(segment.id)) return prevSegments
-      return [
-        ...prevSegments,
-        {
-          participant: participant,
-          ...segment,
-        },
-      ]
+      const existingIndex = prevSegments.findIndex(
+        (s: TranscriptionSegmentWithParticipant) => s.id === segment.id
+      )
+      if (existingIndex === -1) {
+        return [...prevSegments, { participant, ...segment }]
+      }
+      const next = prevSegments.slice()
+      next[existingIndex] = { ...next[existingIndex], ...segment }
+      return next
     })
   }