✨(agent) support Voxtral realtime as inference engine

The Kyutai open-source model turned out not to be production-ready:
it caused disruptions in the production environment, especially on
long-running meeting sessions.

Switch to the Voxtral realtime model, which looks like a
credible competitor and behaves much better in our setup.

For now, the code handling the Voxtral realtime API lives directly
in the project. It could be extracted into an open-source package
later.

See PR #1277 for the full details of the implementation, proposed
by @cameldev.

CHANGELOG.md

@@ -8,6 +8,10 @@ and this project adheres to
 
 ## [Unreleased]
 
+### Added
+
+- ✨(agent) support Voxtral realtime as inference engine
+
 ### Changed
 
 - 🔥(backend) remove the S3 storage-event webhook for recordings

compose.yml

@@ -251,6 +251,7 @@ services:
     build:
       context: ./src/agents
       target: development
+    command: ["python", "multi_user_transcriber.py", "dev"]
     env_file:
       - env.d/development/multi_user_transcriber
     volumes:

env.d/development/multi_user_transcriber.dist

@@ -1,14 +1,23 @@
+AWS_S3_ENDPOINT_URL=minio:9000
+AWS_S3_ACCESS_KEY_ID=meet
+AWS_S3_SECRET_ACCESS_KEY=password
+
 LIVEKIT_URL=ws://livekit:7880
 LIVEKIT_API_KEY=devkey
 LIVEKIT_API_SECRET=secret
 
-STT_PROVIDER=kyutai # kyutai, deepgram
+STT_PROVIDER=voxtral-vllm # voxtral-vllm, kyutai, deepgram
 ENABLE_SILERO_VAD=False
 
-DEEPGRAM_API_KEY=
+DEEPGRAM_API_KEY=your-deepgram-api-key
 
-KYUTAI_STT_BASE_URL=
+KYUTAI_STT_BASE_URL=url
-KYUTAI_API_KEY=
+KYUTAI_API_KEY=your-kyutai-api-key
+
+VOXTRAL_VLLM_BASE_URL=wss://<host>/v1/realtime
+VOXTRAL_VLLM_MODEL=voxtral-mini-4b-realtime-2602
+VOXTRAL_VLLM_API_KEY=your-vllm-api-key
+VOXTRAL_VLLM_TARGET_STREAMING_DELAY_MS=480
 
 SENTRY_DSN=
 SENTRY_ENVIRONMENT=

src/agents/multi_user_transcriber.py

@@ -1,6 +1,7 @@
 """Multi user transcription agent."""
 
 import asyncio
+import contextlib
 import logging
 import os
 
@@ -25,6 +26,7 @@ from livekit.agents import (
 )
 from livekit.plugins import deepgram, silero
 
+import voxtral_vllm_stt
 from observability import configure_sentry, set_job_context
 from tasks import done_callback
 
@@ -36,9 +38,18 @@ TRANSCRIBER_AGENT_NAME = os.getenv("TRANSCRIBER_AGENT_NAME", "multi-user-transcr
 STT_PROVIDER = os.getenv("STT_PROVIDER", "deepgram")
 ENABLE_SILERO_VAD = os.getenv("ENABLE_SILERO_VAD", "true").lower() == "true"
 
+SESSION_DRAIN_TIMEOUT_S = 15.0
 
-def create_stt_provider():
+
-    """Create STT provider based on environment configuration."""
+def create_stt_provider(vad: silero.VAD | None = None):
+    """Create STT provider based on environment configuration.
+
+    Args:
+        vad: Shared, prewarmed VAD instance. Required in practice for
+            voxtral-vllm (no server-side endpointing): if omitted, the plugin
+            loads its own Silero model synchronously on the event loop, once
+            per participant, freezing all active sessions for the duration.
+    """
     if STT_PROVIDER == "deepgram":
         # Note: Not all Deepgram API parameters are supported by the LiveKit plugin
         # detect_language is NOT supported for real-time streaming
@@ -49,6 +60,9 @@ def create_stt_provider():
         )
     elif STT_PROVIDER == "kyutai":
         _stt_instance = kyutai.STT(base_url=os.getenv("KYUTAI_STT_BASE_URL"))
+    elif STT_PROVIDER == "voxtral-vllm":
+        # The plugin resolves base_url / model / api_key from the environment.
+        _stt_instance = voxtral_vllm_stt.STT(vad=vad)
     else:
         raise ValueError(f"Unknown STT_PROVIDER: {STT_PROVIDER}")
 
@@ -58,9 +72,9 @@ def create_stt_provider():
 class Transcriber(Agent):
     """Create a transcription agent for a specific participant."""
 
-    def __init__(self, *, participant_identity: str):
+    def __init__(self, *, participant_identity: str, vad: silero.VAD | None = None):
         """Init transcription agent."""
-        stt = create_stt_provider()
+        stt = create_stt_provider(vad=vad)
 
         super().__init__(
             instructions="not-needed",
@@ -76,6 +90,7 @@ class MultiUserTranscriber:
         """Init multi user transcription agent."""
         self.ctx = ctx
         self._sessions: dict[str, AgentSession] = {}
+        self._starting: dict[str, asyncio.Task] = {}
         self._tasks: set[asyncio.Task] = set()
 
     def start(self):
@@ -96,22 +111,30 @@ class MultiUserTranscriber:
 
     def on_participant_connected(self, participant: rtc.RemoteParticipant):
         """Handle new participant connection by starting transcription session."""
-        if participant.identity in self._sessions:
+        identity = participant.identity
+        if identity in self._sessions or identity in self._starting:
             return
 
-        logger.info(f"starting session for {participant.identity}")
+        logger.info(f"starting session for {identity}")
         task = asyncio.create_task(self._start_session(participant))
+        self._starting[identity] = task
         self._tasks.add(task)
+        task.add_done_callback(lambda t, i=identity: self._starting.pop(i, None))
         task.add_done_callback(
             done_callback(
                 logger,
                 self._tasks,
-                f"start transcription session for {participant.identity}",
+                f"start transcription session for {identity}",
             )
         )
 
     def on_participant_disconnected(self, participant: rtc.RemoteParticipant):
         """Handle participant disconnection by closing transcription session."""
+        if (start_task := self._starting.pop(participant.identity, None)) is not None:
+            logger.info(f"cancelling pending session start for {participant.identity}")
+            start_task.cancel()
+            return
+
         if (session := self._sessions.pop(participant.identity, None)) is None:
             return
 
@@ -127,10 +150,12 @@ class MultiUserTranscriber:
         )
 
     async def _start_session(self, participant: rtc.RemoteParticipant) -> AgentSession:
-        """Create and start transcription session for participant."""
+        """Create and start transcription session for participant.
-        if participant.identity in self._sessions:
-            return self._sessions[participant.identity]
 
+        Deduplication happens synchronously in on_participant_connected via
+        self._starting; by the time this coroutine runs, the identity is
+        already reserved.
+        """
         vad = self.ctx.proc.userdata.get("vad", None)
         session = AgentSession(vad=vad)
         room_io = RoomIO(
@@ -141,18 +166,30 @@ class MultiUserTranscriber:
                 text_input=False, audio_output=False, text_output=True
             ),
         )
+        try:
             await room_io.start()
             await session.start(
                 agent=Transcriber(
                     participant_identity=participant.identity,
+                    vad=vad,
                 )
             )
+        except BaseException:
+            with contextlib.suppress(Exception):
+                await session.aclose()
+            raise
         self._sessions[participant.identity] = session
         return session
 
     async def _close_session(self, sess: AgentSession) -> None:
         """Close and cleanup transcription session."""
-        await sess.drain()
+        try:
+            await asyncio.wait_for(sess.drain(), timeout=SESSION_DRAIN_TIMEOUT_S)
+        except (TimeoutError, asyncio.TimeoutError):
+            logger.warning(
+                "session drain timed out after %.0fs; forcing close",
+                SESSION_DRAIN_TIMEOUT_S,
+            )
         await sess.aclose()
 
 

src/agents/pyproject.toml

@@ -12,6 +12,8 @@ dependencies = [
     "protobuf==6.33.6",
     "minio==7.2.20",
     "sentry-sdk==2.66.1",
+    "websockets==17.1",
+    "httpx==0.28.1",
 ]
 
 [project.optional-dependencies]

src/agents/voxtral_vllm_stt.py

@@ -0,0 +1,476 @@
+"""LiveKit STT plugin for Voxtral Realtime served via vLLM (/v1/realtime).
+
+vLLM exposes Voxtral Realtime over a WebSocket that follows the OpenAI Realtime
+API protocol (not Mistral's proprietary realtime protocol).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import json
+import logging
+import os
+import weakref
+from collections import deque
+from dataclasses import dataclass, field
+
+import websockets
+from livekit.agents import (
+    DEFAULT_API_CONNECT_OPTIONS,
+    APIConnectionError,
+    APIConnectOptions,
+    APIStatusError,
+    stt,
+    utils,
+)
+from livekit.agents import (
+    vad as vad_module,
+)
+from livekit.agents.types import NOT_GIVEN, NotGivenOr
+from livekit.agents.utils import is_given
+
+logger = logging.getLogger("voxtral-vllm-stt")
+
+SAMPLE_RATE = 16000
+NUM_CHANNELS = 1
+CHUNK_SAMPLES = 1600  # 100 ms @ 16 kHz mono
+PREROLL_CHUNKS = 5  # keep 500 ms of audio before start of speech as detected by VAD
+
+# Reconnect policy: exponential backoff capped at MAX, give up after MAX_ATTEMPTS
+# consecutive failures (a successful handshake resets the counter).
+RECONNECT_BACKOFF_BASE_S = 0.5
+RECONNECT_BACKOFF_MAX_S = 8.0
+RECONNECT_MAX_ATTEMPTS = 5
+
+
+@dataclass
+class _STTOptions:
+    base_url: str
+    model: str
+    api_key: str | None
+    target_streaming_delay_ms: int | None
+
+
+@dataclass
+class _PendingUtterance:
+    """An utterance in flight on the shared websocket used for reconnect.
+
+    `sent_chunks` holds every chunk we have already enqueued for send on this
+    or a prior connection; on reconnect we replay them before resuming reads
+    from `queue`. vLLM concatenates `input_audio_buffer.append` events into a
+    single audio buffer per generation, so duplicates from a partial prior send
+    are harmless.
+    """
+
+    queue: asyncio.Queue[bytes | None]
+    sent_chunks: list[bytes] = field(default_factory=list)
+    ended: bool = False
+
+
+class STT(stt.STT):
+    """LiveKit STT speaking the OpenAI Realtime protocol served by vLLM."""
+
+    def __init__(
+        self,
+        *,
+        base_url: NotGivenOr[str] = NOT_GIVEN,
+        model: NotGivenOr[str] = NOT_GIVEN,
+        api_key: NotGivenOr[str] = NOT_GIVEN,
+        target_streaming_delay_ms: NotGivenOr[int] = NOT_GIVEN,
+        vad: vad_module.VAD | None = None,
+    ) -> None:
+        """Build the STT.
+
+        Args:
+            base_url: WebSocket URL of the vLLM realtime endpoint, e.g.
+                ws://example:8000/v1/realtime. Falls back to $VOXTRAL_VLLM_BASE_URL.
+            model: Model name exposed by vLLM, default
+                mistralai/Voxtral-Mini-4B-Realtime-2602.
+            api_key: Optional bearer token. Falls back to $VOXTRAL_VLLM_API_KEY.
+            target_streaming_delay_ms: Target streaming delay in ms forwarded to
+                vLLM via session.update. Falls back to
+                $VOXTRAL_VLLM_TARGET_STREAMING_DELAY_MS, else server default.
+            vad: Voice Activity Detector. If omitted, Silero VAD is loaded.
+        """
+        super().__init__(
+            capabilities=stt.STTCapabilities(streaming=True, interim_results=True)
+        )
+
+        resolved_url = (
+            base_url
+            if is_given(base_url)
+            else os.environ.get(
+                "VOXTRAL_VLLM_BASE_URL", "ws://127.0.0.1:8000/v1/realtime"
+            )
+        )
+        resolved_model = (
+            model
+            if is_given(model)
+            else os.environ.get(
+                "VOXTRAL_VLLM_MODEL", "mistralai/Voxtral-Mini-4B-Realtime-2602"
+            )
+        )
+        resolved_key = (
+            api_key if is_given(api_key) else os.environ.get("VOXTRAL_VLLM_API_KEY")
+        )
+        resolved_delay = (
+            target_streaming_delay_ms
+            if is_given(target_streaming_delay_ms)
+            else (
+                int(os.environ["VOXTRAL_VLLM_TARGET_STREAMING_DELAY_MS"])
+                if os.environ.get("VOXTRAL_VLLM_TARGET_STREAMING_DELAY_MS")
+                else None
+            )
+        )
+
+        if vad is None:
+            try:
+                from livekit.plugins.silero import VAD as SileroVAD  # noqa: PLC0415
+            except ImportError as exc:
+                raise ImportError(
+                    "livekit-plugins-silero is required for vLLM Voxtral realtime "
+                    "(no server-side endpointing)."
+                ) from exc
+            vad = SileroVAD.load()
+        self._vad = vad
+
+        self._opts = _STTOptions(
+            base_url=resolved_url,
+            model=resolved_model,
+            api_key=resolved_key,
+            target_streaming_delay_ms=resolved_delay,
+        )
+        self._streams: weakref.WeakSet[SpeechStream] = weakref.WeakSet()
+
+    @property
+    def model(self) -> str:
+        """Return the configured vLLM model name."""
+        return self._opts.model
+
+    @property
+    def provider(self) -> str:
+        """Return the provider identifier."""
+        return "vllm-voxtral-realtime"
+
+    async def _recognize_impl(self, *_args, **_kwargs) -> stt.SpeechEvent:
+        raise NotImplementedError(
+            "vLLM Voxtral Realtime STT only supports streaming recognition."
+        )
+
+    def stream(
+        self,
+        *,
+        conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
+    ) -> SpeechStream:
+        """Open a new streaming recognition stream."""
+        s = SpeechStream(
+            stt=self,
+            opts=self._opts,
+            vad_instance=self._vad,
+            conn_options=conn_options,
+        )
+        self._streams.add(s)
+        return s
+
+
+class SpeechStream(stt.RecognizeStream):
+    """Voxtral realtime handler."""
+
+    def __init__(
+        self,
+        *,
+        stt: STT,
+        opts: _STTOptions,
+        vad_instance: vad_module.VAD,
+        conn_options: APIConnectOptions,
+    ) -> None:
+        """Init the speech stream."""
+        super().__init__(stt=stt, conn_options=conn_options, sample_rate=SAMPLE_RATE)
+        self._opts = opts
+        self._vad = vad_instance
+        self._utterance_q: asyncio.Queue[bytes | None] | None = None
+        self._speaking = False
+        self._preroll: deque[bytes] = deque(maxlen=PREROLL_CHUNKS)
+        # Voxtral realtime is strictly sequential: only one generation runs at a
+        # time, and a new `commit` is ignored while the previous one is still
+        # producing. We queue per-utterance audio buffers here and let the
+        # pipeline process them one by one on the shared websocket.
+        self._utterance_chan: asyncio.Queue[asyncio.Queue[bytes | None] | None] = (
+            asyncio.Queue()
+        )
+
+    @utils.log_exceptions(logger=logger)
+    async def _run(self) -> None:
+        vad_stream = self._vad.stream()
+
+        bstream = utils.audio.AudioByteStream(
+            sample_rate=SAMPLE_RATE,
+            num_channels=NUM_CHANNELS,
+            samples_per_channel=CHUNK_SAMPLES,
+        )
+
+        async def input_task() -> None:
+            async for data in self._input_ch:
+                if isinstance(data, self._FlushSentinel):
+                    for frame in bstream.flush():
+                        self._handle_chunk(frame.data.tobytes())
+                    continue
+
+                vad_stream.push_frame(data)
+                for frame in bstream.write(data.data.tobytes()):
+                    self._handle_chunk(frame.data.tobytes())
+
+            vad_stream.end_input()
+
+        async def vad_task() -> None:
+            async for ev in vad_stream:
+                if ev.type == vad_module.VADEventType.START_OF_SPEECH:
+                    self._on_start_of_speech()
+                elif ev.type == vad_module.VADEventType.END_OF_SPEECH:
+                    self._on_end_of_speech()
+
+        pipeline_t = asyncio.create_task(self._utterance_pipeline())
+        try:
+            await asyncio.gather(input_task(), vad_task())
+            # signal end-of-stream; pipeline finishes pending utterances first
+            self._utterance_chan.put_nowait(None)
+            await pipeline_t
+        except (APIStatusError, APIConnectionError, asyncio.CancelledError):
+            raise
+        except Exception as exc:
+            logger.exception("vLLM realtime stream failed")
+            raise APIConnectionError() from exc
+        finally:
+            if not pipeline_t.done():
+                pipeline_t.cancel()
+            try:
+                await pipeline_t
+            except asyncio.CancelledError:
+                # CancelledError is the expected flow on cancel()
+                pass
+            except Exception:
+                logger.exception("utterance pipeline failed during finalize")
+            await vad_stream.aclose()
+
+    def _handle_chunk(self, chunk: bytes) -> None:
+        self._preroll.append(chunk)
+        if self._speaking and self._utterance_q is not None:
+            self._utterance_q.put_nowait(chunk)
+
+    def _on_start_of_speech(self) -> None:
+        if self._speaking:
+            return
+        self._speaking = True
+        q: asyncio.Queue[bytes | None] = asyncio.Queue()
+        for chunk in self._preroll:
+            q.put_nowait(chunk)
+        self._utterance_q = q
+        self._utterance_chan.put_nowait(q)
+        self._event_ch.send_nowait(
+            stt.SpeechEvent(type=stt.SpeechEventType.START_OF_SPEECH)
+        )
+
+    def _on_end_of_speech(self) -> None:
+        if not self._speaking:
+            return
+        self._speaking = False
+        if self._utterance_q is not None:
+            self._utterance_q.put_nowait(None)
+        self._utterance_q = None
+        self._event_ch.send_nowait(
+            stt.SpeechEvent(type=stt.SpeechEventType.END_OF_SPEECH)
+        )
+
+    async def _handshake(self, ws: websockets.ClientConnection) -> str:
+        created = json.loads(await ws.recv())
+        if created.get("type") != "session.created":
+            raise APIStatusError(
+                f"expected session.created, got {created}",
+                status_code=500,
+                body=created,
+            )
+        session_update: dict = {"type": "session.update", "model": self._opts.model}
+        if self._opts.target_streaming_delay_ms is not None:
+            session_update["target_streaming_delay_ms"] = (
+                self._opts.target_streaming_delay_ms
+            )
+        await ws.send(json.dumps(session_update))
+        return created.get("id", "")
+
+    def _auth_headers(self) -> dict[str, str]:
+        if self._opts.api_key:
+            return {"Authorization": f"Bearer {self._opts.api_key}"}
+        return {}
+
+    async def _utterance_pipeline(self) -> None:
+        # Owns the websocket lifecycle. On drop, reopens and resumes the
+        # in-flight utterance (if any) by replaying its already-sent chunks.
+        pending: _PendingUtterance | None = None
+        attempt = 0
+        while True:
+            try:
+                async with websockets.connect(
+                    self._opts.base_url,
+                    additional_headers=self._auth_headers(),
+                    open_timeout=self._conn_options.timeout,
+                ) as ws:
+                    request_id = await self._handshake(ws)
+                    attempt = 0
+                    while True:
+                        if pending is None:
+                            q = await self._utterance_chan.get()
+                            if q is None:
+                                return
+                            pending = _PendingUtterance(queue=q)
+                        await self._process_utterance(ws, pending, request_id)
+                        pending = None
+            except (websockets.WebSocketException, OSError, TimeoutError) as exc:
+                attempt += 1
+                if attempt > RECONNECT_MAX_ATTEMPTS:
+                    logger.exception(
+                        "vLLM realtime: giving up after %d reconnect attempts",
+                        RECONNECT_MAX_ATTEMPTS,
+                    )
+                    raise APIConnectionError() from exc
+                backoff = min(
+                    RECONNECT_BACKOFF_BASE_S * (2 ** (attempt - 1)),
+                    RECONNECT_BACKOFF_MAX_S,
+                )
+                if pending is None:
+                    logger.warning(
+                        "vLLM WS connection lost between utterances "
+                        "(attempt %d/%d): %s; retrying in %.1fs",
+                        attempt,
+                        RECONNECT_MAX_ATTEMPTS,
+                        exc,
+                        backoff,
+                    )
+                else:
+                    logger.warning(
+                        "vLLM WS dropped mid-utterance (%d chunks buffered, "
+                        "ended=%s, attempt %d/%d): %s; retrying in %.1fs",
+                        len(pending.sent_chunks),
+                        pending.ended,
+                        attempt,
+                        RECONNECT_MAX_ATTEMPTS,
+                        exc,
+                        backoff,
+                    )
+                await asyncio.sleep(backoff)
+
+    async def _process_utterance(
+        self,
+        ws: websockets.ClientConnection,
+        pending: _PendingUtterance,
+        request_id: str,
+    ) -> None:
+        # Start a fresh generation. Safe to send here: the previous utterance's
+        # transcription.done has already been received (we await it below), so
+        # the server-side generation_task is done and won't ignore this commit.
+        await ws.send(json.dumps({"type": "input_audio_buffer.commit"}))
+        send_t = asyncio.create_task(self._send_audio(ws, pending))
+        try:
+            await self._receive_one_transcription(ws, request_id)
+        finally:
+            if not send_t.done():
+                send_t.cancel()
+            try:
+                await send_t
+            except (asyncio.CancelledError, websockets.WebSocketException):
+                pass
+            except Exception:
+                logger.exception("send-audio task failed during finalize")
+
+    @staticmethod
+    async def _send_audio(
+        ws: websockets.ClientConnection, pending: _PendingUtterance
+    ) -> None:
+        # Replay anything already sent on a previous (now-dead) connection.
+        # sent_chunks is appended before send, so a chunk that failed to send
+        # last time is still present and gets retried here.
+        for chunk in pending.sent_chunks:
+            await ws.send(
+                json.dumps(
+                    {
+                        "type": "input_audio_buffer.append",
+                        "audio": base64.b64encode(chunk).decode("ascii"),
+                    }
+                )
+            )
+        if pending.ended:
+            await ws.send(
+                json.dumps({"type": "input_audio_buffer.commit", "final": True})
+            )
+            return
+        while True:
+            chunk = await pending.queue.get()
+            if chunk is None:
+                pending.ended = True
+                await ws.send(
+                    json.dumps({"type": "input_audio_buffer.commit", "final": True})
+                )
+                return
+            pending.sent_chunks.append(chunk)
+            await ws.send(
+                json.dumps(
+                    {
+                        "type": "input_audio_buffer.append",
+                        "audio": base64.b64encode(chunk).decode("ascii"),
+                    }
+                )
+            )
+
+    async def _receive_one_transcription(
+        self, ws: websockets.ClientConnection, request_id: str
+    ) -> None:
+        # Use recv() rather than `async for`: the latter swallows
+        # ConnectionClosed on close-mid-iteration, which would let a dropped
+        # WS look like a clean "no transcription" return.
+        current_text = ""
+        while True:
+            raw = await ws.recv()
+            data = json.loads(raw)
+            event_type = data.get("type")
+
+            if event_type == "transcription.delta":
+                delta = data.get("delta", "")
+                if not delta:
+                    continue
+                current_text += delta
+                self._event_ch.send_nowait(
+                    stt.SpeechEvent(
+                        type=stt.SpeechEventType.INTERIM_TRANSCRIPT,
+                        request_id=request_id,
+                        alternatives=[stt.SpeechData(text=current_text, language="")],
+                    )
+                )
+            elif event_type == "transcription.done":
+                final_text = data.get("text") or current_text
+                self._event_ch.send_nowait(
+                    stt.SpeechEvent(
+                        type=stt.SpeechEventType.FINAL_TRANSCRIPT,
+                        request_id=request_id,
+                        alternatives=[stt.SpeechData(text=final_text, language="")],
+                    )
+                )
+                usage = data.get("usage") or {}
+                self._event_ch.send_nowait(
+                    stt.SpeechEvent(
+                        type=stt.SpeechEventType.RECOGNITION_USAGE,
+                        request_id=request_id,
+                        recognition_usage=stt.RecognitionUsage(
+                            audio_duration=float(
+                                usage.get("audio_seconds")
+                                or usage.get("prompt_audio_seconds")
+                                or 0
+                            ),
+                            input_tokens=int(usage.get("prompt_tokens") or 0),
+                            output_tokens=int(usage.get("completion_tokens") or 0),
+                        ),
+                    )
+                )
+                return
+            elif event_type == "error":
+                err = data.get("error")
+                raise APIStatusError(str(err), status_code=500, body=data)

src/frontend/src/features/subtitle/component/Subtitles.tsx

@@ -75,15 +75,15 @@ const useTranscriptionState = () => {
     const segment = segments[0]
 
     setTranscriptionSegments((prevSegments) => {
-      const existingSegmentIds = new Set(prevSegments.map((s) => s.id))
+      const existingIndex = prevSegments.findIndex(
-      if (existingSegmentIds.has(segment.id)) return prevSegments
+        (s: TranscriptionSegmentWithParticipant) => s.id === segment.id
-      return [
+      )
-        ...prevSegments,
+      if (existingIndex === -1) {
-        {
+        return [...prevSegments, { participant, ...segment }]
-          participant: participant,
+      }
-          ...segment,
+      const next = prevSegments.slice()
-        },
+      next[existingIndex] = { ...next[existingIndex], ...segment }
-      ]
+      return next
     })
   }