modify .dist

compose.yml

@@ -335,7 +335,8 @@ services:
   multi-user-transcriber:
     build:
       context: ./src/agents
-      target: development
+      target: development    
+    command: ["python", "multi_user_transcriber.py", "dev"]
     env_file:
       - env.d/development/multi_user_transcriber
     volumes:

env.d/development/multi_user_transcriber.dist

@@ -1,9 +1,21 @@
+AWS_S3_ENDPOINT_URL=minio:9000
+AWS_S3_ACCESS_KEY_ID=meet
+AWS_S3_SECRET_ACCESS_KEY=password
+
 LIVEKIT_URL=ws://livekit:7880
 LIVEKIT_API_KEY=devkey
 LIVEKIT_API_SECRET=secret
 
-STT_PROVIDER=kyutai
+STT_PROVIDER=voxtral-vllm # voxtral-vllm, kyutai, deepgram
 ENABLE_SILERO_VAD=False
 
-KYUTAI_STT_BASE_URL=
-KYUTAI_API_KEY=
+DEEPGRAM_API_KEY=your-deepgram-api-key
+
+KYUTAI_STT_BASE_URL=url
+KYUTAI_API_KEY=your-kyutai-api-key
+
+VOXTRAL_VLLM_BASE_URL=wss://<host>/v1/realtime
+VOXTRAL_VLLM_MODEL=voxtral-mini-4b-realtime-2602
+VOXTRAL_VLLM_API_KEY=your-vllm-api-key
+VOXTRAL_VLLM_TARGET_STREAMING_DELAY_MS=480
+

src/agents/multi_user_transcriber.py

@@ -25,6 +25,8 @@ from livekit.agents import (
 )
 from livekit.plugins import deepgram, silero
 
+import voxtral_vllm_stt
+
 load_dotenv()
 
 logger = logging.getLogger("transcriber")
@@ -46,6 +48,9 @@ def create_stt_provider():
         )
     elif STT_PROVIDER == "kyutai":
         _stt_instance = kyutai.STT(base_url=os.getenv("KYUTAI_STT_BASE_URL"))
+    elif STT_PROVIDER == "voxtral-vllm":
+        # The plugin resolves base_url / model / api_key from the environment.
+        _stt_instance = voxtral_vllm_stt.STT()
     else:
         raise ValueError(f"Unknown STT_PROVIDER: {STT_PROVIDER}")
 

src/agents/uv.lock

@@ -28,7 +28,7 @@ dev = [
 
 [package.metadata]
 requires-dist = [
-    { name = "livekit-agents", specifier = "==1.4.5" },
+    { name = "livekit-agents", specifier = ">=1.4.5" },
     { name = "livekit-plugins-deepgram", specifier = "==1.4.5" },
     { name = "livekit-plugins-kyutai-lasuite", specifier = "==0.0.6" },
     { name = "livekit-plugins-silero", specifier = "==1.4.5" },
@@ -731,7 +731,7 @@ wheels = [
 
 [[package]]
 name = "livekit"
-version = "1.1.2"
+version = "1.1.7"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "aiofiles" },
@@ -739,18 +739,18 @@ dependencies = [
     { name = "protobuf" },
     { name = "types-protobuf" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/9c/9c/0a9b9e1f88226df372b9ccc58868ce9a1eb97c8bf5257463a643dda2a6da/livekit-1.1.2.tar.gz", hash = "sha256:ef826e94dd039767fcabc2f0d810b1b2335c9cf249f52320b6ab018b06d5ccd7", size = 320006, upload-time = "2026-02-17T01:18:46.828Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/e1/1e/3ad724abacb6514fc64c7b4a8c82a41780cbc82baee621b8dcc92dded005/livekit-1.1.7.tar.gz", hash = "sha256:29e5f13e2639c041f18e29d1066019a6380031391b6363509f06e3dbef8426a6", size = 332704, upload-time = "2026-04-27T13:49:48.432Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/07/7b/1b2d448d8976b14794bfba3d003e1451c79afa77e6b9fa1593f19175ef90/livekit-1.1.2-py3-none-macosx_10_15_x86_64.whl", hash = "sha256:78be23f3f6315354aaacee664eb19b793009bc06faa8184ad9c07cffbe8d7f74", size = 9844157, upload-time = "2026-02-17T01:18:34.036Z" },
-    { url = "https://files.pythonhosted.org/packages/f9/fb/19f7fc4a7df3b1385ba2e32b907c5a502fe01c10e6ee2fb76629c068667d/livekit-1.1.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:656f4d9e4692f3d7ab2e51b0bbf4ec03b356a487b7ff220576dab496e60f99f3", size = 8651703, upload-time = "2026-02-17T01:18:36.416Z" },
-    { url = "https://files.pythonhosted.org/packages/29/59/ac6a3987bfd11687f86156627a8c7f6a0047a72877748facbd427e63b157/livekit-1.1.2-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:a20e681d8a4929e27df69f818888ae649700c9d52a262abbec296c84937bc337", size = 14085891, upload-time = "2026-02-17T01:18:38.561Z" },
-    { url = "https://files.pythonhosted.org/packages/0a/a5/680548ef7bf5034144ae01f1f742a6c49e4428942b3119ac6553207fea9b/livekit-1.1.2-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:0e4d8105a0c317b513a118b48340b5773239103cb6305768451ef99ac7567823", size = 11448902, upload-time = "2026-02-17T01:18:42.006Z" },
-    { url = "https://files.pythonhosted.org/packages/00/05/e484f8fc079c5de7690e58ed48f44e77436b8732c3050da742bdfca51a5c/livekit-1.1.2-py3-none-win_amd64.whl", hash = "sha256:dd4a436fa16de589353bfbabde91068ab64241afd05b04f21fb1f22bfe155dc0", size = 10394562, upload-time = "2026-02-17T01:18:44.589Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/f8/176fe3d5569ed03b2c60991f4a90edfd2421a5ad0c4a9a70974d409b3329/livekit-1.1.7-py3-none-macosx_10_15_x86_64.whl", hash = "sha256:26df0760e9b9d3eeb0d773614bc344b6aae7d37c60a9b0c93b1c58dcb6bb2f6b", size = 10127354, upload-time = "2026-04-27T13:49:35.805Z" },
+    { url = "https://files.pythonhosted.org/packages/29/f2/05bd6f3729ecb564ddc9052afc43c3b9e39a43d24f7821946bc78a5dddd2/livekit-1.1.7-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e49e79d39b2c74104c377b6e3cb789d47dd92cb993327234966c41f19df28292", size = 8954208, upload-time = "2026-04-27T13:49:38.359Z" },
+    { url = "https://files.pythonhosted.org/packages/58/32/99f52a9c71dcc3b3370783ad427e5974b9491948b6d9beb5b3f933df8f1e/livekit-1.1.7-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:c3db851f510edac945b9bc402f6629d5969b4f3252e8d0b24dd158d9e9a5485d", size = 9962579, upload-time = "2026-04-27T13:49:40.28Z" },
+    { url = "https://files.pythonhosted.org/packages/bf/9e/53768dd73f1ce2d3eff7d6c1d87c3531f795c57880a3b51eb5cee1aec6fa/livekit-1.1.7-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:6c227e2f840aa547224e2fbd45f953c8a0b30939a18e1d56deca8c4268c95804", size = 11355003, upload-time = "2026-04-27T13:49:42.881Z" },
+    { url = "https://files.pythonhosted.org/packages/42/55/2c68732dbcdb3fc1a58592db27697c6d27e2f77ade7eaeb818e88637020b/livekit-1.1.7-py3-none-win_amd64.whl", hash = "sha256:3b0bcc48430cb2138eade01f6b59ddd3be1c1e90f6c251291d0dd4dc1623fc0d", size = 10696832, upload-time = "2026-04-27T13:49:46.362Z" },
 ]
 
 [[package]]
 name = "livekit-agents"
-version = "1.4.5"
+version = "1.5.8"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "aiofiles" },
@@ -782,9 +782,9 @@ dependencies = [
     { name = "typing-extensions" },
     { name = "watchfiles" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/9c/06/1a23902ffea2742e60e30f957f3a96766be125892a2573ea1754051b3d8d/livekit_agents-1.4.5.tar.gz", hash = "sha256:19fdf45a2c4d02d86b4ad132b73bf3f58eadb75f79819e5015e6fadfb1263852", size = 2416100, upload-time = "2026-03-11T06:45:03.912Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/bb/23/8af550199aef211304290596c1ce073de9ac22d6b2beea51ca362d70b2eb/livekit_agents-1.5.8.tar.gz", hash = "sha256:ab8951fb7a4f79f81c13f962a401a76c9ecbcb79c9ebd5cad5f0554062b70b6d", size = 2494457, upload-time = "2026-05-05T18:46:42.965Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/71/ad/a7613ddd5cfb29e015913ae44b804505d41c1d02d19ffeaf2f7015d23664/livekit_agents-1.4.5-py3-none-any.whl", hash = "sha256:aff0b61dbd24b1a100cccca8e2a0056f414b861ce71ba8e4338bc5f89e149f7a", size = 2499369, upload-time = "2026-03-11T06:45:01.772Z" },
+    { url = "https://files.pythonhosted.org/packages/37/1e/3f5e2580f9ffdb5267a7c9be003f8f711ea58c4e0764a74999c800ed3d0e/livekit_agents-1.5.8-py3-none-any.whl", hash = "sha256:737a2d517cf2d6c1d22221d720ed12fcd47a21019abb4b5b20533407b57bb9ea", size = 2594846, upload-time = "2026-05-05T18:46:40.156Z" },
 ]
 
 [package.optional-dependencies]
@@ -878,15 +878,15 @@ wheels = [
 
 [[package]]
 name = "livekit-protocol"
-version = "1.1.6"
+version = "1.1.8"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "protobuf" },
     { name = "types-protobuf" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/5b/28/8a7ee5a4126476a0ce770a482274d51fc81ae0d9f578c07f07d6655acfc4/livekit_protocol-1.1.6.tar.gz", hash = "sha256:43648863440a6ce064f7f8b8b287dda8f5ac82b8da64738b20f42cad51e70f9b", size = 93925, upload-time = "2026-04-20T15:48:43.032Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/6b/17/761372e1db01925bda77c679ffde19a9d128a4ba53d864dc50a0a9f93ce8/livekit_protocol-1.1.8.tar.gz", hash = "sha256:b3850c09847d54aaa538c87a90ffb5cf267123a0725f2b03874e39e88fc59ccb", size = 98588, upload-time = "2026-05-05T09:12:33.171Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/76/36/525cd9364036617aa97c597c93d7bddb6be311b86288cd54de77e4892544/livekit_protocol-1.1.6-py3-none-any.whl", hash = "sha256:286ac0bd555b6b75cd8a0c5d417572d7dab524129fcea43080202d170f957710", size = 115012, upload-time = "2026-04-20T15:48:41.312Z" },
+    { url = "https://files.pythonhosted.org/packages/6d/54/fca19cd45d0c74bb5ef88b0257cf594ec5549192a15957dd22596acdec32/livekit_protocol-1.1.8-py3-none-any.whl", hash = "sha256:dc379b3e713dc8d8642f694c72d9a38b41220f05b2ce8fc6662f445d20953c17", size = 121119, upload-time = "2026-05-05T09:12:31.787Z" },
 ]
 
 [[package]]

src/agents/voxtral_vllm_stt.py

@@ -0,0 +1,474 @@
+"""LiveKit STT plugin for Voxtral Realtime served via vLLM (/v1/realtime).
+
+vLLM exposes Voxtral Realtime over a WebSocket that follows the OpenAI Realtime
+API protocol (not Mistral's proprietary realtime protocol).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import json
+import logging
+import os
+import weakref
+from collections import deque
+from dataclasses import dataclass, field
+
+import websockets
+from livekit.agents import (
+    DEFAULT_API_CONNECT_OPTIONS,
+    APIConnectionError,
+    APIConnectOptions,
+    APIStatusError,
+    stt,
+    utils,
+)
+from livekit.agents import (
+    vad as vad_module,
+)
+from livekit.agents.types import NOT_GIVEN, NotGivenOr
+from livekit.agents.utils import is_given
+
+logger = logging.getLogger("voxtral-vllm-stt")
+
+SAMPLE_RATE = 16000
+NUM_CHANNELS = 1
+CHUNK_SAMPLES = 1600  # 100 ms @ 16 kHz mono
+PREROLL_CHUNKS = 5  # keep 500 ms of audio before start of speech as detected by VAD
+
+# Reconnect policy: exponential backoff capped at MAX, give up after MAX_ATTEMPTS
+# consecutive failures (a successful handshake resets the counter).
+RECONNECT_BACKOFF_BASE_S = 0.5
+RECONNECT_BACKOFF_MAX_S = 8.0
+RECONNECT_MAX_ATTEMPTS = 5
+
+
+@dataclass
+class _STTOptions:
+    base_url: str
+    model: str
+    api_key: str | None
+    target_streaming_delay_ms: int | None
+
+
+@dataclass
+class _PendingUtterance:
+    """An utterance in flight on the shared websocket used for reconnect.
+
+    `sent_chunks` holds every chunk we have already enqueued for send on this
+    or a prior connection; on reconnect we replay them before resuming reads
+    from `queue`. vLLM concatenates `input_audio_buffer.append` events into a
+    single audio buffer per generation, so duplicates from a partial prior send
+    are harmless.
+    """
+
+    queue: asyncio.Queue[bytes | None]
+    sent_chunks: list[bytes] = field(default_factory=list)
+    ended: bool = False
+
+
+class STT(stt.STT):
+    """LiveKit STT speaking the OpenAI Realtime protocol served by vLLM."""
+
+    def __init__(
+        self,
+        *,
+        base_url: NotGivenOr[str] = NOT_GIVEN,
+        model: NotGivenOr[str] = NOT_GIVEN,
+        api_key: NotGivenOr[str] = NOT_GIVEN,
+        target_streaming_delay_ms: NotGivenOr[int] = NOT_GIVEN,
+        vad: vad_module.VAD | None = None,
+    ) -> None:
+        """Build the STT.
+
+        Args:
+            base_url: WebSocket URL of the vLLM realtime endpoint, e.g.
+                ws://example:8000/v1/realtime. Falls back to $VOXTRAL_VLLM_BASE_URL.
+            model: Model name exposed by vLLM, default
+                mistralai/Voxtral-Mini-4B-Realtime-2602.
+            api_key: Optional bearer token. Falls back to $VOXTRAL_VLLM_API_KEY.
+            target_streaming_delay_ms: Target streaming delay in ms forwarded to
+                vLLM via session.update. Falls back to
+                $VOXTRAL_VLLM_TARGET_STREAMING_DELAY_MS, else server default.
+            vad: Voice Activity Detector. If omitted, Silero VAD is loaded.
+        """
+        super().__init__(
+            capabilities=stt.STTCapabilities(streaming=True, interim_results=True)
+        )
+
+        resolved_url = (
+            base_url
+            if is_given(base_url)
+            else os.environ.get(
+                "VOXTRAL_VLLM_BASE_URL", "ws://127.0.0.1:8000/v1/realtime"
+            )
+        )
+        resolved_model = (
+            model
+            if is_given(model)
+            else os.environ.get(
+                "VOXTRAL_VLLM_MODEL", "mistralai/Voxtral-Mini-4B-Realtime-2602"
+            )
+        )
+        resolved_key = (
+            api_key if is_given(api_key) else os.environ.get("VOXTRAL_VLLM_API_KEY")
+        )
+        resolved_delay = (
+            target_streaming_delay_ms
+            if is_given(target_streaming_delay_ms)
+            else (
+                int(os.environ["VOXTRAL_VLLM_TARGET_STREAMING_DELAY_MS"])
+                if os.environ.get("VOXTRAL_VLLM_TARGET_STREAMING_DELAY_MS")
+                else None
+            )
+        )
+
+        if vad is None:
+            try:
+                from livekit.plugins.silero import VAD as SileroVAD  # noqa: PLC0415
+            except ImportError as exc:
+                raise ImportError(
+                    "livekit-plugins-silero is required for vLLM Voxtral realtime "
+                    "(no server-side endpointing)."
+                ) from exc
+            vad = SileroVAD.load()
+        self._vad = vad
+
+        self._opts = _STTOptions(
+            base_url=resolved_url,
+            model=resolved_model,
+            api_key=resolved_key,
+            target_streaming_delay_ms=resolved_delay,
+        )
+        self._streams: weakref.WeakSet[SpeechStream] = weakref.WeakSet()
+
+    @property
+    def model(self) -> str:
+        """Return the configured vLLM model name."""
+        return self._opts.model
+
+    @property
+    def provider(self) -> str:
+        """Return the provider identifier."""
+        return "vllm-voxtral-realtime"
+
+    async def _recognize_impl(self, *_args, **_kwargs) -> stt.SpeechEvent:
+        raise NotImplementedError(
+            "vLLM Voxtral Realtime STT only supports streaming recognition."
+        )
+
+    def stream(
+        self,
+        *,
+        conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
+    ) -> SpeechStream:
+        """Open a new streaming recognition stream."""
+        s = SpeechStream(
+            stt=self,
+            opts=self._opts,
+            vad_instance=self._vad,
+            conn_options=conn_options,
+        )
+        self._streams.add(s)
+        return s
+
+
+class SpeechStream(stt.RecognizeStream):
+    def __init__(
+        self,
+        *,
+        stt: STT,
+        opts: _STTOptions,
+        vad_instance: vad_module.VAD,
+        conn_options: APIConnectOptions,
+    ) -> None:
+        """Init the speech stream."""
+        super().__init__(stt=stt, conn_options=conn_options, sample_rate=SAMPLE_RATE)
+        self._opts = opts
+        self._vad = vad_instance
+        self._utterance_q: asyncio.Queue[bytes | None] | None = None
+        self._speaking = False
+        self._preroll: deque[bytes] = deque(maxlen=PREROLL_CHUNKS)
+        # Voxtral realtime is strictly sequential: only one generation runs at a
+        # time, and a new `commit` is ignored while the previous one is still
+        # producing. We queue per-utterance audio buffers here and let the
+        # pipeline process them one by one on the shared websocket.
+        self._utterance_chan: asyncio.Queue[asyncio.Queue[bytes | None] | None] = (
+            asyncio.Queue()
+        )
+
+    @utils.log_exceptions(logger=logger)
+    async def _run(self) -> None:
+        vad_stream = self._vad.stream()
+
+        bstream = utils.audio.AudioByteStream(
+            sample_rate=SAMPLE_RATE,
+            num_channels=NUM_CHANNELS,
+            samples_per_channel=CHUNK_SAMPLES,
+        )
+
+        async def input_task() -> None:
+            async for data in self._input_ch:
+                if isinstance(data, self._FlushSentinel):
+                    for frame in bstream.flush():
+                        self._handle_chunk(frame.data.tobytes())
+                    continue
+
+                vad_stream.push_frame(data)
+                for frame in bstream.write(data.data.tobytes()):
+                    self._handle_chunk(frame.data.tobytes())
+
+            vad_stream.end_input()
+
+        async def vad_task() -> None:
+            async for ev in vad_stream:
+                if ev.type == vad_module.VADEventType.START_OF_SPEECH:
+                    self._on_start_of_speech()
+                elif ev.type == vad_module.VADEventType.END_OF_SPEECH:
+                    self._on_end_of_speech()
+
+        pipeline_t = asyncio.create_task(self._utterance_pipeline())
+        try:
+            await asyncio.gather(input_task(), vad_task())
+            # signal end-of-stream; pipeline finishes pending utterances first
+            self._utterance_chan.put_nowait(None)
+            await pipeline_t
+        except (APIStatusError, APIConnectionError, asyncio.CancelledError):
+            raise
+        except Exception as exc:
+            logger.exception("vLLM realtime stream failed")
+            raise APIConnectionError() from exc
+        finally:
+            if not pipeline_t.done():
+                pipeline_t.cancel()
+            try:
+                await pipeline_t
+            except asyncio.CancelledError:
+                # CancelledError is the expected flow on cancel()
+                pass
+            except Exception:
+                logger.exception("utterance pipeline failed during finalize")
+            await vad_stream.aclose()
+
+    def _handle_chunk(self, chunk: bytes) -> None:
+        self._preroll.append(chunk)
+        if self._speaking and self._utterance_q is not None:
+            self._utterance_q.put_nowait(chunk)
+
+    def _on_start_of_speech(self) -> None:
+        if self._speaking:
+            return
+        self._speaking = True
+        q: asyncio.Queue[bytes | None] = asyncio.Queue()
+        for chunk in self._preroll:
+            q.put_nowait(chunk)
+        self._utterance_q = q
+        self._utterance_chan.put_nowait(q)
+        self._event_ch.send_nowait(
+            stt.SpeechEvent(type=stt.SpeechEventType.START_OF_SPEECH)
+        )
+
+    def _on_end_of_speech(self) -> None:
+        if not self._speaking:
+            return
+        self._speaking = False
+        if self._utterance_q is not None:
+            self._utterance_q.put_nowait(None)
+        self._utterance_q = None
+        self._event_ch.send_nowait(
+            stt.SpeechEvent(type=stt.SpeechEventType.END_OF_SPEECH)
+        )
+
+    async def _handshake(self, ws: websockets.ClientConnection) -> str:
+        created = json.loads(await ws.recv())
+        if created.get("type") != "session.created":
+            raise APIStatusError(
+                f"expected session.created, got {created}",
+                status_code=500,
+                body=created,
+            )
+        session_update: dict = {"type": "session.update", "model": self._opts.model}
+        if self._opts.target_streaming_delay_ms is not None:
+            session_update["target_streaming_delay_ms"] = (
+                self._opts.target_streaming_delay_ms
+            )
+        await ws.send(json.dumps(session_update))
+        return created.get("id", "")
+
+    def _auth_headers(self) -> dict[str, str]:
+        if self._opts.api_key:
+            return {"Authorization": f"Bearer {self._opts.api_key}"}
+        return {}
+
+    async def _utterance_pipeline(self) -> None:
+        # Owns the websocket lifecycle. On drop, reopens and resumes the
+        # in-flight utterance (if any) by replaying its already-sent chunks.
+        pending: _PendingUtterance | None = None
+        attempt = 0
+        while True:
+            try:
+                async with websockets.connect(
+                    self._opts.base_url,
+                    additional_headers=self._auth_headers(),
+                    open_timeout=self._conn_options.timeout,
+                ) as ws:
+                    request_id = await self._handshake(ws)
+                    attempt = 0
+                    while True:
+                        if pending is None:
+                            q = await self._utterance_chan.get()
+                            if q is None:
+                                return
+                            pending = _PendingUtterance(queue=q)
+                        await self._process_utterance(ws, pending, request_id)
+                        pending = None
+            except (websockets.WebSocketException, OSError, TimeoutError) as exc:
+                attempt += 1
+                if attempt > RECONNECT_MAX_ATTEMPTS:
+                    logger.exception(
+                        "vLLM realtime: giving up after %d reconnect attempts",
+                        RECONNECT_MAX_ATTEMPTS,
+                    )
+                    raise APIConnectionError() from exc
+                backoff = min(
+                    RECONNECT_BACKOFF_BASE_S * (2 ** (attempt - 1)),
+                    RECONNECT_BACKOFF_MAX_S,
+                )
+                if pending is None:
+                    logger.warning(
+                        "vLLM WS connection lost between utterances "
+                        "(attempt %d/%d): %s; retrying in %.1fs",
+                        attempt,
+                        RECONNECT_MAX_ATTEMPTS,
+                        exc,
+                        backoff,
+                    )
+                else:
+                    logger.warning(
+                        "vLLM WS dropped mid-utterance (%d chunks buffered, "
+                        "ended=%s, attempt %d/%d): %s; retrying in %.1fs",
+                        len(pending.sent_chunks),
+                        pending.ended,
+                        attempt,
+                        RECONNECT_MAX_ATTEMPTS,
+                        exc,
+                        backoff,
+                    )
+                await asyncio.sleep(backoff)
+
+    async def _process_utterance(
+        self,
+        ws: websockets.ClientConnection,
+        pending: _PendingUtterance,
+        request_id: str,
+    ) -> None:
+        # Start a fresh generation. Safe to send here: the previous utterance's
+        # transcription.done has already been received (we await it below), so
+        # the server-side generation_task is done and won't ignore this commit.
+        await ws.send(json.dumps({"type": "input_audio_buffer.commit"}))
+        send_t = asyncio.create_task(self._send_audio(ws, pending))
+        try:
+            await self._receive_one_transcription(ws, request_id)
+        finally:
+            if not send_t.done():
+                send_t.cancel()
+            try:
+                await send_t
+            except (asyncio.CancelledError, websockets.WebSocketException):
+                pass
+            except Exception:
+                logger.exception("send-audio task failed during finalize")
+
+    @staticmethod
+    async def _send_audio(
+        ws: websockets.ClientConnection, pending: _PendingUtterance
+    ) -> None:
+        # Replay anything already sent on a previous (now-dead) connection.
+        # sent_chunks is appended before send, so a chunk that failed to send
+        # last time is still present and gets retried here.
+        for chunk in pending.sent_chunks:
+            await ws.send(
+                json.dumps(
+                    {
+                        "type": "input_audio_buffer.append",
+                        "audio": base64.b64encode(chunk).decode("ascii"),
+                    }
+                )
+            )
+        if pending.ended:
+            await ws.send(
+                json.dumps({"type": "input_audio_buffer.commit", "final": True})
+            )
+            return
+        while True:
+            chunk = await pending.queue.get()
+            if chunk is None:
+                pending.ended = True
+                await ws.send(
+                    json.dumps({"type": "input_audio_buffer.commit", "final": True})
+                )
+                return
+            pending.sent_chunks.append(chunk)
+            await ws.send(
+                json.dumps(
+                    {
+                        "type": "input_audio_buffer.append",
+                        "audio": base64.b64encode(chunk).decode("ascii"),
+                    }
+                )
+            )
+
+    async def _receive_one_transcription(
+        self, ws: websockets.ClientConnection, request_id: str
+    ) -> None:
+        # Use recv() rather than `async for`: the latter swallows
+        # ConnectionClosed on close-mid-iteration, which would let a dropped
+        # WS look like a clean "no transcription" return.
+        current_text = ""
+        while True:
+            raw = await ws.recv()
+            data = json.loads(raw)
+            event_type = data.get("type")
+
+            if event_type == "transcription.delta":
+                delta = data.get("delta", "")
+                if not delta:
+                    continue
+                current_text += delta
+                self._event_ch.send_nowait(
+                    stt.SpeechEvent(
+                        type=stt.SpeechEventType.INTERIM_TRANSCRIPT,
+                        request_id=request_id,
+                        alternatives=[stt.SpeechData(text=current_text, language="")],
+                    )
+                )
+            elif event_type == "transcription.done":
+                final_text = data.get("text") or current_text
+                self._event_ch.send_nowait(
+                    stt.SpeechEvent(
+                        type=stt.SpeechEventType.FINAL_TRANSCRIPT,
+                        request_id=request_id,
+                        alternatives=[stt.SpeechData(text=final_text, language="")],
+                    )
+                )
+                usage = data.get("usage") or {}
+                self._event_ch.send_nowait(
+                    stt.SpeechEvent(
+                        type=stt.SpeechEventType.RECOGNITION_USAGE,
+                        request_id=request_id,
+                        recognition_usage=stt.RecognitionUsage(
+                            audio_duration=float(
+                                usage.get("audio_seconds")
+                                or usage.get("prompt_audio_seconds")
+                                or 0
+                            ),
+                            input_tokens=int(usage.get("prompt_tokens") or 0),
+                            output_tokens=int(usage.get("completion_tokens") or 0),
+                        ),
+                    )
+                )
+                return
+            elif event_type == "error":
+                err = data.get("error")
+                raise APIStatusError(str(err), status_code=500, body=data)

src/frontend/src/features/subtitle/component/Subtitles.tsx

@@ -75,15 +75,15 @@ const useTranscriptionState = () => {
     const segment = segments[0]
 
     setTranscriptionSegments((prevSegments) => {
-      const existingSegmentIds = new Set(prevSegments.map((s) => s.id))
-      if (existingSegmentIds.has(segment.id)) return prevSegments
-      return [
-        ...prevSegments,
-        {
-          participant: participant,
-          ...segment,
-        },
-      ]
+      const existingIndex = prevSegments.findIndex(
+        (s: TranscriptionSegmentWithParticipant) => s.id === segment.id
+      )
+      if (existingIndex === -1) {
+        return [...prevSegments, { participant, ...segment }]
+      }
+      const next = prevSegments.slice()
+      next[existingIndex] = { ...next[existingIndex], ...segment }
+      return next
     })
   }