✨(audio) assign users to diarization speaker results using VAD

Introduce a new user assignment mechanism to for more friendly output than the current (SPEAKER_0, SPEAKER_1, ...). Use the VAD metadata to compare speech intervals with those returned by WhisperX. User with the highest overlap score above a defined threshold is assigned to each segment. This method allows for multi-speaker scenarios for a single account.
2026-07-26 11:58:53 +00:00 · 2026-05-07 11:26:59 +02:00
parent f8937fc0a1
commit 1612d8b2d4
9 changed files with 929 additions and 6 deletions
@@ -13,6 +13,7 @@ and this project adheres to
 - 🔒️(backend) add validation of Room.configuration
 - ✨(helm) add support multiple transcribe worker / endpoint #1247
 - ✨(backend) make LiveKit Egress recording encoding configurable #1288
+- ✨(summary) add speaker-to-participant assignment

 ### Changed

@@ -177,7 +177,7 @@ class MetadataCollector:

    def save(self):
        """Serialize collected events and upload as JSON to S3."""
-        logger.info("Persisting metadata…")
+        logger.info("Persisting metadata...")

        participants = []
        for k, v in self.participants.items():
@@ -372,7 +372,7 @@ async def entrypoint(ctx: JobContext):
    await ctx.connect(auto_subscribe=AutoSubscribe.AUDIO_ONLY)

    async def cleanup():
-        logger.info("Shutting down metadata collector…")
+        logger.info("Shutting down metadata collector...")
        await metadata_collector.aclose()

    ctx.add_shutdown_callback(cleanup)
@@ -363,6 +363,7 @@ class RoomViewSet(
        ):
            try:
                MetadataCollectorService().start(recording)
+                logger.debug("Started MetadataCollectorService")
            except MetadataCollectorException:
                logger.warning("Failed to start MetadataCollectorService")

@@ -213,7 +213,7 @@ class NotificationService:
        payload = {
            "owner_id": str(owner_access.user.id),
            "recording_filename": recording.key,
-            "metadata_filename": metadata_filename,  # For future use
+            "metadata_filename": metadata_filename,
            "email": owner_access.user.email,
            "sub": owner_access.user.sub,
            "room": recording.room.name,
@@ -222,9 +222,7 @@ class NotificationService:
            "download_link": f"{get_recording_download_base_url()}/{recording.id}",
            "context_language": owner_access.user.language,
            "recording_start_at": (started_at.isoformat() if started_at else None),
-            "recording_end_at": (
-                ended_at.isoformat() if ended_at else None
-            ),  # For future use
+            "recording_end_at": (ended_at.isoformat() if ended_at else None),
        }

        headers = {
@@ -4,6 +4,7 @@

 import json
 import time
+from datetime import datetime

 import openai
 import sentry_sdk
@@ -39,6 +40,7 @@ from summary.core.shared_models import (
    webhook_payload_adapter,
 )
 from summary.core.transcript_formatter import TranscriptFormatter
+from summary.core.user_assign import resolve_speaker_identities
 from summary.core.webhook_service import (
    call_webhook_v2,
    submit_content,
@@ -160,6 +162,65 @@ def transcribe_audio(
    return transcription


+def resolve_speaker_identities_and_apply_to(
+    transcription, recording_start_at, recording_end_at, metadata_filename, task_id
+):
+    """Assign users to detected speakers and rewrite the transcriptions.
+
+    Args:
+        transcription: output of meet-whisperx after transcription and diarization
+        recording_start_at: sourced from LiveKit FileInfo via the egress_ended webhook
+        recording_end_at: sourced from LiveKit FileInfo via the egress_ended webhook
+        metadata_filename: name of metadata file containing VAD information in S3
+        task_id: current task id, for logging purposes
+    """
+    recording_start_dt = (
+        datetime.fromisoformat(recording_start_at) if recording_start_at else None
+    )
+    recording_end_dt = (
+        datetime.fromisoformat(recording_end_at) if recording_end_at else None
+    )
+
+    logger.debug(
+        "recording_start_dt: %s ; recording_end_dt: %s",
+        recording_start_dt,
+        recording_end_dt,
+    )
+    if (recording_start_dt is None) or (recording_end_dt is None):
+        logger.debug("Skipping resolve_speaker_identities")
+        return transcription
+
+    logger.debug("Running resolve_speaker_identities")
+    try:
+        metadata = file_service.read_json(metadata_filename)
+        speaker_mapping = resolve_speaker_identities(
+            metadata,
+            transcription,
+            recording_start_dt,
+            recording_end_dt,
+        )
+        new_transcription = speaker_mapping.apply_to(transcription.model_dump())
+        return new_transcription
+
+    except FileServiceException as exc:
+        logger.error(
+            "Error reading metadata for task %s; skipping speaker assignment."
+            " Error: %s",
+            task_id,
+            exc,
+        )
+        return transcription
+
+    except Exception as exc:
+        logger.exception(
+            "resolve_speaker_identities failed for task %s; skipping"
+            " speaker assignment. Error: %s",
+            task_id,
+            exc,
+        )
+        return transcription
+
+
 def format_transcript(
    transcription,
    context_language: str | None,
@@ -269,6 +330,18 @@ def process_audio_transcribe_summarize_v2(
    if transcription is None:
        return

+    # Assign speakers and rewrite transcription/diarization output
+    if settings.is_resolve_speaker_identities_enabled and (
+        metadata_filename is not None
+    ):
+        transcription = resolve_speaker_identities_and_apply_to(
+            transcription,
+            recording_start_at,
+            recording_end_at,
+            metadata_filename,
+            task_id,
+        )
+
    # Format output
    content, title = format_transcript(
        transcription,
@@ -103,6 +103,9 @@ class Settings(BaseSettings):
    # Transcription processing
    hallucination_patterns: List[str] = ["Vap'n'Roll Thierry"]

+    # Speaker to user assignment
+    is_resolve_speaker_identities_enabled: bool = True
+
    # Webhook-related settings
    webhook_max_retries: int = 2
    webhook_status_forcelist: List[int] = [502, 503, 504]
@@ -229,6 +229,28 @@ class FileService:
                os.remove(output_path)
            raise RuntimeError("Failed to extract audio.") from e

+    def read_json(self, object_name: str) -> dict:
+        """Read and parse a JSON file from MinIO storage."""
+        logger.info("Reading JSON: %s", object_name)
+
+        if not object_name:
+            raise ValueError("Invalid object_name")
+
+        response = None
+        try:
+            response = self._minio_client.get_object(self._bucket_name, object_name)
+            return json.loads(response.read())
+        except (MinioException, S3Error) as e:
+            raise FileServiceException(
+                "Unexpected error while reading JSON object."
+            ) from e
+        except (json.JSONDecodeError, UnicodeDecodeError) as e:
+            raise FileServiceException("Invalid JSON content.") from e
+        finally:
+            if response:
+                response.close()
+                response.release_conn()
+
    @contextmanager
    def prepare_audio_file(
        self,
@@ -0,0 +1,297 @@
+"""Assign WhisperX diarization speakers to participant identities.
+
+Uses per-stream VAD events to match generic SPEAKER_XX labels provided
+by diarization to real user id's by computing time overlap between
+diarization segments and VAD intervals.
+
+Multiple speakers can map to the same participant (e.g. two people sharing
+one microphone). A participant with no matching speaker gets no assignment.
+"""
+
+import logging
+from collections import defaultdict
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+# Minimum fraction of a speaker's total duration that must overlap with a
+# participant's VAD to accept the assignment.
+DEFAULT_OVERLAP_THRESHOLD = 0.5
+
+
+@dataclass
+class Interval:
+    """A time interval in seconds relative to recording start."""
+
+    start: float
+    end: float
+
+
+@dataclass
+class SpeakerAssignment:
+    """Maps a diarization speaker label to a participant."""
+
+    speaker_label: str
+    participant_id: str
+    participant_name: str
+    score: float
+
+
+@dataclass
+class AssignmentResult:
+    """Result of speaker-to-participant assignment."""
+
+    assignments: list[SpeakerAssignment] = field(default_factory=list)
+    unassigned_speakers: list[str] = field(default_factory=list)
+
+    def apply_to(self, diarization: dict[str, Any]) -> dict[str, Any]:
+        """Return a copy of diarization with speaker labels replaced by names.
+
+        Replaces `"speaker"` fields in segments and word_segments with the
+        assigned participant name.  Unassigned speakers are left as-is.
+
+        Args:
+            diarization: WhisperX dict with `segments` and optionally
+                `word_segments`.
+
+        Returns:
+            New dict with speaker labels replaced.
+        """
+        speaker_to_name = {
+            a.speaker_label: a.participant_name for a in self.assignments
+        }
+
+        name_to_speaker_count = defaultdict(int)
+        for name in speaker_to_name.values():
+            name_to_speaker_count[name] += 1
+
+        def _replace_speaker(item: dict[str, Any]) -> dict[str, Any]:
+            if "speaker" in item and item["speaker"] in speaker_to_name:
+                name = speaker_to_name[item["speaker"]]
+                suffix = (
+                    f" ({item['speaker']})" if name_to_speaker_count[name] > 1 else ""
+                )  # Add suffix only if there are multiple detected speakers per user
+                return {**item, "speaker": f"{name}{suffix}"}
+            return item
+
+        def _process_segment(
+            item: dict[str, Any], include_words: bool = False
+        ) -> dict[str, Any]:
+            new_item = _replace_speaker(item)
+            if include_words and "words" in item:
+                new_item["words"] = [_replace_speaker(w) for w in item["words"]]
+            return new_item
+
+        result: dict[str, Any] = {}
+        for key, value in diarization.items():
+            if key not in ("segments", "word_segments"):
+                result[key] = value
+                continue
+            result[key] = [
+                _process_segment(item, include_words=(key == "segments"))
+                for item in value
+            ]
+        return result
+
+
+def _merge_intervals(intervals: list[Interval]) -> list[Interval]:
+    """Return a list of non-overlapping intervals sorted by start time."""
+    if not intervals:
+        return []
+    sorted_intervals = sorted(intervals, key=lambda interval: interval.start)
+    merged: list[Interval] = [
+        Interval(sorted_intervals[0].start, sorted_intervals[0].end)
+    ]
+    for interval in sorted_intervals[1:]:
+        if interval.start <= merged[-1].end:
+            merged[-1].end = max(merged[-1].end, interval.end)
+        else:
+            merged.append(Interval(interval.start, interval.end))
+    return merged
+
+
+def _total_duration(intervals: list[Interval]) -> float:
+    """Return the sum of all interval durations."""
+    return sum(interval.end - interval.start for interval in intervals)
+
+
+def _overlap_duration(
+    a_intervals: list[Interval],
+    b_intervals: list[Interval],
+) -> float:
+    """Compute total overlap between two merged interval lists, sorted by start time."""
+    overlap = 0.0
+    i = j = 0
+    while i < len(a_intervals) and j < len(b_intervals):
+        a = a_intervals[i]
+        b = b_intervals[j]
+        lo = max(a.start, b.start)
+        hi = min(a.end, b.end)
+        if lo < hi:
+            overlap += hi - lo
+        if a.end <= b.end:
+            i += 1
+        else:
+            j += 1
+    return overlap
+
+
+def _build_participant_timelines(
+    metadata: dict[str, Any],
+    recording_start_datetime: datetime,
+    recording_end_datetime: datetime | None = None,
+) -> tuple[dict[str, list[Interval]], dict[str, str]]:
+    """Build VAD interval timelines for each participant.
+
+    Args:
+        metadata: Dict with `events` and `participants` keys.
+        recording_start_datetime: UTC datetime used as t=0 reference.
+        recording_end_datetime: UTC datetime of recording end. When provided,
+            any open speech_start without a matching speech_end is closed at
+            this time (the participant is assumed to be speaking until the end).
+
+    Returns:
+        participant_id → merged VAD intervals
+            (seconds relative to recording_start_datetime).
+        participant_id → display name.
+        Intervals are in seconds relative to recording_start_datetime.
+        Events before recording start are clamped to 0.
+    """
+    events = metadata.get("events", [])
+    participants_info = {
+        p["participantId"]: p.get("name", p["participantId"])
+        for p in metadata.get("participants", [])
+    }
+
+    ref_epoch = recording_start_datetime.timestamp()
+
+    open_starts: dict[str, float] = {}
+    intervals: dict[str, list[Interval]] = {}
+
+    for event in events:
+        pid = event["participant_id"]
+        ts = datetime.fromisoformat(event["timestamp"]).timestamp() - ref_epoch
+        etype = event["type"]
+
+        if etype == "speech_start":
+            open_starts[pid] = max(ts, 0.0)
+        elif etype == "speech_end":
+            start = open_starts.pop(pid, None)
+            if start is not None:
+                end = max(ts, 0.0)
+                if end > start:
+                    intervals.setdefault(pid, []).append(Interval(start, end))
+
+    # Close any speech_start that was never matched by a speech_end.
+    # Assume the participant kept speaking until the recording ended.
+    if recording_end_datetime is not None and open_starts:
+        recording_end = recording_end_datetime.timestamp() - ref_epoch
+        for pid, start in open_starts.items():
+            end = max(recording_end, 0.0)
+            if end > start:
+                intervals.setdefault(pid, []).append(Interval(start, end))
+
+    for pid, pid_intervals in intervals.items():
+        intervals[pid] = _merge_intervals(pid_intervals)
+
+    return intervals, participants_info
+
+
+def _build_speaker_timelines(
+    transcription: Any,
+) -> dict[str, list[Interval]]:
+    """Build interval timelines from WhisperX transcription segments."""
+    intervals: dict[str, list[Interval]] = {}
+
+    segments = transcription.segments if hasattr(transcription, "segments") else []
+    for segment in segments:
+        speaker = segment.get("speaker")
+        if speaker is None:
+            continue
+        intervals.setdefault(speaker, []).append(
+            Interval(segment["start"], segment["end"])
+        )
+
+    for speaker, speaker_intervals in intervals.items():
+        intervals[speaker] = _merge_intervals(speaker_intervals)
+
+    return intervals
+
+
+def resolve_speaker_identities(
+    metadata: dict[str, Any],
+    transcription: Any,
+    recording_start_datetime: datetime,
+    recording_end_datetime: datetime,
+    overlap_threshold: float = DEFAULT_OVERLAP_THRESHOLD,
+) -> AssignmentResult:
+    """Match WhisperX speaker labels to participants.
+
+    Args:
+        metadata: User metadata with `events` and `participants`.
+        transcription: WhisperX Transcription object with a `segments` attribute.
+        recording_start_datetime: UTC datetime for t=0 reference.
+        recording_end_datetime: UTC datetime of recording end. Open speech
+            intervals are closed at this time.
+        overlap_threshold: Minimum overlap/speaker_duration to accept.
+
+    Returns:
+        AssignmentResult with per-speaker assignments and unassigned
+        speakers.
+    """
+    participant_timelines, participant_names = _build_participant_timelines(
+        metadata, recording_start_datetime, recording_end_datetime
+    )
+    speaker_timelines = _build_speaker_timelines(transcription)
+
+    logger.debug(
+        "Assignment inputs: %d participants, %d speakers",
+        len(participant_timelines),
+        len(speaker_timelines),
+    )
+
+    result = AssignmentResult()
+
+    for speaker, speaker_intervals in speaker_timelines.items():
+        speaker_duration = _total_duration(speaker_intervals)
+        if speaker_duration == 0:
+            result.unassigned_speakers.append(speaker)
+            continue
+
+        best_pid: str | None = None
+        best_score: float = 0.0
+
+        for pid, part_intervals in participant_timelines.items():
+            overlap = _overlap_duration(speaker_intervals, part_intervals)
+            score = overlap / speaker_duration
+            if score > best_score:
+                best_score = score
+                best_pid = pid
+
+        if best_pid is not None and best_score >= overlap_threshold:
+            result.assignments.append(
+                SpeakerAssignment(
+                    speaker_label=speaker,
+                    participant_id=best_pid,
+                    participant_name=participant_names.get(best_pid, best_pid),
+                    score=best_score,
+                )
+            )
+            logger.info(
+                "Assigned %s -> %s (score=%.3f)",
+                speaker,
+                participant_names.get(best_pid, best_pid),
+                best_score,
+            )
+        else:
+            result.unassigned_speakers.append(speaker)
+            logger.info(
+                "Speaker %s unassigned (best=%.3f, threshold=%.3f)",
+                speaker,
+                best_score,
+                overlap_threshold,
+            )
+
+    return result
@@ -0,0 +1,528 @@
+"""Tests for the speaker-to-user assignment service."""
+
+import math
+from dataclasses import dataclass, field
+from datetime import datetime
+
+from summary.core.user_assign import (
+    AssignmentResult,
+    Interval,
+    SpeakerAssignment,
+    _merge_intervals,
+    _overlap_duration,
+    _total_duration,
+    resolve_speaker_identities,
+)
+
+
+@dataclass
+class FakeTranscription:
+    """Mimics the OpenAI Transcription pydantic model for testing."""
+
+    segments: list = field(default_factory=list)
+
+
+RECORDING_START = datetime.fromisoformat("2026-03-17T15:30:33.000001")
+RECORDING_END = datetime.fromisoformat("2026-03-17T15:31:33.000001")
+
+METADATA_SINGLE_USER = {
+    "events": [
+        {
+            "participant_id": "da8d39ff-3b1c-4e8d-9a70-c630c9871bcb",
+            "type": "participant_connected",
+            "timestamp": "2026-03-17T15:30:33.000001",
+        },
+        {
+            "participant_id": "da8d39ff-3b1c-4e8d-9a70-c630c9871bcb",
+            "type": "speech_start",
+            "timestamp": "2026-03-17T15:30:36.039456",
+        },
+        {
+            "participant_id": "da8d39ff-3b1c-4e8d-9a70-c630c9871bcb",
+            "type": "speech_end",
+            "timestamp": "2026-03-17T15:30:36.589114",
+        },
+        {
+            "participant_id": "da8d39ff-3b1c-4e8d-9a70-c630c9871bcb",
+            "type": "speech_start",
+            "timestamp": "2026-03-17T15:30:38.887518",
+        },
+        {
+            "participant_id": "da8d39ff-3b1c-4e8d-9a70-c630c9871bcb",
+            "type": "speech_end",
+            "timestamp": "2026-03-17T15:30:39.438141",
+        },
+        {
+            "participant_id": "da8d39ff-3b1c-4e8d-9a70-c630c9871bcb",
+            "type": "participant_disconnected",
+            "timestamp": "2026-03-17T15:30:43.223255",
+        },
+    ],
+    "participants": [
+        {
+            "participantId": "da8d39ff-3b1c-4e8d-9a70-c630c9871bcb",
+            "name": "cameledev",
+        }
+    ],
+}
+
+DIARIZATION_SINGLE_SPEAKER = FakeTranscription(
+    segments=[
+        {
+            "start": 1.363,
+            "end": 3.545,
+            "text": " The stale smell.",
+            "speaker": "SPEAKER_00",
+        },
+        {
+            "start": 4.466,
+            "end": 6.247,
+            "text": "It takes heat.",
+            "speaker": "SPEAKER_00",
+        },
+    ],
+)
+
+USER_ID = "da8d39ff-3b1c-4e8d-9a70-c630c9871bcb"
+
+
+class TestMergeIntervals:
+    """Tests for _merge_intervals."""
+
+    def test_empty(self):
+        """Empty input returns empty list."""
+        assert _merge_intervals([]) == []
+
+    def test_no_overlap(self):
+        """Non-overlapping intervals stay separate."""
+        result = _merge_intervals([Interval(1, 2), Interval(3, 4)])
+        assert len(result) == 2
+
+    def test_overlap(self):
+        """Overlapping intervals are merged."""
+        result = _merge_intervals([Interval(1, 3), Interval(2, 4)])
+        assert len(result) == 1
+        assert result[0].start == 1
+        assert result[0].end == 4
+
+    def test_adjacent(self):
+        """Adjacent intervals are merged."""
+        result = _merge_intervals([Interval(1, 2), Interval(2, 3)])
+        assert len(result) == 1
+        assert result[0].start == 1
+        assert result[0].end == 3
+
+    def test_unsorted(self):
+        """Unsorted input is sorted before merging."""
+        result = _merge_intervals([Interval(5, 6), Interval(1, 2)])
+        assert len(result) == 2
+        assert result[0].start == 1
+
+
+class TestOverlapDuration:
+    """Tests for _overlap_duration."""
+
+    def test_no_overlap(self):
+        """Disjoint intervals have zero overlap."""
+        a = [Interval(1, 2)]
+        b = [Interval(3, 4)]
+        assert math.isclose(_overlap_duration(a, b), 0.0)
+
+    def test_full_overlap(self):
+        """Identical intervals have full overlap."""
+        a = [Interval(1, 3)]
+        b = [Interval(1, 3)]
+        assert math.isclose(_overlap_duration(a, b), 2.0)
+
+    def test_partial_overlap(self):
+        """Partially overlapping intervals."""
+        a = [Interval(1, 3)]
+        b = [Interval(2, 4)]
+        assert math.isclose(_overlap_duration(a, b), 1.0)
+
+    def test_multiple_intervals(self):
+        """Multiple intervals with a spanning interval."""
+        a = [Interval(1, 3), Interval(5, 7)]
+        b = [Interval(2, 6)]
+        assert math.isclose(_overlap_duration(a, b), 2.0)
+
+    def test_empty(self):
+        """Empty input yields zero overlap."""
+        assert math.isclose(_overlap_duration([], [Interval(1, 2)]), 0.0)
+        assert math.isclose(_overlap_duration([Interval(1, 2)], []), 0.0)
+
+
+class TestTotalDuration:
+    """Tests for _total_duration."""
+
+    def test_basic(self):
+        """Sum of durations for multiple intervals."""
+        ivs = [Interval(0, 1), Interval(2, 5)]
+        assert math.isclose(_total_duration(ivs), 4.0)
+
+    def test_empty(self):
+        """Empty input returns zero."""
+        assert math.isclose(_total_duration([]), 0.0)
+
+
+class TestResolveSpeakerIdentities:
+    """Tests for resolve_speaker_identities."""
+
+    def test_single_speaker_single_user(self):
+        """Single speaker assigned to single user with low threshold."""
+        result = resolve_speaker_identities(
+            METADATA_SINGLE_USER,
+            DIARIZATION_SINGLE_SPEAKER,
+            RECORDING_START,
+            RECORDING_END,
+            overlap_threshold=0.2,
+        )
+        assert len(result.assignments) == 1
+        assert result.assignments[0].speaker_label == "SPEAKER_00"
+        assert result.assignments[0].participant_id == USER_ID
+        assert result.assignments[0].participant_name == "cameledev"
+        assert result.assignments[0].score > 0
+        assert result.unassigned_speakers == []
+
+    def test_no_vad_events(self):
+        """Participant with no speech leaves speakers unassigned."""
+        metadata = {
+            "events": [
+                {
+                    "participant_id": "user-a",
+                    "type": "participant_connected",
+                    "timestamp": "2026-03-17T15:30:33.000000",
+                },
+            ],
+            "participants": [{"participantId": "user-a", "name": "Silent User"}],
+        }
+        result = resolve_speaker_identities(
+            metadata,
+            DIARIZATION_SINGLE_SPEAKER,
+            RECORDING_START,
+            RECORDING_END,
+        )
+        assert len(result.assignments) == 0
+        assert "SPEAKER_00" in result.unassigned_speakers
+
+    def test_multiple_speakers_same_user(self):
+        """Two speakers from same mic both assigned to same user."""
+        metadata = {
+            "events": [
+                {
+                    "participant_id": "user-a",
+                    "type": "speech_start",
+                    "timestamp": "2026-03-17T15:30:35.000000",
+                },
+                {
+                    "participant_id": "user-a",
+                    "type": "speech_end",
+                    "timestamp": "2026-03-17T15:30:50.000000",
+                },
+            ],
+            "participants": [{"participantId": "user-a", "name": "Shared Mic"}],
+        }
+        transcription = FakeTranscription(
+            segments=[
+                {"start": 1.0, "end": 3.0, "speaker": "SPEAKER_00"},
+                {"start": 5.0, "end": 7.0, "speaker": "SPEAKER_01"},
+            ],
+        )
+        result = resolve_speaker_identities(
+            metadata, transcription, RECORDING_START, RECORDING_END
+        )
+        assert len(result.assignments) == 2
+        pids = {a.participant_id for a in result.assignments}
+        assert pids == {"user-a"}
+
+    def test_two_users_two_speakers(self):
+        """Each speaker maps to correct user by VAD overlap."""
+        metadata = {
+            "events": [
+                {
+                    "participant_id": "user-a",
+                    "type": "speech_start",
+                    "timestamp": "2026-03-17T15:30:34.000001",
+                },
+                {
+                    "participant_id": "user-a",
+                    "type": "speech_end",
+                    "timestamp": "2026-03-17T15:30:37.000001",
+                },
+                {
+                    "participant_id": "user-b",
+                    "type": "speech_start",
+                    "timestamp": "2026-03-17T15:30:38.000001",
+                },
+                {
+                    "participant_id": "user-b",
+                    "type": "speech_end",
+                    "timestamp": "2026-03-17T15:30:41.000001",
+                },
+            ],
+            "participants": [
+                {"participantId": "user-a", "name": "Alice"},
+                {"participantId": "user-b", "name": "Bob"},
+            ],
+        }
+        transcription = FakeTranscription(
+            segments=[
+                {"start": 1.5, "end": 3.5, "speaker": "SPEAKER_00"},
+                {"start": 5.5, "end": 7.5, "speaker": "SPEAKER_01"},
+            ],
+        )
+        result = resolve_speaker_identities(
+            metadata, transcription, RECORDING_START, RECORDING_END
+        )
+        assert len(result.assignments) == 2
+        by_speaker = {a.speaker_label: a for a in result.assignments}
+        assert by_speaker["SPEAKER_00"].participant_name == "Alice"
+        assert by_speaker["SPEAKER_01"].participant_name == "Bob"
+
+    def test_overlapping_speech_two_users(self):
+        """Simultaneous speech from two users still assigns each speaker correctly."""
+        # user-a speaks from t=1s to t=6s, user-b speaks from t=3s to t=8s
+        # (3s overlap where both are speaking)
+        # SPEAKER_00 diarization covers t=1.5–5.5 (mostly user-a)
+        # SPEAKER_01 diarization covers t=4.0–7.5 (mostly user-b)
+        metadata = {
+            "events": [
+                {
+                    "participant_id": "user-a",
+                    "type": "speech_start",
+                    "timestamp": "2026-03-17T15:30:34.000001",
+                },
+                {
+                    "participant_id": "user-b",
+                    "type": "speech_start",
+                    "timestamp": "2026-03-17T15:30:36.000001",
+                },
+                {
+                    "participant_id": "user-a",
+                    "type": "speech_end",
+                    "timestamp": "2026-03-17T15:30:39.000001",
+                },
+                {
+                    "participant_id": "user-b",
+                    "type": "speech_end",
+                    "timestamp": "2026-03-17T15:30:41.000001",
+                },
+            ],
+            "participants": [
+                {"participantId": "user-a", "name": "Alice"},
+                {"participantId": "user-b", "name": "Bob"},
+            ],
+        }
+        transcription = FakeTranscription(
+            segments=[
+                {"start": 1.5, "end": 5.5, "speaker": "SPEAKER_00"},
+                {"start": 4.0, "end": 7.5, "speaker": "SPEAKER_01"},
+            ],
+        )
+        result = resolve_speaker_identities(
+            metadata,
+            transcription,
+            RECORDING_START,
+            RECORDING_END,
+            overlap_threshold=0.3,
+        )
+        assert len(result.assignments) == 2
+        by_speaker = {a.speaker_label: a for a in result.assignments}
+        assert by_speaker["SPEAKER_00"].participant_name == "Alice"
+        assert by_speaker["SPEAKER_01"].participant_name == "Bob"
+        assert result.unassigned_speakers == []
+
+    def test_below_threshold(self):
+        """Speaker with minimal overlap stays unassigned."""
+        metadata = {
+            "events": [
+                {
+                    "participant_id": "user-a",
+                    "type": "speech_start",
+                    "timestamp": "2026-03-17T15:30:34.000001",
+                },
+                {
+                    "participant_id": "user-a",
+                    "type": "speech_end",
+                    "timestamp": "2026-03-17T15:30:35.169950",
+                },
+            ],
+            "participants": [{"participantId": "user-a", "name": "Brief User"}],
+        }
+        transcription = FakeTranscription(
+            segments=[
+                {"start": 1.0, "end": 10.0, "speaker": "SPEAKER_00"},
+            ],
+        )
+        result = resolve_speaker_identities(
+            metadata,
+            transcription,
+            RECORDING_START,
+            RECORDING_END,
+            overlap_threshold=0.5,
+        )
+        assert len(result.assignments) == 0
+        assert "SPEAKER_00" in result.unassigned_speakers
+
+    def test_events_before_recording_start_clamped(self):
+        """Speech events before recording start are clamped to t=0."""
+        metadata = {
+            "events": [
+                {
+                    "participant_id": "user-a",
+                    "type": "speech_start",
+                    "timestamp": "2026-03-17T15:30:31.000001",  # before RECORDING_START
+                },
+                {
+                    "participant_id": "user-a",
+                    "type": "speech_end",
+                    "timestamp": "2026-03-17T15:30:36.000001",  # after RECORDING_START
+                },
+            ],
+            "participants": [{"participantId": "user-a", "name": "Early User"}],
+        }
+        transcription = FakeTranscription(
+            segments=[
+                {"start": 0.0, "end": 3.0, "speaker": "SPEAKER_00"},
+            ],
+        )
+        result = resolve_speaker_identities(
+            metadata, transcription, RECORDING_START, RECORDING_END
+        )
+        assert len(result.assignments) == 1
+        assert result.assignments[0].participant_name == "Early User"
+
+    def test_empty_diarization(self):
+        """No segments produces empty result."""
+        result = resolve_speaker_identities(
+            METADATA_SINGLE_USER,
+            FakeTranscription(segments=[]),
+            RECORDING_START,
+            RECORDING_END,
+        )
+        assert result == AssignmentResult()
+
+    def test_segment_without_speaker_ignored(self):
+        """Segments missing speaker key are skipped."""
+        transcription = FakeTranscription(
+            segments=[
+                {"start": 1.0, "end": 3.0, "text": "no speaker"},
+            ],
+        )
+        result = resolve_speaker_identities(
+            METADATA_SINGLE_USER, transcription, RECORDING_START, RECORDING_END
+        )
+        assert result == AssignmentResult()
+
+    def test_unclosed_speech_closed_at_recording_end(self):
+        """Open speech_start without speech_end is closed at recording end."""
+        recording_end = datetime.fromisoformat("2026-03-17T15:30:43.000001")
+        metadata = {
+            "events": [
+                {
+                    "participant_id": "user-a",
+                    "type": "speech_start",
+                    "timestamp": "2026-03-17T15:30:35.000001",
+                },
+                # No speech_end — participant kept speaking until recording stopped
+            ],
+            "participants": [{"participantId": "user-a", "name": "Still Talking"}],
+        }
+        transcription = FakeTranscription(
+            segments=[
+                {"start": 2.0, "end": 9.0, "speaker": "SPEAKER_00"},
+            ],
+        )
+        result = resolve_speaker_identities(
+            metadata,
+            transcription,
+            RECORDING_START,
+            recording_end,
+            overlap_threshold=0.5,
+        )
+        assert len(result.assignments) == 1
+        assert result.assignments[0].participant_name == "Still Talking"
+        assert result.unassigned_speakers == []
+
+
+class TestApply:
+    """Tests for AssignmentResult.apply_to."""
+
+    def test_replaces_speakers_in_segments_and_words(self):
+        """Speaker labels replaced in segments, words, and word_segments."""
+        diarization = {
+            "segments": [
+                {
+                    "start": 1.0,
+                    "end": 3.0,
+                    "text": "Hello",
+                    "speaker": "SPEAKER_00",
+                    "words": [
+                        {
+                            "word": "Hello",
+                            "start": 1.0,
+                            "end": 1.5,
+                            "speaker": "SPEAKER_00",
+                        },
+                    ],
+                },
+                {
+                    "start": 4.0,
+                    "end": 6.0,
+                    "text": "World",
+                    "speaker": "SPEAKER_01",
+                    "words": [
+                        {
+                            "word": "World",
+                            "start": 4.0,
+                            "end": 4.5,
+                            "speaker": "SPEAKER_01",
+                        },
+                    ],
+                },
+            ],
+            "word_segments": [
+                {"word": "Hello", "start": 1.0, "end": 1.5, "speaker": "SPEAKER_00"},
+                {"word": "World", "start": 4.0, "end": 4.5, "speaker": "SPEAKER_01"},
+            ],
+        }
+        assignment = AssignmentResult(
+            assignments=[
+                SpeakerAssignment("SPEAKER_00", "id-a", "Alice", 0.9),
+                SpeakerAssignment("SPEAKER_01", "id-b", "Bob", 0.8),
+            ],
+        )
+        result = assignment.apply_to(diarization)
+
+        assert result["segments"][0]["speaker"] == "Alice"
+        assert result["segments"][0]["words"][0]["speaker"] == "Alice"
+        assert result["segments"][1]["speaker"] == "Bob"
+        assert result["word_segments"][0]["speaker"] == "Alice"
+        assert result["word_segments"][1]["speaker"] == "Bob"
+
+    def test_unassigned_speakers_unchanged(self):
+        """Unassigned speaker labels are left as-is."""
+        diarization = {
+            "segments": [
+                {"start": 1.0, "end": 3.0, "speaker": "SPEAKER_02"},
+            ],
+        }
+        assignment = AssignmentResult(
+            assignments=[
+                SpeakerAssignment("SPEAKER_00", "id-a", "Alice", 0.9),
+            ],
+            unassigned_speakers=["SPEAKER_02"],
+        )
+        result = assignment.apply_to(diarization)
+        assert result["segments"][0]["speaker"] == "SPEAKER_02"
+
+    def test_preserves_extra_keys(self):
+        """Non-segment keys in diarization are preserved."""
+        diarization = {
+            "segments": [],
+            "language": "en",
+            "custom_field": 42,
+        }
+        result = AssignmentResult().apply_to(diarization)
+        assert result["language"] == "en"
+        assert result["custom_field"] == 42