From 96f97ed2d0510a158d2c444cdfd0ec328ac8a74f Mon Sep 17 00:00:00 2001
From: leo <260626284+cameledev@users.noreply.github.com>
Date: Tue, 12 May 2026 16:11:46 +0200
Subject: [PATCH] =?UTF-8?q?=E2=9C=A8(summary)=20improve=20speaker=20assign?=
 =?UTF-8?q?ment?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Speaker-to-participant assignment relie on WhisperX word timings, but
incorrect word durations in the output can lead to inaccurate overlap
scoring and wrong user attribution. Add a custom heuristic to trim
overly long word durations before computing assignments.
---
 CHANGELOG.md                               |   1 +
 src/summary/summary/core/config.py         |   3 +
 src/summary/summary/core/user_assign.py    | 133 +++++++++++++--
 src/summary/tests/unit/test_user_assign.py | 180 +++++++++++++++++++++
 4 files changed, 302 insertions(+), 15 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ec98e532..89b58ddc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -23,6 +23,7 @@ and this project adheres to
 ### Changed
 
 - 🧑‍💻(agents) use `uv` for package management
+- ✨(summary) improve speaker-to-participant assignment
 
 ### Fixed
 
diff --git a/src/summary/summary/core/config.py b/src/summary/summary/core/config.py
index d5099f1f..6858a49d 100644
--- a/src/summary/summary/core/config.py
+++ b/src/summary/summary/core/config.py
@@ -105,6 +105,9 @@ class Settings(BaseSettings):
 
     # Speaker to user assignment
     is_resolve_speaker_identities_enabled: bool = True
+    resolve_speaker_identities_default_overlap_threshold: float = 0.5
+    resolve_speaker_identities_enable_split_on_words: bool = True
+    resolve_speaker_identities_max_word_duration: float = 1  # seconds
 
     # Webhook-related settings
     webhook_max_retries: int = 2
diff --git a/src/summary/summary/core/user_assign.py b/src/summary/summary/core/user_assign.py
index 6aab94e8..84873b8d 100644
--- a/src/summary/summary/core/user_assign.py
+++ b/src/summary/summary/core/user_assign.py
@@ -14,11 +14,11 @@ from dataclasses import dataclass, field
 from datetime import datetime
 from typing import Any
 
-logger = logging.getLogger(__name__)
+from summary.core.config import get_settings
 
-# Minimum fraction of a speaker's total duration that must overlap with a
-# participant's VAD to accept the assignment.
-DEFAULT_OVERLAP_THRESHOLD = 0.5
+settings = get_settings()
+
+logger = logging.getLogger(__name__)
 
 
 @dataclass
@@ -74,7 +74,7 @@ class AssignmentResult:
                     f" ({item['speaker']})" if name_to_speaker_count[name] > 1 else ""
                 )  # Add suffix only if there are multiple detected speakers per user
                 return {**item, "speaker": f"{name}{suffix}"}
-            return item
+            return {**item}
 
         def _process_segment(
             item: dict[str, Any], include_words: bool = False
@@ -138,6 +138,78 @@ def _overlap_duration(
     return overlap
 
 
+def _format_timelines_debug(
+    participant_timelines: dict[str, list[Interval]],
+    participant_names: dict[str, str],
+    speaker_timelines: dict[str, list[Interval]],
+) -> str:
+    """Render participant and speaker timelines side-by-side for debugging.
+
+    Each row is the slice between two consecutive interval boundaries
+    (drawn from both sides). A filled cell marks an active participant
+    (left block) or speaker (right block) during that slice, so vertical
+    alignment makes overlap visually obvious.
+    """
+    participant_ids = sorted(participant_timelines.keys())
+    speaker_labels = sorted(speaker_timelines.keys())
+
+    if not participant_ids and not speaker_labels:
+        return "(no timelines)"
+
+    boundaries: set[float] = set()
+    for intervals in (*participant_timelines.values(), *speaker_timelines.values()):
+        for iv in intervals:
+            boundaries.add(iv.start)
+            boundaries.add(iv.end)
+    sorted_boundaries = sorted(boundaries)
+    if len(sorted_boundaries) < 2:
+        return "(no intervals)"
+
+    p_headers = [participant_names.get(pid, pid) for pid in participant_ids]
+    s_headers = list(speaker_labels)
+    p_widths = [max(len(h), 3) for h in p_headers]
+    s_widths = [max(len(h), 3) for h in s_headers]
+
+    def _active(intervals: list[Interval], lo: float, hi: float) -> bool:
+        mid = (lo + hi) / 2
+        return any(iv.start <= mid < iv.end for iv in intervals)
+
+    def _cells(
+        intervals_list: list[list[Interval]],
+        widths: list[int],
+        lo: float,
+        hi: float,
+    ) -> str:
+        return " ".join(
+            ("█" * w if _active(iv, lo, hi) else "·" * w)
+            for iv, w in zip(intervals_list, widths, strict=True)
+        )
+
+    p_iv_list = [participant_timelines[pid] for pid in participant_ids]
+    s_iv_list = [speaker_timelines[sl] for sl in speaker_labels]
+
+    time_col = "[   start →      end]"
+    p_hdr = (
+        " ".join(h.center(w) for h, w in zip(p_headers, p_widths, strict=True))
+        or "(none)"
+    )
+    s_hdr = (
+        " ".join(h.center(w) for h, w in zip(s_headers, s_widths, strict=True))
+        or "(none)"
+    )
+    sep = "  ||  "
+    lines = [
+        f"{time_col}  {p_hdr}{sep}{s_hdr}",
+        "-" * (len(time_col) + 2 + len(p_hdr) + len(sep) + len(s_hdr)),
+    ]
+    for lo, hi in zip(sorted_boundaries, sorted_boundaries[1:], strict=False):
+        time_str = f"[{lo:8.2f} → {hi:8.2f}]"
+        p_row = _cells(p_iv_list, p_widths, lo, hi) or " " * len(p_hdr)
+        s_row = _cells(s_iv_list, s_widths, lo, hi) or " " * len(s_hdr)
+        lines.append(f"{time_str}  {p_row}{sep}{s_row}")
+    return "\n".join(lines)
+
+
 def _build_participant_timelines(
     metadata: dict[str, Any],
     recording_start_datetime: datetime,
@@ -199,24 +271,50 @@ def _build_participant_timelines(
     return intervals, participants_info
 
 
-def _build_speaker_timelines(
-    transcription: Any,
-) -> dict[str, list[Interval]]:
+def _build_speaker_timelines(transcription: Any) -> dict[str, list[Interval]]:
     """Build interval timelines from WhisperX transcription segments."""
     intervals: dict[str, list[Interval]] = {}
-
     segments = transcription.segments if hasattr(transcription, "segments") else []
+    max_word_duration = settings.resolve_speaker_identities_max_word_duration
+
     for segment in segments:
         speaker = segment.get("speaker")
         if speaker is None:
             continue
-        intervals.setdefault(speaker, []).append(
-            Interval(segment["start"], segment["end"])
-        )
+
+        words = [
+            w
+            for w in segment.get("words", [])
+            if w.get("start") is not None and w.get("end") is not None
+        ]
+        if not words:
+            intervals.setdefault(speaker, []).append(
+                Interval(segment["start"], segment["end"])
+            )
+            continue
+
+        start_time: float | None = segment["start"]
+        for word in words:
+            if start_time is None:
+                start_time = word["start"]
+            if not settings.resolve_speaker_identities_enable_split_on_words:
+                continue
+            if word["end"] - word["start"] > max_word_duration:
+                end_time = word["start"] + max_word_duration
+                if end_time > start_time:
+                    intervals.setdefault(speaker, []).append(
+                        Interval(start_time, end_time)
+                    )
+                start_time = None
+
+        if start_time is not None:
+            last = words[-1]
+            end_time = min(last["end"], last["start"] + max_word_duration)
+            if end_time > start_time:
+                intervals.setdefault(speaker, []).append(Interval(start_time, end_time))
 
     for speaker, speaker_intervals in intervals.items():
         intervals[speaker] = _merge_intervals(speaker_intervals)
-
     return intervals
 
 
@@ -225,7 +323,7 @@ def resolve_speaker_identities(
     transcription: Any,
     recording_start_datetime: datetime,
     recording_end_datetime: datetime,
-    overlap_threshold: float = DEFAULT_OVERLAP_THRESHOLD,
+    overlap_threshold: float = settings.resolve_speaker_identities_default_overlap_threshold,  # noqa: E501
 ) -> AssignmentResult:
     """Match WhisperX speaker labels to participants.
 
@@ -247,9 +345,14 @@ def resolve_speaker_identities(
     speaker_timelines = _build_speaker_timelines(transcription)
 
     logger.debug(
-        "Assignment inputs: %d participants, %d speakers",
+        "Assignment inputs: %d participants, %d speakers\n%s\n%s\n%s",
         len(participant_timelines),
         len(speaker_timelines),
+        participant_timelines,
+        speaker_timelines,
+        _format_timelines_debug(
+            participant_timelines, participant_names, speaker_timelines
+        ),
     )
 
     result = AssignmentResult()
diff --git a/src/summary/tests/unit/test_user_assign.py b/src/summary/tests/unit/test_user_assign.py
index 9fd36c84..20d54838 100644
--- a/src/summary/tests/unit/test_user_assign.py
+++ b/src/summary/tests/unit/test_user_assign.py
@@ -4,10 +4,12 @@ import math
 from dataclasses import dataclass, field
 from datetime import datetime
 
+from summary.core import user_assign
 from summary.core.user_assign import (
     AssignmentResult,
     Interval,
     SpeakerAssignment,
+    _build_speaker_timelines,
     _merge_intervals,
     _overlap_duration,
     _total_duration,
@@ -73,12 +75,22 @@ DIARIZATION_SINGLE_SPEAKER = FakeTranscription(
             "end": 3.545,
             "text": " The stale smell.",
             "speaker": "SPEAKER_00",
+            "words": [
+                {"word": "The", "start": 1.363, "end": 1.8},
+                {"word": "stale", "start": 1.8, "end": 2.7},
+                {"word": "smell.", "start": 2.7, "end": 3.545},
+            ],
         },
         {
             "start": 4.466,
             "end": 6.247,
             "text": "It takes heat.",
             "speaker": "SPEAKER_00",
+            "words": [
+                {"word": "It", "start": 4.466, "end": 4.7},
+                {"word": "takes", "start": 4.7, "end": 5.5},
+                {"word": "heat.", "start": 5.5, "end": 6.247},
+            ],
         },
     ],
 )
@@ -165,6 +177,174 @@ class TestTotalDuration:
         assert math.isclose(_total_duration([]), 0.0)
 
 
+class TestBuildSpeakerTimelines:
+    """Tests for _build_speaker_timelines."""
+
+    def test_segment_without_words_falls_back_to_segment_bounds(self):
+        """Segments missing a `words` key use the segment start/end as one interval."""
+        transcription = FakeTranscription(
+            segments=[{"start": 1.5, "end": 3.5, "speaker": "SPEAKER_00"}],
+        )
+        result = _build_speaker_timelines(transcription)
+        assert result == {"SPEAKER_00": [Interval(1.5, 3.5)]}
+
+    def test_segment_with_only_none_word_timestamps_falls_back(self):
+        """If every word has None start/end, fall back to segment bounds."""
+        transcription = FakeTranscription(
+            segments=[
+                {
+                    "start": 1.0,
+                    "end": 4.0,
+                    "speaker": "SPEAKER_00",
+                    "words": [
+                        {"word": "hi", "start": None, "end": None},
+                        {"word": "there", "start": None, "end": None},
+                    ],
+                },
+            ],
+        )
+        result = _build_speaker_timelines(transcription)
+        assert result == {"SPEAKER_00": [Interval(1.0, 4.0)]}
+
+    def test_short_words_only_uses_segment_start_and_last_word_end(self):
+        """With no overly long words, the interval runs segment start to end."""
+        transcription = FakeTranscription(
+            segments=[
+                {
+                    "start": 1.0,
+                    "end": 5.0,
+                    "speaker": "SPEAKER_00",
+                    "words": [
+                        {"word": "a", "start": 1.0, "end": 1.3},
+                        {"word": "b", "start": 1.4, "end": 1.7},
+                        {"word": "c", "start": 1.8, "end": 2.1},
+                    ],
+                },
+            ],
+        )
+        result = _build_speaker_timelines(transcription)
+        # Tail: min(2.1, 1.8 + 1.0) = 2.1
+        assert result == {"SPEAKER_00": [Interval(1.0, 2.1)]}
+
+    def test_long_word_caps_interval_at_max_duration(self):
+        """A word longer than the max-word-duration cap truncates the segment."""
+        max_word_duration = (
+            user_assign.settings.resolve_speaker_identities_max_word_duration
+        )
+        transcription = FakeTranscription(
+            segments=[
+                {
+                    "start": 0.0,
+                    "end": max_word_duration + 7,
+                    "speaker": "SPEAKER_00",
+                    "words": [
+                        {
+                            "word": "pause",
+                            "start": 0.0,
+                            "end": max_word_duration + 7,
+                        },
+                    ],
+                },
+            ],
+        )
+        result = _build_speaker_timelines(transcription)
+        assert result == {"SPEAKER_00": [Interval(0.0, max_word_duration)]}
+
+    def test_long_word_in_middle_splits_segment(self):
+        """Short words around a long word produce two intervals (before-cap + after)."""
+        transcription = FakeTranscription(
+            segments=[
+                {
+                    "start": 0.0,
+                    "end": 20.0,
+                    "speaker": "SPEAKER_00",
+                    "words": [
+                        {"word": "a", "start": 0.0, "end": 0.5},
+                        {"word": "long", "start": 1.0, "end": 15.0},
+                        {"word": "z", "start": 18.0, "end": 18.4},
+                    ],
+                },
+            ],
+        )
+        result = _build_speaker_timelines(transcription)
+        # First emit: (0.0, 1.0 + 1.0). Then start_time resets, picks up at "z" (18.0).
+        # Tail: min(18.4, 18.0 + 1.0) = 18.4. So second interval is (18.0, 18.4).
+        assert result == {
+            "SPEAKER_00": [Interval(0.0, 2.0), Interval(18.0, 18.4)],
+        }
+
+    def test_tail_word_is_capped_at_max_duration(self):
+        """The trailing word's end is capped at word.start + max_word_duration."""
+        transcription = FakeTranscription(
+            segments=[
+                {
+                    "start": 0.0,
+                    "end": 50.0,
+                    "speaker": "SPEAKER_00",
+                    "words": [
+                        {"word": "a", "start": 0.0, "end": 0.4},
+                        # Last word ends inside the cap, so the cap doesn't apply.
+                        {"word": "b", "start": 1.0, "end": 1.5},
+                    ],
+                },
+            ],
+        )
+        result = _build_speaker_timelines(transcription)
+        # Tail: min(1.5, 1.0 + 1.0) = 1.5
+        assert result == {"SPEAKER_00": [Interval(0.0, 1.5)]}
+
+    def test_split_on_words_disabled_keeps_segment_as_one_interval(self, monkeypatch):
+        """With splitting disabled, long words don't split the interval."""
+        monkeypatch.setattr(
+            user_assign,
+            "settings",
+            user_assign.settings.model_copy(
+                update={"resolve_speaker_identities_enable_split_on_words": False},
+            ),
+        )
+        transcription = FakeTranscription(
+            segments=[
+                {
+                    "start": 0.0,
+                    "end": 20.0,
+                    "speaker": "SPEAKER_00",
+                    "words": [
+                        {"word": "a", "start": 0.0, "end": 0.5},
+                        {"word": "long", "start": 1.0, "end": 15.0},
+                        {"word": "z", "start": 18.0, "end": 18.4},
+                    ],
+                },
+            ],
+        )
+        result = _build_speaker_timelines(transcription)
+        # No mid-segment split; tail caps at min(18.4, 18.0 + 1.0) = 18.4.
+        assert result == {"SPEAKER_00": [Interval(0.0, 18.4)]}
+
+    def test_multiple_speakers_keep_separate_timelines(self):
+        """Segments from different speakers populate independent timeline entries."""
+        transcription = FakeTranscription(
+            segments=[
+                {
+                    "start": 0.0,
+                    "end": 1.0,
+                    "speaker": "SPEAKER_00",
+                    "words": [{"word": "hi", "start": 0.0, "end": 0.5}],
+                },
+                {
+                    "start": 2.0,
+                    "end": 3.0,
+                    "speaker": "SPEAKER_01",
+                    "words": [{"word": "yo", "start": 2.0, "end": 2.5}],
+                },
+            ],
+        )
+        result = _build_speaker_timelines(transcription)
+        assert result == {
+            "SPEAKER_00": [Interval(0.0, 0.5)],
+            "SPEAKER_01": [Interval(2.0, 2.5)],
+        }
+
+
 class TestResolveSpeakerIdentities:
     """Tests for resolve_speaker_identities."""