mirror of
https://github.com/suitenumerique/meet.git
synced 2026-07-26 11:58:53 +00:00
✨(summary) improve speaker assignment
Speaker-to-participant assignment relie on WhisperX word timings, but incorrect word durations in the output can lead to inaccurate overlap scoring and wrong user attribution. Add a custom heuristic to trim overly long word durations before computing assignments.
This commit is contained in:
leo
@@ -23,6 +23,7 @@ and this project adheres to
|
||||
### Changed
|
||||
|
||||
- 🧑💻(agents) use `uv` for package management
|
||||
- ✨(summary) improve speaker-to-participant assignment
|
||||
|
||||
### Fixed
|
||||
|
||||
|
||||
@@ -105,6 +105,9 @@ class Settings(BaseSettings):
|
||||
|
||||
# Speaker to user assignment
|
||||
is_resolve_speaker_identities_enabled: bool = True
|
||||
resolve_speaker_identities_default_overlap_threshold: float = 0.5
|
||||
resolve_speaker_identities_enable_split_on_words: bool = True
|
||||
resolve_speaker_identities_max_word_duration: float = 1 # seconds
|
||||
|
||||
# Webhook-related settings
|
||||
webhook_max_retries: int = 2
|
||||
|
||||
@@ -14,11 +14,11 @@ from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
from summary.core.config import get_settings
|
||||
|
||||
# Minimum fraction of a speaker's total duration that must overlap with a
|
||||
# participant's VAD to accept the assignment.
|
||||
DEFAULT_OVERLAP_THRESHOLD = 0.5
|
||||
settings = get_settings()
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -74,7 +74,7 @@ class AssignmentResult:
|
||||
f" ({item['speaker']})" if name_to_speaker_count[name] > 1 else ""
|
||||
) # Add suffix only if there are multiple detected speakers per user
|
||||
return {**item, "speaker": f"{name}{suffix}"}
|
||||
return item
|
||||
return {**item}
|
||||
|
||||
def _process_segment(
|
||||
item: dict[str, Any], include_words: bool = False
|
||||
@@ -138,6 +138,78 @@ def _overlap_duration(
|
||||
return overlap
|
||||
|
||||
|
||||
def _format_timelines_debug(
|
||||
participant_timelines: dict[str, list[Interval]],
|
||||
participant_names: dict[str, str],
|
||||
speaker_timelines: dict[str, list[Interval]],
|
||||
) -> str:
|
||||
"""Render participant and speaker timelines side-by-side for debugging.
|
||||
|
||||
Each row is the slice between two consecutive interval boundaries
|
||||
(drawn from both sides). A filled cell marks an active participant
|
||||
(left block) or speaker (right block) during that slice, so vertical
|
||||
alignment makes overlap visually obvious.
|
||||
"""
|
||||
participant_ids = sorted(participant_timelines.keys())
|
||||
speaker_labels = sorted(speaker_timelines.keys())
|
||||
|
||||
if not participant_ids and not speaker_labels:
|
||||
return "(no timelines)"
|
||||
|
||||
boundaries: set[float] = set()
|
||||
for intervals in (*participant_timelines.values(), *speaker_timelines.values()):
|
||||
for iv in intervals:
|
||||
boundaries.add(iv.start)
|
||||
boundaries.add(iv.end)
|
||||
sorted_boundaries = sorted(boundaries)
|
||||
if len(sorted_boundaries) < 2:
|
||||
return "(no intervals)"
|
||||
|
||||
p_headers = [participant_names.get(pid, pid) for pid in participant_ids]
|
||||
s_headers = list(speaker_labels)
|
||||
p_widths = [max(len(h), 3) for h in p_headers]
|
||||
s_widths = [max(len(h), 3) for h in s_headers]
|
||||
|
||||
def _active(intervals: list[Interval], lo: float, hi: float) -> bool:
|
||||
mid = (lo + hi) / 2
|
||||
return any(iv.start <= mid < iv.end for iv in intervals)
|
||||
|
||||
def _cells(
|
||||
intervals_list: list[list[Interval]],
|
||||
widths: list[int],
|
||||
lo: float,
|
||||
hi: float,
|
||||
) -> str:
|
||||
return " ".join(
|
||||
("█" * w if _active(iv, lo, hi) else "·" * w)
|
||||
for iv, w in zip(intervals_list, widths, strict=True)
|
||||
)
|
||||
|
||||
p_iv_list = [participant_timelines[pid] for pid in participant_ids]
|
||||
s_iv_list = [speaker_timelines[sl] for sl in speaker_labels]
|
||||
|
||||
time_col = "[ start → end]"
|
||||
p_hdr = (
|
||||
" ".join(h.center(w) for h, w in zip(p_headers, p_widths, strict=True))
|
||||
or "(none)"
|
||||
)
|
||||
s_hdr = (
|
||||
" ".join(h.center(w) for h, w in zip(s_headers, s_widths, strict=True))
|
||||
or "(none)"
|
||||
)
|
||||
sep = " || "
|
||||
lines = [
|
||||
f"{time_col} {p_hdr}{sep}{s_hdr}",
|
||||
"-" * (len(time_col) + 2 + len(p_hdr) + len(sep) + len(s_hdr)),
|
||||
]
|
||||
for lo, hi in zip(sorted_boundaries, sorted_boundaries[1:], strict=False):
|
||||
time_str = f"[{lo:8.2f} → {hi:8.2f}]"
|
||||
p_row = _cells(p_iv_list, p_widths, lo, hi) or " " * len(p_hdr)
|
||||
s_row = _cells(s_iv_list, s_widths, lo, hi) or " " * len(s_hdr)
|
||||
lines.append(f"{time_str} {p_row}{sep}{s_row}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _build_participant_timelines(
|
||||
metadata: dict[str, Any],
|
||||
recording_start_datetime: datetime,
|
||||
@@ -199,24 +271,50 @@ def _build_participant_timelines(
|
||||
return intervals, participants_info
|
||||
|
||||
|
||||
def _build_speaker_timelines(
|
||||
transcription: Any,
|
||||
) -> dict[str, list[Interval]]:
|
||||
def _build_speaker_timelines(transcription: Any) -> dict[str, list[Interval]]:
|
||||
"""Build interval timelines from WhisperX transcription segments."""
|
||||
intervals: dict[str, list[Interval]] = {}
|
||||
|
||||
segments = transcription.segments if hasattr(transcription, "segments") else []
|
||||
max_word_duration = settings.resolve_speaker_identities_max_word_duration
|
||||
|
||||
for segment in segments:
|
||||
speaker = segment.get("speaker")
|
||||
if speaker is None:
|
||||
continue
|
||||
intervals.setdefault(speaker, []).append(
|
||||
Interval(segment["start"], segment["end"])
|
||||
)
|
||||
|
||||
words = [
|
||||
w
|
||||
for w in segment.get("words", [])
|
||||
if w.get("start") is not None and w.get("end") is not None
|
||||
]
|
||||
if not words:
|
||||
intervals.setdefault(speaker, []).append(
|
||||
Interval(segment["start"], segment["end"])
|
||||
)
|
||||
continue
|
||||
|
||||
start_time: float | None = segment["start"]
|
||||
for word in words:
|
||||
if start_time is None:
|
||||
start_time = word["start"]
|
||||
if not settings.resolve_speaker_identities_enable_split_on_words:
|
||||
continue
|
||||
if word["end"] - word["start"] > max_word_duration:
|
||||
end_time = word["start"] + max_word_duration
|
||||
if end_time > start_time:
|
||||
intervals.setdefault(speaker, []).append(
|
||||
Interval(start_time, end_time)
|
||||
)
|
||||
start_time = None
|
||||
|
||||
if start_time is not None:
|
||||
last = words[-1]
|
||||
end_time = min(last["end"], last["start"] + max_word_duration)
|
||||
if end_time > start_time:
|
||||
intervals.setdefault(speaker, []).append(Interval(start_time, end_time))
|
||||
|
||||
for speaker, speaker_intervals in intervals.items():
|
||||
intervals[speaker] = _merge_intervals(speaker_intervals)
|
||||
|
||||
return intervals
|
||||
|
||||
|
||||
@@ -225,7 +323,7 @@ def resolve_speaker_identities(
|
||||
transcription: Any,
|
||||
recording_start_datetime: datetime,
|
||||
recording_end_datetime: datetime,
|
||||
overlap_threshold: float = DEFAULT_OVERLAP_THRESHOLD,
|
||||
overlap_threshold: float = settings.resolve_speaker_identities_default_overlap_threshold, # noqa: E501
|
||||
) -> AssignmentResult:
|
||||
"""Match WhisperX speaker labels to participants.
|
||||
|
||||
@@ -247,9 +345,14 @@ def resolve_speaker_identities(
|
||||
speaker_timelines = _build_speaker_timelines(transcription)
|
||||
|
||||
logger.debug(
|
||||
"Assignment inputs: %d participants, %d speakers",
|
||||
"Assignment inputs: %d participants, %d speakers\n%s\n%s\n%s",
|
||||
len(participant_timelines),
|
||||
len(speaker_timelines),
|
||||
participant_timelines,
|
||||
speaker_timelines,
|
||||
_format_timelines_debug(
|
||||
participant_timelines, participant_names, speaker_timelines
|
||||
),
|
||||
)
|
||||
|
||||
result = AssignmentResult()
|
||||
|
||||
@@ -4,10 +4,12 @@ import math
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
|
||||
from summary.core import user_assign
|
||||
from summary.core.user_assign import (
|
||||
AssignmentResult,
|
||||
Interval,
|
||||
SpeakerAssignment,
|
||||
_build_speaker_timelines,
|
||||
_merge_intervals,
|
||||
_overlap_duration,
|
||||
_total_duration,
|
||||
@@ -73,12 +75,22 @@ DIARIZATION_SINGLE_SPEAKER = FakeTranscription(
|
||||
"end": 3.545,
|
||||
"text": " The stale smell.",
|
||||
"speaker": "SPEAKER_00",
|
||||
"words": [
|
||||
{"word": "The", "start": 1.363, "end": 1.8},
|
||||
{"word": "stale", "start": 1.8, "end": 2.7},
|
||||
{"word": "smell.", "start": 2.7, "end": 3.545},
|
||||
],
|
||||
},
|
||||
{
|
||||
"start": 4.466,
|
||||
"end": 6.247,
|
||||
"text": "It takes heat.",
|
||||
"speaker": "SPEAKER_00",
|
||||
"words": [
|
||||
{"word": "It", "start": 4.466, "end": 4.7},
|
||||
{"word": "takes", "start": 4.7, "end": 5.5},
|
||||
{"word": "heat.", "start": 5.5, "end": 6.247},
|
||||
],
|
||||
},
|
||||
],
|
||||
)
|
||||
@@ -165,6 +177,174 @@ class TestTotalDuration:
|
||||
assert math.isclose(_total_duration([]), 0.0)
|
||||
|
||||
|
||||
class TestBuildSpeakerTimelines:
|
||||
"""Tests for _build_speaker_timelines."""
|
||||
|
||||
def test_segment_without_words_falls_back_to_segment_bounds(self):
|
||||
"""Segments missing a `words` key use the segment start/end as one interval."""
|
||||
transcription = FakeTranscription(
|
||||
segments=[{"start": 1.5, "end": 3.5, "speaker": "SPEAKER_00"}],
|
||||
)
|
||||
result = _build_speaker_timelines(transcription)
|
||||
assert result == {"SPEAKER_00": [Interval(1.5, 3.5)]}
|
||||
|
||||
def test_segment_with_only_none_word_timestamps_falls_back(self):
|
||||
"""If every word has None start/end, fall back to segment bounds."""
|
||||
transcription = FakeTranscription(
|
||||
segments=[
|
||||
{
|
||||
"start": 1.0,
|
||||
"end": 4.0,
|
||||
"speaker": "SPEAKER_00",
|
||||
"words": [
|
||||
{"word": "hi", "start": None, "end": None},
|
||||
{"word": "there", "start": None, "end": None},
|
||||
],
|
||||
},
|
||||
],
|
||||
)
|
||||
result = _build_speaker_timelines(transcription)
|
||||
assert result == {"SPEAKER_00": [Interval(1.0, 4.0)]}
|
||||
|
||||
def test_short_words_only_uses_segment_start_and_last_word_end(self):
|
||||
"""With no overly long words, the interval runs segment start to end."""
|
||||
transcription = FakeTranscription(
|
||||
segments=[
|
||||
{
|
||||
"start": 1.0,
|
||||
"end": 5.0,
|
||||
"speaker": "SPEAKER_00",
|
||||
"words": [
|
||||
{"word": "a", "start": 1.0, "end": 1.3},
|
||||
{"word": "b", "start": 1.4, "end": 1.7},
|
||||
{"word": "c", "start": 1.8, "end": 2.1},
|
||||
],
|
||||
},
|
||||
],
|
||||
)
|
||||
result = _build_speaker_timelines(transcription)
|
||||
# Tail: min(2.1, 1.8 + 1.0) = 2.1
|
||||
assert result == {"SPEAKER_00": [Interval(1.0, 2.1)]}
|
||||
|
||||
def test_long_word_caps_interval_at_max_duration(self):
|
||||
"""A word longer than the max-word-duration cap truncates the segment."""
|
||||
max_word_duration = (
|
||||
user_assign.settings.resolve_speaker_identities_max_word_duration
|
||||
)
|
||||
transcription = FakeTranscription(
|
||||
segments=[
|
||||
{
|
||||
"start": 0.0,
|
||||
"end": max_word_duration + 7,
|
||||
"speaker": "SPEAKER_00",
|
||||
"words": [
|
||||
{
|
||||
"word": "pause",
|
||||
"start": 0.0,
|
||||
"end": max_word_duration + 7,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
)
|
||||
result = _build_speaker_timelines(transcription)
|
||||
assert result == {"SPEAKER_00": [Interval(0.0, max_word_duration)]}
|
||||
|
||||
def test_long_word_in_middle_splits_segment(self):
|
||||
"""Short words around a long word produce two intervals (before-cap + after)."""
|
||||
transcription = FakeTranscription(
|
||||
segments=[
|
||||
{
|
||||
"start": 0.0,
|
||||
"end": 20.0,
|
||||
"speaker": "SPEAKER_00",
|
||||
"words": [
|
||||
{"word": "a", "start": 0.0, "end": 0.5},
|
||||
{"word": "long", "start": 1.0, "end": 15.0},
|
||||
{"word": "z", "start": 18.0, "end": 18.4},
|
||||
],
|
||||
},
|
||||
],
|
||||
)
|
||||
result = _build_speaker_timelines(transcription)
|
||||
# First emit: (0.0, 1.0 + 1.0). Then start_time resets, picks up at "z" (18.0).
|
||||
# Tail: min(18.4, 18.0 + 1.0) = 18.4. So second interval is (18.0, 18.4).
|
||||
assert result == {
|
||||
"SPEAKER_00": [Interval(0.0, 2.0), Interval(18.0, 18.4)],
|
||||
}
|
||||
|
||||
def test_tail_word_is_capped_at_max_duration(self):
|
||||
"""The trailing word's end is capped at word.start + max_word_duration."""
|
||||
transcription = FakeTranscription(
|
||||
segments=[
|
||||
{
|
||||
"start": 0.0,
|
||||
"end": 50.0,
|
||||
"speaker": "SPEAKER_00",
|
||||
"words": [
|
||||
{"word": "a", "start": 0.0, "end": 0.4},
|
||||
# Last word ends inside the cap, so the cap doesn't apply.
|
||||
{"word": "b", "start": 1.0, "end": 1.5},
|
||||
],
|
||||
},
|
||||
],
|
||||
)
|
||||
result = _build_speaker_timelines(transcription)
|
||||
# Tail: min(1.5, 1.0 + 1.0) = 1.5
|
||||
assert result == {"SPEAKER_00": [Interval(0.0, 1.5)]}
|
||||
|
||||
def test_split_on_words_disabled_keeps_segment_as_one_interval(self, monkeypatch):
|
||||
"""With splitting disabled, long words don't split the interval."""
|
||||
monkeypatch.setattr(
|
||||
user_assign,
|
||||
"settings",
|
||||
user_assign.settings.model_copy(
|
||||
update={"resolve_speaker_identities_enable_split_on_words": False},
|
||||
),
|
||||
)
|
||||
transcription = FakeTranscription(
|
||||
segments=[
|
||||
{
|
||||
"start": 0.0,
|
||||
"end": 20.0,
|
||||
"speaker": "SPEAKER_00",
|
||||
"words": [
|
||||
{"word": "a", "start": 0.0, "end": 0.5},
|
||||
{"word": "long", "start": 1.0, "end": 15.0},
|
||||
{"word": "z", "start": 18.0, "end": 18.4},
|
||||
],
|
||||
},
|
||||
],
|
||||
)
|
||||
result = _build_speaker_timelines(transcription)
|
||||
# No mid-segment split; tail caps at min(18.4, 18.0 + 1.0) = 18.4.
|
||||
assert result == {"SPEAKER_00": [Interval(0.0, 18.4)]}
|
||||
|
||||
def test_multiple_speakers_keep_separate_timelines(self):
|
||||
"""Segments from different speakers populate independent timeline entries."""
|
||||
transcription = FakeTranscription(
|
||||
segments=[
|
||||
{
|
||||
"start": 0.0,
|
||||
"end": 1.0,
|
||||
"speaker": "SPEAKER_00",
|
||||
"words": [{"word": "hi", "start": 0.0, "end": 0.5}],
|
||||
},
|
||||
{
|
||||
"start": 2.0,
|
||||
"end": 3.0,
|
||||
"speaker": "SPEAKER_01",
|
||||
"words": [{"word": "yo", "start": 2.0, "end": 2.5}],
|
||||
},
|
||||
],
|
||||
)
|
||||
result = _build_speaker_timelines(transcription)
|
||||
assert result == {
|
||||
"SPEAKER_00": [Interval(0.0, 0.5)],
|
||||
"SPEAKER_01": [Interval(2.0, 2.5)],
|
||||
}
|
||||
|
||||
|
||||
class TestResolveSpeakerIdentities:
|
||||
"""Tests for resolve_speaker_identities."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user