From 9b7c449ca550282574afd66344db213d3ff38b4b Mon Sep 17 00:00:00 2001 From: Florent Chehab Date: Fri, 12 Jun 2026 12:47:02 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B(summary)=20explicit=20transcriptio?= =?UTF-8?q?n=20response=20format?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We should be able to use other transcription services, those usually relie on response_format="diarized_json" to produce what we need. Note that as part of this change, we stop using openai library for making this call to avoid casting the result to a payload that doesn't contain the elements we used to rely on. (setting this specific format auto cast the results in openai lib). We keep the old result class used. --- CHANGELOG.md | 4 ++ docs/features/transcription.md | 1 - src/summary/summary/core/celery_worker.py | 60 ++++++++++++++++++++--- src/summary/summary/core/config.py | 1 - 4 files changed, 56 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 787a817d..6b1249a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,10 @@ and this project adheres to - ✨(backend) add command to clean pending and deleted files - 🧱(helm) run clean files command as cronjob +### Changed + +- ✨(summary) generalized stt api call #1420 + ## [1.21.0] - 2026-06-15 ### Added diff --git a/docs/features/transcription.md b/docs/features/transcription.md index e5533795..c6ab9323 100644 --- a/docs/features/transcription.md +++ b/docs/features/transcription.md @@ -80,7 +80,6 @@ sequenceDiagram | whisperx_api_key | Secret | — | API key for accessing WhisperX. | | whisperx_base_url | String | `"https://api.whisperx.com/v1"` | Base URL for the WhisperX API. | | whisperx_asr_model | String | `"whisper-1"` | ASR model used for transcription. | -| whisperx_max_retries | Integer | `0` | Maximum number of retries for WhisperX API requests. | | webhook_max_retries | Integer | `2` | Maximum retries for webhook requests. | | webhook_status_forcelist | List[Int] | `[502, 503, 504]` | HTTP status codes triggering webhook retry. | | webhook_backoff_factor | Float | `0.1` | Exponential backoff factor for webhook retries. | diff --git a/src/summary/summary/core/celery_worker.py b/src/summary/summary/core/celery_worker.py index 944b1d46..ada448b7 100644 --- a/src/summary/summary/core/celery_worker.py +++ b/src/summary/summary/core/celery_worker.py @@ -5,11 +5,14 @@ import json import time from datetime import datetime +from typing import Any +from urllib.parse import urljoin -import openai +import requests import sentry_sdk from celery import Celery, signals from celery.utils.log import get_task_logger +from openai.types.audio import Transcription from requests import exceptions from summary.core.analytics import MetadataManager, get_analytics @@ -101,11 +104,6 @@ def transcribe_audio( ) logger.info("Initiating WhisperX client") - whisperx_client = openai.OpenAI( - api_key=settings.whisperx_api_key.get_secret_value(), - base_url=settings.whisperx_base_url, - max_retries=settings.whisperx_max_retries, - ) # Transcription try: @@ -130,8 +128,54 @@ def transcribe_audio( # Call remote service for transcription transcription_start_time = time.time() - transcription = whisperx_client.audio.transcriptions.create( - model=settings.whisperx_asr_model, file=audio_file, language=language + + api_key = settings.whisperx_api_key.get_secret_value() + base_url = settings.whisperx_base_url + + # We use a manual call to the transcripion endpoint, and we do not + # directly use the OpenAI lib for this. + # This is because, depending on the requested response format, + # the OpenAI lib will cast the response to a different dataclass, + # which can result in stripping out keys & data that we are interested in. + # This is in particular true for word_segments and words. + # WhisperX response is slightly different from OpenAI STT endpoints + # response. + # At the same time "diarized_json" should be the value + # provided to STT endpoints in our context. + url = urljoin(base_url.rstrip("/") + "/", "audio/transcriptions") + res = requests.post( + url, + data={ + "model": settings.whisperx_asr_model, + "language": language, + "timestamp_granularities": ["word", "segment"], + "response_format": "diarized_json", + }, + files={"file": audio_file}, + headers={"Authorization": f"Bearer {api_key}"}, + # Mimic OpenAI's timeout settings + timeout=(60, 10 * 60), + ) + try: + res.raise_for_status() + except requests.exceptions.HTTPError as e: + raise RuntimeError("WhisperX transcription failed, %s", res.text) from e + + transcription_json: dict[str, Any] = res.json() + # We remove the "usage" key from the transcription_json dictionary + # as it may cause issues with parsing inside the Transcription model + # Some API don't share the exact same structure for the "usage" key + transcription_json.pop("usage", None) + + # We force the use of the Transcription model here + # to avoid changing too much code for now. + # Note that it should be WhisperXResponse instead. + transcription = Transcription.model_validate( + # We add a dummy "text" to make the model validate, + # Some API responses lack the "text" key. + {"text": "", **transcription_json}, + extra="allow", + strict=False, ) # Logging diff --git a/src/summary/summary/core/config.py b/src/summary/summary/core/config.py index 28a4a44e..b536713e 100644 --- a/src/summary/summary/core/config.py +++ b/src/summary/summary/core/config.py @@ -92,7 +92,6 @@ class Settings(BaseSettings): whisperx_api_key: SecretStr whisperx_base_url: str = "https://api.openai.com/v1" whisperx_asr_model: str = "whisper-1" - whisperx_max_retries: int = 0 # ISO 639-1 language code (e.g., "en", "fr", "es") whisperx_default_language: Optional[str] = None whisperx_allowed_languages: Set[str] = {"en", "fr", "de", "nl"}