From 3922bb6ed5f57142d80a231d223284ed39190aa4 Mon Sep 17 00:00:00 2001 From: Geoffray Viossat <4362195+gvioss@users.noreply.github.com> Date: Sun, 2 Nov 2025 18:45:18 +0100 Subject: [PATCH] fix: return the diarized transcript when it's required in the request (#16133) --- .../audio_transcription/transformation.py | 57 ++++- ...gram_audio_transcription_transformation.py | 220 ++++++++++++++++++ 2 files changed, 275 insertions(+), 2 deletions(-) diff --git a/litellm/llms/deepgram/audio_transcription/transformation.py b/litellm/llms/deepgram/audio_transcription/transformation.py index faa68e97eae..fe63ad11bc7 100644 --- a/litellm/llms/deepgram/audio_transcription/transformation.py +++ b/litellm/llms/deepgram/audio_transcription/transformation.py @@ -90,8 +90,21 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig): first_channel = response_json["results"]["channels"][0] first_alternative = first_channel["alternatives"][0] - # Extract the full transcript - text = first_alternative["transcript"] + # Detect if diarization is active by checking if words have 'speaker' field + has_diarization = False + if "words" in first_alternative and len(first_alternative["words"]) > 0: + has_diarization = "speaker" in first_alternative["words"][0] + + # Extract the transcript based on diarization mode + if not has_diarization: + # No diarization: use the standard transcript + text = first_alternative["transcript"] + elif "paragraphs" in first_alternative: + # Diarization with paragraphs: use the pre-formatted diarized transcript + text = first_alternative["paragraphs"]["transcript"] + else: + # Diarization without paragraphs: reconstruct from words + text = self._reconstruct_diarized_transcript(first_alternative["words"]) # Create TranscriptionResponse object response = TranscriptionResponse(text=text) @@ -122,6 +135,46 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig): f"Error transforming Deepgram response: {str(e)}\nResponse: {raw_response.text}" ) + def _reconstruct_diarized_transcript(self, words: list) -> str: + """ + Reconstructs a diarized transcript from words with speaker information. + + Args: + words: List of word objects with speaker, word, and optionally punctuated_word + + Returns: + Formatted transcript with speaker labels + """ + if not words: + return "" + + segments = [] + current_speaker = None + current_words = [] + + for word_obj in words: + speaker = word_obj.get("speaker") + # Use punctuated_word if available, otherwise fall back to word + word_text = word_obj.get("punctuated_word", word_obj.get("word", "")) + + if speaker != current_speaker: + # New speaker: save previous segment and start new one + if current_words: + segments.append( + f"Speaker {current_speaker}: {' '.join(current_words)}" + ) + current_speaker = speaker + current_words = [word_text] + else: + # Same speaker: add word to current segment + current_words.append(word_text) + + # Add the last segment + if current_words: + segments.append(f"\nSpeaker {current_speaker}: {' '.join(current_words)}\n") + + return "\n".join(segments) + def get_complete_url( self, api_base: Optional[str], diff --git a/tests/test_litellm/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py b/tests/test_litellm/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py index 6a0f4f8fa33..0962206476d 100644 --- a/tests/test_litellm/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py +++ b/tests/test_litellm/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py @@ -2,6 +2,7 @@ import io import os import pathlib import sys +from unittest.mock import MagicMock import pytest @@ -16,6 +17,7 @@ from litellm.llms.base_llm.audio_transcription.transformation import ( from litellm.llms.deepgram.audio_transcription.transformation import ( DeepgramAudioTranscriptionConfig, ) +from litellm.types.utils import TranscriptionResponse @pytest.fixture @@ -238,3 +240,221 @@ def test_get_complete_url_with_detect_language_and_other_params(): assert "punctuate=true" in url assert "diarize=false" in url assert url.startswith("https://api.deepgram.com/v1/listen?") + + +def test_transform_response_without_diarization(): + """Test response transformation without diarization""" + handler = DeepgramAudioTranscriptionConfig() + + # Mock response without diarization + mock_response = MagicMock() + mock_response.json.return_value = { + "metadata": { + "duration": 10.5, + }, + "results": { + "channels": [ + { + "alternatives": [ + { + "transcript": "Hello this is a test.", + "confidence": 0.99, + "words": [ + {"word": "Hello", "start": 0.0, "end": 0.5}, + {"word": "this", "start": 0.6, "end": 0.8}, + {"word": "is", "start": 0.9, "end": 1.1}, + {"word": "a", "start": 1.2, "end": 1.3}, + {"word": "test", "start": 1.4, "end": 1.8}, + ], + } + ] + } + ] + }, + } + + result = handler.transform_audio_transcription_response(mock_response) + + assert isinstance(result, TranscriptionResponse) + assert result.text == "Hello this is a test." + assert result["task"] == "transcribe" + assert result["duration"] == 10.5 + assert len(result["words"]) == 5 + + +def test_transform_response_with_diarization_and_paragraphs(): + """Test response transformation with diarization and paragraphs property""" + handler = DeepgramAudioTranscriptionConfig() + + # Mock response with diarization and paragraphs + mock_response = MagicMock() + mock_response.json.return_value = { + "metadata": { + "duration": 15.0, + }, + "results": { + "channels": [ + { + "alternatives": [ + { + "transcript": "Hello how are you I am fine thanks", + "paragraphs": { + "transcript": "\nSpeaker 0: Hello how are you\n\nSpeaker 1: I am fine thanks\n" + }, + "words": [ + {"word": "Hello", "start": 0.0, "end": 0.5, "speaker": 0}, + {"word": "how", "start": 0.6, "end": 0.8, "speaker": 0}, + {"word": "are", "start": 0.9, "end": 1.1, "speaker": 0}, + {"word": "you", "start": 1.2, "end": 1.3, "speaker": 0}, + {"word": "I", "start": 2.0, "end": 2.2, "speaker": 1}, + {"word": "am", "start": 2.3, "end": 2.5, "speaker": 1}, + {"word": "fine", "start": 2.6, "end": 2.9, "speaker": 1}, + {"word": "thanks", "start": 3.0, "end": 3.5, "speaker": 1}, + ], + } + ] + } + ] + }, + } + + result = handler.transform_audio_transcription_response(mock_response) + + assert isinstance(result, TranscriptionResponse) + # Should use the pre-formatted paragraphs transcript + assert result.text == "\nSpeaker 0: Hello how are you\n\nSpeaker 1: I am fine thanks\n" + assert result["task"] == "transcribe" + assert result["duration"] == 15.0 + + +def test_transform_response_with_diarization_without_paragraphs(): + """Test response transformation with diarization but no paragraphs property""" + handler = DeepgramAudioTranscriptionConfig() + + # Mock response with diarization but without paragraphs + mock_response = MagicMock() + mock_response.json.return_value = { + "metadata": { + "duration": 15.0, + }, + "results": { + "channels": [ + { + "alternatives": [ + { + "transcript": "Hello how are you I am fine thanks", + "words": [ + {"word": "hello", "punctuated_word": "Hello", "start": 0.0, "end": 0.5, "speaker": 0}, + {"word": "how", "punctuated_word": "how", "start": 0.6, "end": 0.8, "speaker": 0}, + {"word": "are", "punctuated_word": "are", "start": 0.9, "end": 1.1, "speaker": 0}, + {"word": "you", "punctuated_word": "you", "start": 1.2, "end": 1.3, "speaker": 0}, + {"word": "i", "punctuated_word": "I", "start": 2.0, "end": 2.2, "speaker": 1}, + {"word": "am", "punctuated_word": "am", "start": 2.3, "end": 2.5, "speaker": 1}, + {"word": "fine", "punctuated_word": "fine", "start": 2.6, "end": 2.9, "speaker": 1}, + {"word": "thanks", "punctuated_word": "thanks.", "start": 3.0, "end": 3.5, "speaker": 1}, + ], + } + ] + } + ] + }, + } + + result = handler.transform_audio_transcription_response(mock_response) + + assert isinstance(result, TranscriptionResponse) + # Should reconstruct from words using punctuated_word + expected_text = "Speaker 0: Hello how are you\n\nSpeaker 1: I am fine thanks.\n" + assert result.text == expected_text + assert result["task"] == "transcribe" + assert result["duration"] == 15.0 + + +def test_reconstruct_diarized_transcript_with_punctuated_words(): + """Test reconstruction uses punctuated_word when available""" + handler = DeepgramAudioTranscriptionConfig() + + words = [ + {"word": "hello", "punctuated_word": "Hello", "speaker": 0}, + {"word": "world", "punctuated_word": "world!", "speaker": 0}, + {"word": "how", "punctuated_word": "How", "speaker": 1}, + {"word": "are", "punctuated_word": "are", "speaker": 1}, + {"word": "you", "punctuated_word": "you?", "speaker": 1}, + ] + + result = handler._reconstruct_diarized_transcript(words) + + # Check that punctuated_word is used and speakers are properly separated + assert "Hello world!" in result + assert "How are you?" in result + assert "Speaker 0:" in result + assert "Speaker 1:" in result + + +def test_reconstruct_diarized_transcript_fallback_to_word(): + """Test reconstruction falls back to 'word' when punctuated_word is missing""" + handler = DeepgramAudioTranscriptionConfig() + + words = [ + {"word": "Hello", "speaker": 0}, # No punctuated_word + {"word": "world", "speaker": 0}, + {"word": "test", "punctuated_word": "test.", "speaker": 1}, # Has punctuated_word + ] + + result = handler._reconstruct_diarized_transcript(words) + + # Should use 'word' when punctuated_word is not available + assert "Hello world" in result + assert "test." in result + assert "Speaker 0:" in result + assert "Speaker 1:" in result + + +def test_reconstruct_diarized_transcript_empty_words(): + """Test reconstruction with empty words list""" + handler = DeepgramAudioTranscriptionConfig() + + result = handler._reconstruct_diarized_transcript([]) + + assert result == "" + + +def test_reconstruct_diarized_transcript_single_speaker(): + """Test reconstruction with single speaker""" + handler = DeepgramAudioTranscriptionConfig() + + words = [ + {"word": "This", "punctuated_word": "This", "speaker": 0}, + {"word": "is", "punctuated_word": "is", "speaker": 0}, + {"word": "a", "punctuated_word": "a", "speaker": 0}, + {"word": "test", "punctuated_word": "test.", "speaker": 0}, + ] + + result = handler._reconstruct_diarized_transcript(words) + + # Should have only one speaker segment + assert result.count("Speaker 0:") == 1 + assert "This is a test." in result + + +def test_reconstruct_diarized_transcript_multiple_speaker_changes(): + """Test reconstruction with multiple speaker changes""" + handler = DeepgramAudioTranscriptionConfig() + + words = [ + {"word": "Hi", "speaker": 0}, + {"word": "there", "speaker": 0}, + {"word": "Hello", "speaker": 1}, + {"word": "back", "speaker": 0}, # Speaker 0 again + {"word": "Thanks", "speaker": 1}, # Speaker 1 again + ] + + result = handler._reconstruct_diarized_transcript(words) + + # Should have 4 speaker segments (0, 1, 0, 1) + assert result.count("Speaker 0:") == 2 + assert result.count("Speaker 1:") == 2 + assert "Hi there" in result + assert "Hello" in result + assert "back" in result + assert "Thanks" in result