fix: return the diarized transcript when it's required in the request (#16133)

litellm/llms/deepgram/audio_transcription/transformation.py

@@ -90,8 +90,21 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
             first_channel = response_json["results"]["channels"][0]
             first_alternative = first_channel["alternatives"][0]
 
-            # Extract the full transcript
-            text = first_alternative["transcript"]
+            # Detect if diarization is active by checking if words have 'speaker' field
+            has_diarization = False
+            if "words" in first_alternative and len(first_alternative["words"]) > 0:
+                has_diarization = "speaker" in first_alternative["words"][0]
+
+            # Extract the transcript based on diarization mode
+            if not has_diarization:
+                # No diarization: use the standard transcript
+                text = first_alternative["transcript"]
+            elif "paragraphs" in first_alternative:
+                # Diarization with paragraphs: use the pre-formatted diarized transcript
+                text = first_alternative["paragraphs"]["transcript"]
+            else:
+                # Diarization without paragraphs: reconstruct from words
+                text = self._reconstruct_diarized_transcript(first_alternative["words"])
 
             # Create TranscriptionResponse object
             response = TranscriptionResponse(text=text)
@@ -122,6 +135,46 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
                 f"Error transforming Deepgram response: {str(e)}\nResponse: {raw_response.text}"
             )
 
+    def _reconstruct_diarized_transcript(self, words: list) -> str:
+        """
+        Reconstructs a diarized transcript from words with speaker information.
+
+        Args:
+            words: List of word objects with speaker, word, and optionally punctuated_word
+
+        Returns:
+            Formatted transcript with speaker labels
+        """
+        if not words:
+            return ""
+
+        segments = []
+        current_speaker = None
+        current_words = []
+
+        for word_obj in words:
+            speaker = word_obj.get("speaker")
+            # Use punctuated_word if available, otherwise fall back to word
+            word_text = word_obj.get("punctuated_word", word_obj.get("word", ""))
+
+            if speaker != current_speaker:
+                # New speaker: save previous segment and start new one
+                if current_words:
+                    segments.append(
+                        f"Speaker {current_speaker}: {' '.join(current_words)}"
+                    )
+                current_speaker = speaker
+                current_words = [word_text]
+            else:
+                # Same speaker: add word to current segment
+                current_words.append(word_text)
+
+        # Add the last segment
+        if current_words:
+            segments.append(f"\nSpeaker {current_speaker}: {' '.join(current_words)}\n")
+
+        return "\n".join(segments)
+
     def get_complete_url(
         self,
         api_base: Optional[str],

tests/test_litellm/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py

@@ -2,6 +2,7 @@ import io
 import os
 import pathlib
 import sys
+from unittest.mock import MagicMock
 
 import pytest
 
@@ -16,6 +17,7 @@ from litellm.llms.base_llm.audio_transcription.transformation import (
 from litellm.llms.deepgram.audio_transcription.transformation import (
     DeepgramAudioTranscriptionConfig,
 )
+from litellm.types.utils import TranscriptionResponse
 
 
 @pytest.fixture
@@ -238,3 +240,221 @@ def test_get_complete_url_with_detect_language_and_other_params():
     assert "punctuate=true" in url
     assert "diarize=false" in url
     assert url.startswith("https://api.deepgram.com/v1/listen?")
+
+
+def test_transform_response_without_diarization():
+    """Test response transformation without diarization"""
+    handler = DeepgramAudioTranscriptionConfig()
+
+    # Mock response without diarization
+    mock_response = MagicMock()
+    mock_response.json.return_value = {
+        "metadata": {
+            "duration": 10.5,
+        },
+        "results": {
+            "channels": [
+                {
+                    "alternatives": [
+                        {
+                            "transcript": "Hello this is a test.",
+                            "confidence": 0.99,
+                            "words": [
+                                {"word": "Hello", "start": 0.0, "end": 0.5},
+                                {"word": "this", "start": 0.6, "end": 0.8},
+                                {"word": "is", "start": 0.9, "end": 1.1},
+                                {"word": "a", "start": 1.2, "end": 1.3},
+                                {"word": "test", "start": 1.4, "end": 1.8},
+                            ],
+                        }
+                    ]
+                }
+            ]
+        },
+    }
+
+    result = handler.transform_audio_transcription_response(mock_response)
+
+    assert isinstance(result, TranscriptionResponse)
+    assert result.text == "Hello this is a test."
+    assert result["task"] == "transcribe"
+    assert result["duration"] == 10.5
+    assert len(result["words"]) == 5
+
+
+def test_transform_response_with_diarization_and_paragraphs():
+    """Test response transformation with diarization and paragraphs property"""
+    handler = DeepgramAudioTranscriptionConfig()
+
+    # Mock response with diarization and paragraphs
+    mock_response = MagicMock()
+    mock_response.json.return_value = {
+        "metadata": {
+            "duration": 15.0,
+        },
+        "results": {
+            "channels": [
+                {
+                    "alternatives": [
+                        {
+                            "transcript": "Hello how are you I am fine thanks",
+                            "paragraphs": {
+                                "transcript": "\nSpeaker 0: Hello how are you\n\nSpeaker 1: I am fine thanks\n"
+                            },
+                            "words": [
+                                {"word": "Hello", "start": 0.0, "end": 0.5, "speaker": 0},
+                                {"word": "how", "start": 0.6, "end": 0.8, "speaker": 0},
+                                {"word": "are", "start": 0.9, "end": 1.1, "speaker": 0},
+                                {"word": "you", "start": 1.2, "end": 1.3, "speaker": 0},
+                                {"word": "I", "start": 2.0, "end": 2.2, "speaker": 1},
+                                {"word": "am", "start": 2.3, "end": 2.5, "speaker": 1},
+                                {"word": "fine", "start": 2.6, "end": 2.9, "speaker": 1},
+                                {"word": "thanks", "start": 3.0, "end": 3.5, "speaker": 1},
+                            ],
+                        }
+                    ]
+                }
+            ]
+        },
+    }
+
+    result = handler.transform_audio_transcription_response(mock_response)
+
+    assert isinstance(result, TranscriptionResponse)
+    # Should use the pre-formatted paragraphs transcript
+    assert result.text == "\nSpeaker 0: Hello how are you\n\nSpeaker 1: I am fine thanks\n"
+    assert result["task"] == "transcribe"
+    assert result["duration"] == 15.0
+
+
+def test_transform_response_with_diarization_without_paragraphs():
+    """Test response transformation with diarization but no paragraphs property"""
+    handler = DeepgramAudioTranscriptionConfig()
+
+    # Mock response with diarization but without paragraphs
+    mock_response = MagicMock()
+    mock_response.json.return_value = {
+        "metadata": {
+            "duration": 15.0,
+        },
+        "results": {
+            "channels": [
+                {
+                    "alternatives": [
+                        {
+                            "transcript": "Hello how are you I am fine thanks",
+                            "words": [
+                                {"word": "hello", "punctuated_word": "Hello", "start": 0.0, "end": 0.5, "speaker": 0},
+                                {"word": "how", "punctuated_word": "how", "start": 0.6, "end": 0.8, "speaker": 0},
+                                {"word": "are", "punctuated_word": "are", "start": 0.9, "end": 1.1, "speaker": 0},
+                                {"word": "you", "punctuated_word": "you", "start": 1.2, "end": 1.3, "speaker": 0},
+                                {"word": "i", "punctuated_word": "I", "start": 2.0, "end": 2.2, "speaker": 1},
+                                {"word": "am", "punctuated_word": "am", "start": 2.3, "end": 2.5, "speaker": 1},
+                                {"word": "fine", "punctuated_word": "fine", "start": 2.6, "end": 2.9, "speaker": 1},
+                                {"word": "thanks", "punctuated_word": "thanks.", "start": 3.0, "end": 3.5, "speaker": 1},
+                            ],
+                        }
+                    ]
+                }
+            ]
+        },
+    }
+
+    result = handler.transform_audio_transcription_response(mock_response)
+
+    assert isinstance(result, TranscriptionResponse)
+    # Should reconstruct from words using punctuated_word
+    expected_text = "Speaker 0: Hello how are you\n\nSpeaker 1: I am fine thanks.\n"
+    assert result.text == expected_text
+    assert result["task"] == "transcribe"
+    assert result["duration"] == 15.0
+
+
+def test_reconstruct_diarized_transcript_with_punctuated_words():
+    """Test reconstruction uses punctuated_word when available"""
+    handler = DeepgramAudioTranscriptionConfig()
+
+    words = [
+        {"word": "hello", "punctuated_word": "Hello", "speaker": 0},
+        {"word": "world", "punctuated_word": "world!", "speaker": 0},
+        {"word": "how", "punctuated_word": "How", "speaker": 1},
+        {"word": "are", "punctuated_word": "are", "speaker": 1},
+        {"word": "you", "punctuated_word": "you?", "speaker": 1},
+    ]
+
+    result = handler._reconstruct_diarized_transcript(words)
+
+    # Check that punctuated_word is used and speakers are properly separated
+    assert "Hello world!" in result
+    assert "How are you?" in result
+    assert "Speaker 0:" in result
+    assert "Speaker 1:" in result
+
+
+def test_reconstruct_diarized_transcript_fallback_to_word():
+    """Test reconstruction falls back to 'word' when punctuated_word is missing"""
+    handler = DeepgramAudioTranscriptionConfig()
+
+    words = [
+        {"word": "Hello", "speaker": 0},  # No punctuated_word
+        {"word": "world", "speaker": 0},
+        {"word": "test", "punctuated_word": "test.", "speaker": 1},  # Has punctuated_word
+    ]
+
+    result = handler._reconstruct_diarized_transcript(words)
+
+    # Should use 'word' when punctuated_word is not available
+    assert "Hello world" in result
+    assert "test." in result
+    assert "Speaker 0:" in result
+    assert "Speaker 1:" in result
+
+
+def test_reconstruct_diarized_transcript_empty_words():
+    """Test reconstruction with empty words list"""
+    handler = DeepgramAudioTranscriptionConfig()
+
+    result = handler._reconstruct_diarized_transcript([])
+
+    assert result == ""
+
+
+def test_reconstruct_diarized_transcript_single_speaker():
+    """Test reconstruction with single speaker"""
+    handler = DeepgramAudioTranscriptionConfig()
+
+    words = [
+        {"word": "This", "punctuated_word": "This", "speaker": 0},
+        {"word": "is", "punctuated_word": "is", "speaker": 0},
+        {"word": "a", "punctuated_word": "a", "speaker": 0},
+        {"word": "test", "punctuated_word": "test.", "speaker": 0},
+    ]
+
+    result = handler._reconstruct_diarized_transcript(words)
+
+    # Should have only one speaker segment
+    assert result.count("Speaker 0:") == 1
+    assert "This is a test." in result
+
+
+def test_reconstruct_diarized_transcript_multiple_speaker_changes():
+    """Test reconstruction with multiple speaker changes"""
+    handler = DeepgramAudioTranscriptionConfig()
+
+    words = [
+        {"word": "Hi", "speaker": 0},
+        {"word": "there", "speaker": 0},
+        {"word": "Hello", "speaker": 1},
+        {"word": "back", "speaker": 0},  # Speaker 0 again
+        {"word": "Thanks", "speaker": 1},  # Speaker 1 again
+    ]
+
+    result = handler._reconstruct_diarized_transcript(words)
+
+    # Should have 4 speaker segments (0, 1, 0, 1)
+    assert result.count("Speaker 0:") == 2
+    assert result.count("Speaker 1:") == 2
+    assert "Hi there" in result
+    assert "Hello" in result
+    assert "back" in result
+    assert "Thanks" in result