Merge 4b73d62f4e into 071cb49d32

litellm/llms/elevenlabs/audio_transcription/transformation.py

@@ -23,6 +23,20 @@ from ...base_llm.audio_transcription.transformation import (
 from ..common_utils import ElevenLabsException
 
 
+def _to_form_value(value: object) -> str:
+    """Serialize a multipart form-field value the way ElevenLabs expects.
+
+    httpx (which the ElevenLabs SDK uses) encodes booleans as lowercase
+    ``true``/``false``. Python's ``str(True)`` yields ``"True"``, which the
+    ElevenLabs API does not recognize — so a boolean flag such as
+    ``use_multi_channel`` or ``diarize`` would be silently ignored. Normalize
+    bools to lowercase; everything else is stringified as before.
+    """
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    return str(value)
+
+
 class ElevenLabsAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
     @property
     def custom_llm_provider(self) -> str:
@@ -79,7 +93,7 @@ class ElevenLabsAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
         for key, value in optional_params.items():
             if key in self.get_supported_openai_params(model) and value is not None:
                 # Convert values to strings for form data, but skip None values
-                form_data[key] = str(value)
+                form_data[key] = _to_form_value(value)
 
         #########################################################
         # Add Provider Specific Parameters
@@ -91,7 +105,7 @@ class ElevenLabsAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
         )
 
         for key, value in provider_specific_params.items():
-            form_data[key] = str(value)
+            form_data[key] = _to_form_value(value)
         #########################################################
         #########################################################
 
@@ -140,6 +154,31 @@ class ElevenLabsAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
                             }
                         )
 
+            # Surface ElevenLabs multichannel output. With `use_multi_channel=true`
+            # the response carries a per-channel `transcripts` array (each entry has a
+            # `channel_index` plus its own `words`) instead of a flat top-level `text`.
+            # Pass it through verbatim so callers can attribute words to a speaker by
+            # channel; without this the per-channel data is dropped.
+            transcripts = response_json.get("transcripts")
+            if isinstance(transcripts, list):
+                response["transcripts"] = transcripts
+            elif isinstance(response_json.get("words"), list):
+                # Single-channel (mono) response: no `transcripts` array, only flat
+                # top-level `words`. The OpenAI-format `response["words"]` above is
+                # lossy — it renames `text`->`word` and drops spacing/punctuation and
+                # audio events. Wrap the RAW words as one channel so callers get the
+                # same verbatim per-word tokens as multichannel and can segment
+                # bubbles uniformly; without this a mono transcript loses punctuation.
+                response["transcripts"] = [
+                    {"channel_index": 0, "words": response_json["words"]}
+                ]
+
+            # Carry the billed audio duration (present on both single- and
+            # multi-channel responses) so callers can attribute cost.
+            duration = response_json.get("audio_duration_secs")
+            if duration is not None:
+                response["audio_duration_secs"] = duration
+
             # Store full response in hidden params
             response._hidden_params = response_json
 

tests/test_litellm/llms/elevenlabs/audio_transcription/test_elevenlabs_audio_transcription_transformation.py

@@ -0,0 +1,110 @@
+import json
+
+import httpx
+
+from litellm.llms.base_llm.audio_transcription.transformation import (
+    AudioTranscriptionRequestData,
+)
+from litellm.llms.elevenlabs.audio_transcription.transformation import (
+    ElevenLabsAudioTranscriptionConfig,
+)
+
+
+def _response(payload: dict) -> httpx.Response:
+    return httpx.Response(
+        200,
+        content=json.dumps(payload).encode(),
+        request=httpx.Request("POST", "https://api.elevenlabs.io/v1/speech-to-text"),
+    )
+
+
+def test_request_serializes_bool_form_values_lowercase():
+    """Boolean optional params must reach ElevenLabs as lowercase ``true``/``false``
+    (httpx-style), not Python's ``str(True)`` -> ``"True"``; otherwise flags such as
+    ``use_multi_channel`` / ``diarize`` are silently ignored."""
+    config = ElevenLabsAudioTranscriptionConfig()
+
+    result = config.transform_audio_transcription_request(
+        model="scribe_v2",
+        audio_file=b"\x00\x01\x02\x03",
+        optional_params={"use_multi_channel": True, "diarize": False},
+        litellm_params={},
+    )
+
+    assert isinstance(result, AudioTranscriptionRequestData)
+    assert result.data["model_id"] == "scribe_v2"
+    assert result.data["use_multi_channel"] == "true"
+    assert result.data["diarize"] == "false"
+
+
+def test_response_preserves_multichannel_transcripts():
+    """A ``use_multi_channel`` response carries a per-channel ``transcripts`` array;
+    it must survive on the TranscriptionResponse (along with ``audio_duration_secs``)
+    rather than being flattened to a single ``text``."""
+    config = ElevenLabsAudioTranscriptionConfig()
+
+    payload = {
+        "transcripts": [
+            {
+                "channel_index": 0,
+                "words": [{"text": "hello", "start": 0.0, "end": 0.4, "type": "word"}],
+            },
+            {
+                "channel_index": 1,
+                "words": [{"text": "hi", "start": 0.2, "end": 0.5, "type": "word"}],
+            },
+        ],
+        "audio_duration_secs": 1.5,
+        "language_code": "en",
+    }
+
+    response = config.transform_audio_transcription_response(_response(payload))
+
+    assert [t["channel_index"] for t in response["transcripts"]] == [0, 1]
+    assert response["transcripts"][0]["words"][0]["text"] == "hello"
+    assert response["audio_duration_secs"] == 1.5
+
+
+def test_response_single_channel_is_unchanged():
+    """Single-channel responses keep the flat ``text`` and gain no ``transcripts`` key."""
+    config = ElevenLabsAudioTranscriptionConfig()
+
+    response = config.transform_audio_transcription_response(
+        _response({"text": "hello world", "language_code": "en"})
+    )
+
+    assert response.text == "hello world"
+    assert "transcripts" not in response.model_dump()
+
+
+def test_response_mono_words_synthesize_raw_single_channel():
+    """A mono (single-channel) response carries raw ``words`` but no ``transcripts``.
+    We synthesize a one-channel ``transcripts`` entry holding the RAW words verbatim
+    (``text``/``type``/spacing preserved), so callers get the same per-word tokens as
+    multichannel. The lossy OpenAI-format ``words`` (renamed ``word``, punctuation
+    dropped) stays too for OpenAI-compat consumers."""
+    config = ElevenLabsAudioTranscriptionConfig()
+
+    payload = {
+        "text": "hello world",
+        "language_code": "en",
+        "audio_duration_secs": 1.5,
+        "words": [
+            {"text": "hello", "start": 0.0, "end": 0.4, "type": "word"},
+            {"text": " ", "start": 0.4, "end": 0.5, "type": "spacing"},
+            {"text": "world", "start": 0.5, "end": 0.9, "type": "word"},
+        ],
+    }
+
+    response = config.transform_audio_transcription_response(_response(payload))
+
+    # Synthesized single channel with the RAW words (spacing + type intact).
+    assert [t["channel_index"] for t in response["transcripts"]] == [0]
+    raw = response["transcripts"][0]["words"]
+    assert [w["text"] for w in raw] == ["hello", " ", "world"]
+    assert [w["type"] for w in raw] == ["word", "spacing", "word"]
+    # OpenAI-format words unchanged: only real words, renamed key, no spacing.
+    assert response["words"] == [
+        {"word": "hello", "start": 0.0, "end": 0.4},
+        {"word": "world", "start": 0.5, "end": 0.9},
+    ]