diff --git a/litellm/llms/elevenlabs/audio_transcription/transformation.py b/litellm/llms/elevenlabs/audio_transcription/transformation.py index a2b25760aac..4b34128868e 100644 --- a/litellm/llms/elevenlabs/audio_transcription/transformation.py +++ b/litellm/llms/elevenlabs/audio_transcription/transformation.py @@ -23,6 +23,20 @@ from ...base_llm.audio_transcription.transformation import ( from ..common_utils import ElevenLabsException +def _to_form_value(value: object) -> str: + """Serialize a multipart form-field value the way ElevenLabs expects. + + httpx (which the ElevenLabs SDK uses) encodes booleans as lowercase + ``true``/``false``. Python's ``str(True)`` yields ``"True"``, which the + ElevenLabs API does not recognize — so a boolean flag such as + ``use_multi_channel`` or ``diarize`` would be silently ignored. Normalize + bools to lowercase; everything else is stringified as before. + """ + if isinstance(value, bool): + return "true" if value else "false" + return str(value) + + class ElevenLabsAudioTranscriptionConfig(BaseAudioTranscriptionConfig): @property def custom_llm_provider(self) -> str: @@ -79,7 +93,7 @@ class ElevenLabsAudioTranscriptionConfig(BaseAudioTranscriptionConfig): for key, value in optional_params.items(): if key in self.get_supported_openai_params(model) and value is not None: # Convert values to strings for form data, but skip None values - form_data[key] = str(value) + form_data[key] = _to_form_value(value) ######################################################### # Add Provider Specific Parameters @@ -91,7 +105,7 @@ class ElevenLabsAudioTranscriptionConfig(BaseAudioTranscriptionConfig): ) for key, value in provider_specific_params.items(): - form_data[key] = str(value) + form_data[key] = _to_form_value(value) ######################################################### ######################################################### @@ -140,6 +154,31 @@ class ElevenLabsAudioTranscriptionConfig(BaseAudioTranscriptionConfig): } ) + # Surface ElevenLabs multichannel output. With `use_multi_channel=true` + # the response carries a per-channel `transcripts` array (each entry has a + # `channel_index` plus its own `words`) instead of a flat top-level `text`. + # Pass it through verbatim so callers can attribute words to a speaker by + # channel; without this the per-channel data is dropped. + transcripts = response_json.get("transcripts") + if isinstance(transcripts, list): + response["transcripts"] = transcripts + elif isinstance(response_json.get("words"), list): + # Single-channel (mono) response: no `transcripts` array, only flat + # top-level `words`. The OpenAI-format `response["words"]` above is + # lossy — it renames `text`->`word` and drops spacing/punctuation and + # audio events. Wrap the RAW words as one channel so callers get the + # same verbatim per-word tokens as multichannel and can segment + # bubbles uniformly; without this a mono transcript loses punctuation. + response["transcripts"] = [ + {"channel_index": 0, "words": response_json["words"]} + ] + + # Carry the billed audio duration (present on both single- and + # multi-channel responses) so callers can attribute cost. + duration = response_json.get("audio_duration_secs") + if duration is not None: + response["audio_duration_secs"] = duration + # Store full response in hidden params response._hidden_params = response_json diff --git a/tests/test_litellm/llms/elevenlabs/audio_transcription/test_elevenlabs_audio_transcription_transformation.py b/tests/test_litellm/llms/elevenlabs/audio_transcription/test_elevenlabs_audio_transcription_transformation.py new file mode 100644 index 00000000000..cc94a8aee77 --- /dev/null +++ b/tests/test_litellm/llms/elevenlabs/audio_transcription/test_elevenlabs_audio_transcription_transformation.py @@ -0,0 +1,110 @@ +import json + +import httpx + +from litellm.llms.base_llm.audio_transcription.transformation import ( + AudioTranscriptionRequestData, +) +from litellm.llms.elevenlabs.audio_transcription.transformation import ( + ElevenLabsAudioTranscriptionConfig, +) + + +def _response(payload: dict) -> httpx.Response: + return httpx.Response( + 200, + content=json.dumps(payload).encode(), + request=httpx.Request("POST", "https://api.elevenlabs.io/v1/speech-to-text"), + ) + + +def test_request_serializes_bool_form_values_lowercase(): + """Boolean optional params must reach ElevenLabs as lowercase ``true``/``false`` + (httpx-style), not Python's ``str(True)`` -> ``"True"``; otherwise flags such as + ``use_multi_channel`` / ``diarize`` are silently ignored.""" + config = ElevenLabsAudioTranscriptionConfig() + + result = config.transform_audio_transcription_request( + model="scribe_v2", + audio_file=b"\x00\x01\x02\x03", + optional_params={"use_multi_channel": True, "diarize": False}, + litellm_params={}, + ) + + assert isinstance(result, AudioTranscriptionRequestData) + assert result.data["model_id"] == "scribe_v2" + assert result.data["use_multi_channel"] == "true" + assert result.data["diarize"] == "false" + + +def test_response_preserves_multichannel_transcripts(): + """A ``use_multi_channel`` response carries a per-channel ``transcripts`` array; + it must survive on the TranscriptionResponse (along with ``audio_duration_secs``) + rather than being flattened to a single ``text``.""" + config = ElevenLabsAudioTranscriptionConfig() + + payload = { + "transcripts": [ + { + "channel_index": 0, + "words": [{"text": "hello", "start": 0.0, "end": 0.4, "type": "word"}], + }, + { + "channel_index": 1, + "words": [{"text": "hi", "start": 0.2, "end": 0.5, "type": "word"}], + }, + ], + "audio_duration_secs": 1.5, + "language_code": "en", + } + + response = config.transform_audio_transcription_response(_response(payload)) + + assert [t["channel_index"] for t in response["transcripts"]] == [0, 1] + assert response["transcripts"][0]["words"][0]["text"] == "hello" + assert response["audio_duration_secs"] == 1.5 + + +def test_response_single_channel_is_unchanged(): + """Single-channel responses keep the flat ``text`` and gain no ``transcripts`` key.""" + config = ElevenLabsAudioTranscriptionConfig() + + response = config.transform_audio_transcription_response( + _response({"text": "hello world", "language_code": "en"}) + ) + + assert response.text == "hello world" + assert "transcripts" not in response.model_dump() + + +def test_response_mono_words_synthesize_raw_single_channel(): + """A mono (single-channel) response carries raw ``words`` but no ``transcripts``. + We synthesize a one-channel ``transcripts`` entry holding the RAW words verbatim + (``text``/``type``/spacing preserved), so callers get the same per-word tokens as + multichannel. The lossy OpenAI-format ``words`` (renamed ``word``, punctuation + dropped) stays too for OpenAI-compat consumers.""" + config = ElevenLabsAudioTranscriptionConfig() + + payload = { + "text": "hello world", + "language_code": "en", + "audio_duration_secs": 1.5, + "words": [ + {"text": "hello", "start": 0.0, "end": 0.4, "type": "word"}, + {"text": " ", "start": 0.4, "end": 0.5, "type": "spacing"}, + {"text": "world", "start": 0.5, "end": 0.9, "type": "word"}, + ], + } + + response = config.transform_audio_transcription_response(_response(payload)) + + # Synthesized single channel with the RAW words (spacing + type intact). + assert [t["channel_index"] for t in response["transcripts"]] == [0] + raw = response["transcripts"][0]["words"] + assert [w["text"] for w in raw] == ["hello", " ", "world"] + assert [w["type"] for w in raw] == ["word", "spacing", "word"] + # OpenAI-format words unchanged: only real words, renamed key, no spacing. + assert response["words"] == [ + {"word": "hello", "start": 0.0, "end": 0.4}, + {"word": "world", "start": 0.5, "end": 0.9}, + ]