From 5085022580dede5339f8fe957d0f91ba7dc0daa0 Mon Sep 17 00:00:00 2001 From: qtim0 <154377292+qtim0@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:07:15 -0500 Subject: [PATCH] fix(audio-transcription): keep transcripts out of transcription error messages (#50137) * fix(gemini): omit raw response body from audio transcription error messages * fix(gemini): raise incomplete transcription interactions as non-retryable 400 and drop response bodies from Vertex transcription errors * fix(audio_transcription): keep response bodies out of transcription errors Keep the upstream status on a Gemini interaction that did not complete, so the router still retries it as it does on main. Gemini stops the same long clip on some tries and not on others, so a retry can recover it. Stop quoting the reply in the Deepgram, ElevenLabs, Watsonx, Mistral, OVHcloud and Scaleway parse errors, and hide pydantic input values on the Gemini and Vertex response models. * fix(audio-transcription): keep reply bodies out of watsonx, whisper, xai and validation errors Watsonx and whisper no longer print the parsed reply after Got: when it has no text field. TranscriptionResponse and the xAI response model hide input in validation errors, so a malformed field cannot quote transcript fragments. Deepgram and ElevenLabs keep the original error text, which is now safe. * fix(audio-transcription): keep transcripts out of the shared converter's parse errors * test(audio-transcription): compare wrapped lookup errors instead of pinning upstream wording --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../convert_dict_to_response.py | 5 +- .../audio_transcription/transformation.py | 2 +- .../audio_transcription/transformation.py | 2 +- .../audio_transcription/transformation.py | 4 +- .../audio_transcription/transformation.py | 2 +- .../transcriptions/whisper_transformation.py | 5 +- .../audio_transcription/transformation.py | 2 +- .../audio_transcription/transformation.py | 2 +- .../gemini_transcribe_transformation.py | 2 +- .../audio_transcription/transformation.py | 2 +- .../audio_transcription/transformation.py | 7 +- .../xai/audio_transcription/transformation.py | 2 +- .../types/llms/gemini_audio_transcription.py | 2 +- .../llms/vertex_ai_gemini_transcription.py | 2 +- .../types/llms/vertex_ai_speech_to_text.py | 2 + litellm/types/utils.py | 2 + .../test_convert_dict_to_response.py | 18 +++++ ...gram_audio_transcription_transformation.py | 28 ++++--- .../test_transformation.py | 35 ++++++--- ...mini_audio_transcription_transformation.py | 77 +++++++++++++++++++ ...tral_audio_transcription_transformation.py | 22 ++++++ .../test_whisper_transformation.py | 9 +++ ...loud_audio_transcription_transformation.py | 10 +++ ...eway_audio_transcription_transformation.py | 11 +++ ...x_ai_audio_transcription_transformation.py | 14 +++- ...tex_ai_gemini_transcribe_transformation.py | 15 ++++ ...sonx_audio_transcription_transformation.py | 26 +++++-- ..._xai_audio_transcription_transformation.py | 19 +++++ 28 files changed, 276 insertions(+), 53 deletions(-) diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 9fb4ad27d75..0fed2cb2fce 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -1072,9 +1072,10 @@ def convert_to_model_response_object( if isinstance(e, APIError): raise + hide_reply: Final = response_type == "audio_transcription" received_args: Final = dict( - response_object=response_object, - model_response_object=model_response_object, + response_object="" if hide_reply else response_object, + model_response_object="" if hide_reply else model_response_object, response_type=response_type, stream=stream, start_time=start_time, diff --git a/litellm/llms/deepgram/audio_transcription/transformation.py b/litellm/llms/deepgram/audio_transcription/transformation.py index 1c8a056116f..2c8b22459bd 100644 --- a/litellm/llms/deepgram/audio_transcription/transformation.py +++ b/litellm/llms/deepgram/audio_transcription/transformation.py @@ -158,7 +158,7 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig): return response except Exception as e: - raise ValueError(f"Error transforming Deepgram response: {e}\nResponse: {raw_response.text}") + raise ValueError(f"Error transforming Deepgram response: {e}") from None def _reconstruct_diarized_transcript(self, words: list) -> str: """ diff --git a/litellm/llms/elevenlabs/audio_transcription/transformation.py b/litellm/llms/elevenlabs/audio_transcription/transformation.py index 3dc42f35d3c..b09aa97f6fe 100644 --- a/litellm/llms/elevenlabs/audio_transcription/transformation.py +++ b/litellm/llms/elevenlabs/audio_transcription/transformation.py @@ -160,7 +160,7 @@ class ElevenLabsAudioTranscriptionConfig(BaseAudioTranscriptionConfig): return response except Exception as e: - raise ValueError(f"Error transforming ElevenLabs response: {e}\nResponse: {raw_response.text}") + raise ValueError(f"Error transforming ElevenLabs response: {e}") from None def get_complete_url( self, diff --git a/litellm/llms/gemini/audio_transcription/transformation.py b/litellm/llms/gemini/audio_transcription/transformation.py index 3457b22faa4..ca4953445ee 100644 --- a/litellm/llms/gemini/audio_transcription/transformation.py +++ b/litellm/llms/gemini/audio_transcription/transformation.py @@ -138,13 +138,13 @@ class GeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig): except ValueError: raise GeminiError( status_code=raw_response.status_code, - message=f"Received non-JSON response from Gemini Interactions API: {raw_response.text}", + message="Received non-JSON response from Gemini Interactions API", ) parsed: Final = GeminiTranscriptionInteractionResponse.model_validate(response_json) if parsed.status != "completed": raise GeminiError( status_code=raw_response.status_code, - message=f"Gemini transcription interaction did not complete (status={parsed.status}): {raw_response.text}", + message=f"Gemini transcription interaction did not complete (status={parsed.status})", ) text_contents: Final = tuple( content diff --git a/litellm/llms/mistral/audio_transcription/transformation.py b/litellm/llms/mistral/audio_transcription/transformation.py index 3eb9e122265..34200effb18 100644 --- a/litellm/llms/mistral/audio_transcription/transformation.py +++ b/litellm/llms/mistral/audio_transcription/transformation.py @@ -133,7 +133,7 @@ class MistralAudioTranscriptionConfig(BaseAudioTranscriptionConfig): response_json: Final = raw_response.json() except Exception: raise MistralAudioTranscriptionException( - message=raw_response.text, + message="Received non-JSON response from Mistral audio transcription", status_code=raw_response.status_code, headers=raw_response.headers, ) diff --git a/litellm/llms/openai/transcriptions/whisper_transformation.py b/litellm/llms/openai/transcriptions/whisper_transformation.py index eb4f1b6c9ce..947d6175122 100644 --- a/litellm/llms/openai/transcriptions/whisper_transformation.py +++ b/litellm/llms/openai/transcriptions/whisper_transformation.py @@ -141,7 +141,4 @@ class OpenAIWhisperAudioTranscriptionConfig(BaseAudioTranscriptionConfig): if any(key in raw_response_json for key in TranscriptionResponse.model_fields): return TranscriptionResponse(**raw_response_json) else: - raise ValueError( - "Invalid response format. Received response does not match the expected format. Got: ", - raw_response_json, - ) + raise ValueError("Invalid response format. Received response does not match the expected format.") diff --git a/litellm/llms/ovhcloud/audio_transcription/transformation.py b/litellm/llms/ovhcloud/audio_transcription/transformation.py index dd61bfb0c13..50bed1fef78 100644 --- a/litellm/llms/ovhcloud/audio_transcription/transformation.py +++ b/litellm/llms/ovhcloud/audio_transcription/transformation.py @@ -144,7 +144,7 @@ class OVHCloudAudioTranscriptionConfig(BaseAudioTranscriptionConfig): response_json: Final = raw_response.json() except Exception: raise OVHCloudException( - message=raw_response.text, + message="Received non-JSON response from OVHCloud audio transcription", status_code=raw_response.status_code, headers=raw_response.headers, ) diff --git a/litellm/llms/scaleway/audio_transcription/transformation.py b/litellm/llms/scaleway/audio_transcription/transformation.py index 8913cc362b3..2cc3ac17073 100644 --- a/litellm/llms/scaleway/audio_transcription/transformation.py +++ b/litellm/llms/scaleway/audio_transcription/transformation.py @@ -134,7 +134,7 @@ class ScalewayAudioTranscriptionConfig(BaseAudioTranscriptionConfig): response_json: Final = raw_response.json() except Exception: raise ScalewayAudioTranscriptionException( - message=raw_response.text, + message="Received non-JSON response from Scaleway audio transcription", status_code=raw_response.status_code, headers=raw_response.headers, ) diff --git a/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py b/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py index 26de755d5ca..09a09ef96dc 100644 --- a/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py +++ b/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py @@ -183,7 +183,7 @@ class VertexGeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexB except ValueError: raise VertexAIError( status_code=raw_response.status_code, - message=f"Received non-JSON response from Vertex AI Gemini transcription: {raw_response.text}", + message="Received non-JSON response from Vertex AI Gemini transcription", ) parsed: Final = VertexGeminiTranscriptionResponse.model_validate(response_json) texts: Final = tuple( diff --git a/litellm/llms/vertex_ai/audio_transcription/transformation.py b/litellm/llms/vertex_ai/audio_transcription/transformation.py index 4fdf93a6307..017d7ff7b2f 100644 --- a/litellm/llms/vertex_ai/audio_transcription/transformation.py +++ b/litellm/llms/vertex_ai/audio_transcription/transformation.py @@ -172,7 +172,7 @@ class VertexAIAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexBase) except ValueError: raise VertexAIError( status_code=raw_response.status_code, - message=f"Received non-JSON response from Google Speech-to-Text: {raw_response.text}", + message="Received non-JSON response from Google Speech-to-Text", ) parsed: Final = VertexSpeechToTextRecognizeResponse.model_validate(response_json) transcripts: Final = tuple( diff --git a/litellm/llms/watsonx/audio_transcription/transformation.py b/litellm/llms/watsonx/audio_transcription/transformation.py index 35fda05c028..b160ee65802 100644 --- a/litellm/llms/watsonx/audio_transcription/transformation.py +++ b/litellm/llms/watsonx/audio_transcription/transformation.py @@ -175,7 +175,7 @@ class IBMWatsonXAudioTranscriptionConfig(IBMWatsonXMixin, OpenAIWhisperAudioTran try: raw_response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: - raise ValueError(f"Error transforming response to json: {e}\nResponse: {raw_response.text}") + raise ValueError(f"Error transforming response to json: {e}") # Extract only valid fields for TranscriptionResponse.__init__() # TranscriptionResponse only accepts 'text' and 'usage' in __init__() @@ -191,10 +191,7 @@ class IBMWatsonXAudioTranscriptionConfig(IBMWatsonXMixin, OpenAIWhisperAudioTran response_kwargs["usage"] = usage if not response_kwargs: - raise ValueError( - "Invalid response format. Received response does not match the expected format. Got: ", - response_object, - ) + raise ValueError("Invalid response format. Received response does not match the expected format.") response: Final = TranscriptionResponse(**response_kwargs) diff --git a/litellm/llms/xai/audio_transcription/transformation.py b/litellm/llms/xai/audio_transcription/transformation.py index e29c3d9334a..c212dce26fe 100644 --- a/litellm/llms/xai/audio_transcription/transformation.py +++ b/litellm/llms/xai/audio_transcription/transformation.py @@ -39,7 +39,7 @@ class _XAISttWord(LiteLLMBaseModel): class _XAISttResponse(LiteLLMBaseModel): - model_config = ConfigDict(extra="allow") + model_config = ConfigDict(extra="allow", hide_input_in_errors=True) text: str = "" language: str = "unknown" duration: float | None = None diff --git a/litellm/types/llms/gemini_audio_transcription.py b/litellm/types/llms/gemini_audio_transcription.py index c1cf3452945..59dd15aa13e 100644 --- a/litellm/types/llms/gemini_audio_transcription.py +++ b/litellm/types/llms/gemini_audio_transcription.py @@ -75,7 +75,7 @@ class GeminiTranscriptionUsage(LiteLLMBaseModel): class GeminiTranscriptionInteractionResponse(LiteLLMBaseModel): - model_config = ConfigDict(extra="ignore") + model_config = ConfigDict(extra="ignore", hide_input_in_errors=True) id: str | None = None status: str | None = None diff --git a/litellm/types/llms/vertex_ai_gemini_transcription.py b/litellm/types/llms/vertex_ai_gemini_transcription.py index e78aafaca9f..d7bc136bcd1 100644 --- a/litellm/types/llms/vertex_ai_gemini_transcription.py +++ b/litellm/types/llms/vertex_ai_gemini_transcription.py @@ -68,7 +68,7 @@ class VertexGeminiTranscriptionUsageMetadata(LiteLLMBaseModel): class VertexGeminiTranscriptionResponse(LiteLLMBaseModel): - model_config = ConfigDict(extra="ignore") + model_config = ConfigDict(extra="ignore", hide_input_in_errors=True) candidates: tuple[VertexGeminiTranscriptionCandidate, ...] = () usageMetadata: VertexGeminiTranscriptionUsageMetadata | None = None diff --git a/litellm/types/llms/vertex_ai_speech_to_text.py b/litellm/types/llms/vertex_ai_speech_to_text.py index f3d343d13ce..3dea7610150 100644 --- a/litellm/types/llms/vertex_ai_speech_to_text.py +++ b/litellm/types/llms/vertex_ai_speech_to_text.py @@ -40,6 +40,8 @@ class VertexSpeechToTextResponseMetadata(LiteLLMBaseModel): class VertexSpeechToTextRecognizeResponse(LiteLLMBaseModel): + model_config = ConfigDict(hide_input_in_errors=True) + results: list[VertexSpeechToTextResult] = Field(default=[]) metadata: VertexSpeechToTextResponseMetadata | None = None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index ca434154f68..7c137b72c3e 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2989,6 +2989,8 @@ _DETECTED_LANGUAGES: Final = TypeAdapter(tuple[TranscriptionDetectedLanguage, .. class TranscriptionResponse(OpenAIObject): + model_config = ConfigDict(hide_input_in_errors=True) + text: str | None = None usage: TranscriptionUsageDurationObject | TranscriptionUsageTokensObject | None = None languages: Sequence[TranscriptionDetectedLanguage] | None = None diff --git a/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py b/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py index 3e197740e98..c41073e087f 100644 --- a/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py +++ b/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py @@ -2814,6 +2814,24 @@ class TestConvertToModelResponseObjectAudioTranscription: assert result.usage.output_tokens == 5 assert result.usage.input_token_details.audio_tokens == 4 + def test_parse_error_keeps_the_transcript_out_of_the_exception(self): + import traceback + + from litellm.types.utils import TranscriptionResponse + + transcript: Final = "CANARYTRANSCRIPT customer account ending 2750" + + with pytest.raises(Exception, match="Invalid response object") as exc_info: + convert_to_model_response_object( + response_object={"text": transcript, "usage": {"type": "duration", "seconds": "not-a-number"}}, + model_response_object=TranscriptionResponse(), + response_type="audio_transcription", + ) + + rendered: Final = "".join(traceback.format_exception(exc_info.value)) + assert "received_args=" in rendered + assert "CANARYTRANSCRIPT" not in rendered + class TestConvertToModelResponseObjectRerank: def test_basic_rerank(self): from litellm.types.utils import RerankResponse diff --git a/tests/unit/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py b/tests/unit/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py index 0951ba6ba92..d298142fe4b 100644 --- a/tests/unit/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py +++ b/tests/unit/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py @@ -1,11 +1,13 @@ import io import os import pathlib +import traceback from typing import Final from unittest.mock import MagicMock import httpx import pytest +from pydantic import ValidationError from litellm.llms.base_llm.audio_transcription.transformation import AudioTranscriptionRequestData from litellm.llms.deepgram.audio_transcription.transformation import ( @@ -588,31 +590,33 @@ def test_transform_response_maps_words_to_openai_word_timestamps(words: object, [{"word": "hello", "start": 0.0}], ], ) -def test_transform_response_wraps_malformed_words_with_the_raw_body(words: object): +def test_transform_response_keeps_the_transcript_out_of_malformed_words_errors(words: object): raw_response = httpx.Response(200, json=_deepgram_payload({"transcript": "hello", "words": words}, {})) with pytest.raises(ValueError, match="Error transforming Deepgram response: ") as exc_info: DeepgramAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) - assert str(exc_info.value).endswith(f"\nResponse: {raw_response.text}") + assert "hello" not in "".join(traceback.format_exception(exc_info.value)) @pytest.mark.parametrize( - ("payload", "error"), + ("payload", "failing_lookup_error"), [ - ({"results": None}, "'NoneType' object is not subscriptable"), - (_deepgram_payload({"transcript": "hello", "words": 5}, {}), "object of type 'int' has no len()"), - ( - {**_deepgram_payload({"transcript": "hello"}, {}), "metadata": None}, - "'NoneType' object is not subscriptable", - ), + ({"results": None}, TypeError), + (_deepgram_payload({"transcript": "hello", "words": 5}, {}), TypeError), + ({**_deepgram_payload({"transcript": "hello"}, {}), "metadata": None}, TypeError), + (_deepgram_payload({"transcript": ["hello"]}, {}), ValidationError), ], ) -def test_transform_response_wraps_the_error_of_the_failing_lookup_in_a_malformed_body(payload: object, error: str): +def test_transform_response_wraps_the_error_of_the_failing_lookup_without_the_body( + payload: object, failing_lookup_error: type[Exception] +): raw_response: Final = httpx.Response(200, json=payload) with pytest.raises(ValueError, match="Error transforming Deepgram response: ") as exc_info: DeepgramAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) - assert exc_info.type is ValueError - assert str(exc_info.value) == f"Error transforming Deepgram response: {error}\nResponse: {raw_response.text}" + failing_lookup: Final = exc_info.value.__context__ + assert isinstance(failing_lookup, failing_lookup_error) + assert str(exc_info.value) == f"Error transforming Deepgram response: {failing_lookup}" + assert "hello" not in "".join(traceback.format_exception(exc_info.value)) diff --git a/tests/unit/llms/elevenlabs/audio_transcription/test_transformation.py b/tests/unit/llms/elevenlabs/audio_transcription/test_transformation.py index f55644eccae..2f5754b3765 100644 --- a/tests/unit/llms/elevenlabs/audio_transcription/test_transformation.py +++ b/tests/unit/llms/elevenlabs/audio_transcription/test_transformation.py @@ -1,9 +1,11 @@ +import traceback from collections.abc import Iterator from typing import Final import httpx import pytest import respx +from pydantic import ValidationError import litellm from litellm.caching.llm_caching_handler import LLMClientCache @@ -109,24 +111,35 @@ def test_transform_audio_transcription_response_without_words_key_has_no_word_li @pytest.mark.parametrize( "payload", [ - ["not", "an", "object"], - "plain text", - 7, - {"text": ["not", "text"]}, - {"text": "t", "words": None}, - {"text": "t", "words": 7}, - {"text": "t", "words": "not a list"}, - {"text": "t", "words": ["not an object"]}, - {"text": "t", "words": [{"type": "word", "text": "ok"}, None]}, + ["sensitive", "an", "object"], + "sensitive text", + {"text": ["sensitive", "text"]}, + {"text": "sensitive", "words": None}, + {"text": "sensitive", "words": 7}, + {"text": "sensitive", "words": "not a list"}, + {"text": "sensitive", "words": ["not an object"]}, + {"text": "sensitive", "words": [{"type": "word", "text": "ok"}, None]}, ], ) -def test_transform_audio_transcription_response_wraps_malformed_payloads_with_the_raw_body(payload: object): +def test_transform_audio_transcription_response_keeps_the_body_out_of_malformed_payload_errors(payload: object): raw_response = httpx.Response(200, json=payload) with pytest.raises(ValueError, match="Error transforming ElevenLabs response: ") as exc_info: ElevenLabsAudioTranscriptionConfig().transform_audio_transcription_response(raw_response=raw_response) - assert str(exc_info.value).endswith(f"\nResponse: {raw_response.text}") + assert "sensitive" not in "".join(traceback.format_exception(exc_info.value)) + + +def test_transform_audio_transcription_response_names_the_failing_lookup_of_a_malformed_payload(): + raw_response = httpx.Response(200, json={"text": "sensitive", "words": [None]}) + + with pytest.raises(ValueError, match="Error transforming ElevenLabs response: ") as exc_info: + ElevenLabsAudioTranscriptionConfig().transform_audio_transcription_response(raw_response=raw_response) + + failing_lookup: Final = exc_info.value.__context__ + assert isinstance(failing_lookup, ValidationError) + assert str(exc_info.value) == f"Error transforming ElevenLabs response: {failing_lookup}" + assert "sensitive" not in "".join(traceback.format_exception(exc_info.value)) class TestElevenLabsAudioTranscription: diff --git a/tests/unit/llms/gemini/audio_transcription/test_gemini_audio_transcription_transformation.py b/tests/unit/llms/gemini/audio_transcription/test_gemini_audio_transcription_transformation.py index 08084c8fac0..a9ef468525f 100644 --- a/tests/unit/llms/gemini/audio_transcription/test_gemini_audio_transcription_transformation.py +++ b/tests/unit/llms/gemini/audio_transcription/test_gemini_audio_transcription_transformation.py @@ -1,8 +1,10 @@ import base64 import json +import traceback import httpx import pytest +import respx import litellm from litellm.llms.gemini.audio_transcription.transformation import ( @@ -241,6 +243,21 @@ class TestTransformResponse: with pytest.raises(GeminiError, match="non-JSON"): config.transform_audio_transcription_response(raw) + def test_non_completed_status_error_omits_response_body(self, config): + payload = json.loads(json.dumps(COMPLETED_RESPONSE)) + payload["status"] = "incomplete" + payload["steps"][0]["content"][0]["text"] = "sensitive transcript text" + with pytest.raises(GeminiError) as excinfo: + config.transform_audio_transcription_response(make_response(payload)) + assert "status=incomplete" in str(excinfo.value) + assert "sensitive transcript text" not in str(excinfo.value) + + def test_non_json_error_omits_response_body(self, config): + raw = httpx.Response(200, text="sensitive body", request=httpx.Request("POST", "https://example.test")) + with pytest.raises(GeminiError) as excinfo: + config.transform_audio_transcription_response(raw) + assert "sensitive body" not in str(excinfo.value) + def test_word_without_offsets_survives(self, config): payload = json.loads(json.dumps(COMPLETED_RESPONSE)) payload["steps"][0]["content"][0]["annotations"] = [{"type": "word_info", "text": "Hello"}] @@ -249,6 +266,66 @@ class TestTransformResponse: assert response.get("duration") is None + +class TestNonCompletedInteractionThroughRouter: + @pytest.fixture + def router(self, monkeypatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + return litellm.Router( + model_list=[ + { + "model_name": "transcribe", + "litellm_params": {"model": "gemini/gemini-3.5-transcribe", "api_key": "test-gemini-key"}, + } + ], + num_retries=2, + ) + + def _interaction(self, status): + payload = json.loads(json.dumps(COMPLETED_RESPONSE)) + payload["status"] = status + payload["steps"][0]["content"][0]["text"] = "sensitive transcript text" + return payload + + @pytest.mark.asyncio + @respx.mock + async def test_retry_recovers_an_incomplete_interaction(self, router): + route = respx.post("https://generativelanguage.googleapis.com/v1beta/interactions").mock( + side_effect=[ + httpx.Response(200, json=self._interaction("incomplete")), + httpx.Response(200, json=COMPLETED_RESPONSE), + ] + ) + response = await router.atranscription(model="transcribe", file=("call.wav", AUDIO_BYTES, "audio/wav")) + assert route.call_count == 2 + assert response.text == COMPLETED_RESPONSE["steps"][0]["content"][0]["text"] + + @pytest.mark.asyncio + @respx.mock + @pytest.mark.parametrize("status", ["incomplete", "failed", "cancelled", "in_progress", None]) + async def test_non_completed_interaction_retries_then_fails_without_transcript(self, router, status): + route = respx.post("https://generativelanguage.googleapis.com/v1beta/interactions").mock( + return_value=httpx.Response(200, json=self._interaction(status)) + ) + with pytest.raises(litellm.APIConnectionError) as excinfo: + await router.atranscription(model="transcribe", file=("call.wav", AUDIO_BYTES, "audio/wav")) + assert route.call_count == 3 + assert f"status={status}" in str(excinfo.value) + assert "sensitive transcript text" not in str(excinfo.value) + + @pytest.mark.asyncio + @respx.mock + async def test_response_field_of_unexpected_type_keeps_transcript_out_of_the_error(self, router): + payload = self._interaction("incomplete") + payload["steps"][0]["content"] = "sensitive transcript text" + respx.post("https://generativelanguage.googleapis.com/v1beta/interactions").mock( + return_value=httpx.Response(200, json=payload) + ) + with pytest.raises(litellm.APIConnectionError) as excinfo: + await router.atranscription(model="transcribe", file=("call.wav", AUDIO_BYTES, "audio/wav")) + assert "sensitive transcript text" not in "".join(traceback.format_exception(excinfo.value)) + + class TestSubtitleSynthesisThroughHandler: def _transform(self, config, response_format): from unittest.mock import Mock diff --git a/tests/unit/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py b/tests/unit/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py index b475cf7e3ff..494b1fe34ad 100644 --- a/tests/unit/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py +++ b/tests/unit/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py @@ -3,12 +3,15 @@ from unittest.mock import MagicMock import httpx import litellm +import pytest +from pydantic import ValidationError from litellm.llms.base_llm.audio_transcription.transformation import ( BaseAudioTranscriptionConfig, ) from litellm.llms.mistral.audio_transcription.transformation import ( MistralAudioTranscriptionConfig, + MistralAudioTranscriptionException, ) from litellm.types.utils import TranscriptionResponse from litellm.utils import ProviderConfigManager @@ -191,3 +194,22 @@ def test_mistral_audio_transcription_response_transform_empty(): assert isinstance(response, TranscriptionResponse) assert response.text == "" + + +def test_mistral_audio_transcription_response_transform_keeps_non_json_body_out_of_the_error(): + raw_response = httpx.Response(200, text='{"text": "sensitive') + + with pytest.raises(MistralAudioTranscriptionException, match="non-JSON") as exc_info: + MistralAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) + + assert "sensitive" not in str(exc_info.value) + + +@pytest.mark.parametrize("text", [["sensitive"], {"value": "sensitive"}]) +def test_mistral_audio_transcription_response_transform_keeps_a_malformed_text_field_out_of_the_error(text: object): + raw_response = httpx.Response(200, json={"text": text, "segments": [{"text": "sensitive"}]}) + + with pytest.raises(ValidationError) as exc_info: + MistralAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) + + assert "sensitive" not in str(exc_info.value) diff --git a/tests/unit/llms/openai/transcriptions/test_whisper_transformation.py b/tests/unit/llms/openai/transcriptions/test_whisper_transformation.py index 2dc24b1b313..339d62ef7ea 100644 --- a/tests/unit/llms/openai/transcriptions/test_whisper_transformation.py +++ b/tests/unit/llms/openai/transcriptions/test_whisper_transformation.py @@ -5,8 +5,10 @@ and transform_audio_transcription_response. import io import json +import traceback from unittest.mock import MagicMock +import httpx import pytest from litellm.llms.openai.transcriptions.whisper_transformation import ( @@ -104,3 +106,10 @@ class TestWhisperTransformResponse: is_json=False, ) ) + + def test_json_object_without_transcription_fields_keeps_the_body_out_of_the_error(self): + config = OpenAIWhisperAudioTranscriptionConfig() + raw_response = httpx.Response(200, json={"segments": [{"text": "sensitive"}]}) + with pytest.raises(ValueError, match="Invalid response format") as exc_info: + config.transform_audio_transcription_response(raw_response) + assert "sensitive" not in "".join(traceback.format_exception(exc_info.value)) diff --git a/tests/unit/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py b/tests/unit/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py index 422658932e6..ed8ec88e209 100644 --- a/tests/unit/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py +++ b/tests/unit/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py @@ -3,6 +3,7 @@ import pytest from pydantic import ValidationError from litellm.llms.ovhcloud.audio_transcription.transformation import OVHCloudAudioTranscriptionConfig +from litellm.llms.ovhcloud.utils import OVHCloudException from litellm.types.utils import TranscriptionResponse @@ -98,3 +99,12 @@ def test_transform_audio_transcription_response_rejects_non_object_bodies(payloa _transform(payload) assert "spoken secret" not in str(exc_info.value) + + +def test_transform_audio_transcription_response_keeps_a_non_json_body_out_of_the_error(): + raw_response = httpx.Response(200, text='{"text": "spoken secret') + + with pytest.raises(OVHCloudException, match="non-JSON") as exc_info: + OVHCloudAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) + + assert "spoken secret" not in str(exc_info.value) diff --git a/tests/unit/llms/scaleway/test_scaleway_audio_transcription_transformation.py b/tests/unit/llms/scaleway/test_scaleway_audio_transcription_transformation.py index f313f8bd873..16e60a15539 100644 --- a/tests/unit/llms/scaleway/test_scaleway_audio_transcription_transformation.py +++ b/tests/unit/llms/scaleway/test_scaleway_audio_transcription_transformation.py @@ -204,6 +204,17 @@ def test_scaleway_transform_response_raises_typed_exception_on_non_json(): ) +def test_scaleway_transform_response_keeps_non_json_body_out_of_the_error(): + raw_response = httpx.Response( + 200, text='{"text": "sensitive', headers={"content-type": "application/json"} + ) + + with pytest.raises(ScalewayAudioTranscriptionException, match="non-JSON") as exc_info: + ScalewayAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) + + assert "sensitive" not in str(exc_info.value) + + def test_scaleway_transform_response_returns_plain_text_for_non_json_content_type(): """When Scaleway responds with text/srt/vtt (response_format="text" etc.), the content-type is not application/json — return the body as plain text diff --git a/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_audio_transcription_transformation.py b/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_audio_transcription_transformation.py index 5898d933941..02b7c0ae71a 100644 --- a/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_audio_transcription_transformation.py +++ b/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_audio_transcription_transformation.py @@ -5,7 +5,7 @@ from urllib.parse import urlparse import httpx import pytest - +from pydantic import ValidationError import litellm from litellm.llms.vertex_ai.audio_transcription.transformation import ( @@ -208,6 +208,18 @@ class TestTransformResponse: response = config.transform_audio_transcription_response(raw_response) assert response.text == "" + def test_non_json_error_omits_response_body(self, config): + raw_response = httpx.Response(status_code=200, text='{"results": [{"alternatives": [{"transcript": "sensitive') + with pytest.raises(VertexAIError, match="non-JSON") as excinfo: + config.transform_audio_transcription_response(raw_response) + assert "sensitive" not in str(excinfo.value) + + def test_field_of_unexpected_type_omits_response_body(self, config): + raw_response = httpx.Response(status_code=200, json={"results": "sensitive transcript"}) + with pytest.raises(ValidationError) as excinfo: + config.transform_audio_transcription_response(raw_response) + assert "sensitive" not in str(excinfo.value) + def test_fractional_billed_duration(self, config): raw_response = httpx.Response( status_code=200, diff --git a/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py b/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py index 82ea034f91b..532e013c3d7 100644 --- a/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py +++ b/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py @@ -4,6 +4,7 @@ import os import httpx import pytest +from pydantic import ValidationError import litellm from litellm.llms.vertex_ai.audio_transcription.gemini_transcribe_transformation import ( @@ -255,6 +256,20 @@ class TestTransformResponse: with pytest.raises(VertexAIError, match="non-JSON"): config.transform_audio_transcription_response(raw_response) + def test_non_json_error_omits_response_body(self, config): + raw_response = httpx.Response(status_code=200, text='{"candidates": [{"content": {"parts": [{"text": "sensitive') + with pytest.raises(VertexAIError) as excinfo: + config.transform_audio_transcription_response(raw_response) + assert "sensitive" not in str(excinfo.value) + + def test_field_of_unexpected_type_omits_response_body(self, config): + raw_response = httpx.Response( + status_code=200, json={"candidates": [{"content": {"parts": "sensitive transcript"}}]} + ) + with pytest.raises(ValidationError) as excinfo: + config.transform_audio_transcription_response(raw_response) + assert "sensitive" not in str(excinfo.value) + class TestValidateEnvironment: def test_sets_oauth_headers(self): diff --git a/tests/unit/llms/watsonx/audio_transcription/test_watsonx_audio_transcription_transformation.py b/tests/unit/llms/watsonx/audio_transcription/test_watsonx_audio_transcription_transformation.py index b6e5007c456..c9f6f566ad6 100644 --- a/tests/unit/llms/watsonx/audio_transcription/test_watsonx_audio_transcription_transformation.py +++ b/tests/unit/llms/watsonx/audio_transcription/test_watsonx_audio_transcription_transformation.py @@ -4,6 +4,7 @@ Tests for IBM WatsonX Audio Transcription. Validates the WatsonX transcription response transformation. """ +import traceback from unittest.mock import MagicMock import httpx @@ -120,17 +121,21 @@ def test_transform_audio_transcription_response_copies_every_field_except_model( assert {key: response[key] for key in expected_extras} == expected_extras -@pytest.mark.parametrize("payload", [{}, {"model": "whisper"}, {"duration": 2.0}, {"text": None, "usage": None}]) -def test_transform_audio_transcription_response_without_text_or_usage_reports_the_body( +@pytest.mark.parametrize( + "payload", + [ + {"model": "sensitive"}, + {"duration": 2.0, "segments": [{"text": "sensitive"}]}, + {"text": None, "usage": None, "note": "sensitive"}, + ], +) +def test_transform_audio_transcription_response_without_text_or_usage_keeps_the_body_out_of_the_error( payload: dict[str, object], ) -> None: with pytest.raises(ValueError, match="Invalid response format") as exc_info: _transform_transcription_response(payload) - assert exc_info.value.args == ( - "Invalid response format. Received response does not match the expected format. Got: ", - payload, - ) + assert "sensitive" not in "".join(traceback.format_exception(exc_info.value)) @pytest.mark.parametrize("payload", [["not", "an", "object"], "plain text", 7, True]) @@ -141,3 +146,12 @@ def test_transform_audio_transcription_response_rejects_non_object_bodies_withou _transform_transcription_response(payload) assert "input_value" not in str(exc_info.value) + + +def test_transform_audio_transcription_response_keeps_a_non_json_body_out_of_the_error() -> None: + raw_response = httpx.Response(200, text='{"text": "sensitive') + + with pytest.raises(ValueError, match="Error transforming response to json") as exc_info: + IBMWatsonXAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) + + assert "sensitive" not in str(exc_info.value) diff --git a/tests/unit/llms/xai/test_xai_audio_transcription_transformation.py b/tests/unit/llms/xai/test_xai_audio_transcription_transformation.py index e2e3fc3d4dc..3505116819b 100644 --- a/tests/unit/llms/xai/test_xai_audio_transcription_transformation.py +++ b/tests/unit/llms/xai/test_xai_audio_transcription_transformation.py @@ -1,3 +1,5 @@ +import traceback + import httpx import pytest @@ -146,6 +148,23 @@ def test_transform_response_raises_on_error_status(): assert "Incorrect API key provided" in exc.value.message +@pytest.mark.parametrize( + "content", + [ + b'{"text": ["sensitive"]}', + b'{"text": "ok", "words": [{"text": ["sensitive"]}]}', + b'{"text": "sensitive', + ], +) +def test_transform_response_keeps_a_malformed_body_out_of_the_error(content): + raw = httpx.Response(200, content=content, request=httpx.Request("POST", "https://api.x.ai/v1/stt")) + + with pytest.raises(XAIAudioTranscriptionError, match="Error parsing xAI response") as exc: + CONFIG.transform_audio_transcription_response(raw_response=raw) + + assert "sensitive" not in "".join(traceback.format_exception(exc.value)) + + def test_transcription_routes_to_xai_stt(monkeypatch): monkeypatch.delenv("XAI_API_KEY", raising=False) monkeypatch.setattr(litellm, "xai_key", None)