diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 9fb4ad27d75..0fed2cb2fce 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -1072,9 +1072,10 @@ def convert_to_model_response_object( if isinstance(e, APIError): raise + hide_reply: Final = response_type == "audio_transcription" received_args: Final = dict( - response_object=response_object, - model_response_object=model_response_object, + response_object="" if hide_reply else response_object, + model_response_object="" if hide_reply else model_response_object, response_type=response_type, stream=stream, start_time=start_time, diff --git a/litellm/llms/deepgram/audio_transcription/transformation.py b/litellm/llms/deepgram/audio_transcription/transformation.py index 1c8a056116f..2c8b22459bd 100644 --- a/litellm/llms/deepgram/audio_transcription/transformation.py +++ b/litellm/llms/deepgram/audio_transcription/transformation.py @@ -158,7 +158,7 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig): return response except Exception as e: - raise ValueError(f"Error transforming Deepgram response: {e}\nResponse: {raw_response.text}") + raise ValueError(f"Error transforming Deepgram response: {e}") from None def _reconstruct_diarized_transcript(self, words: list) -> str: """ diff --git a/litellm/llms/elevenlabs/audio_transcription/transformation.py b/litellm/llms/elevenlabs/audio_transcription/transformation.py index 3dc42f35d3c..b09aa97f6fe 100644 --- a/litellm/llms/elevenlabs/audio_transcription/transformation.py +++ b/litellm/llms/elevenlabs/audio_transcription/transformation.py @@ -160,7 +160,7 @@ class ElevenLabsAudioTranscriptionConfig(BaseAudioTranscriptionConfig): return response except Exception as e: - raise ValueError(f"Error transforming ElevenLabs response: {e}\nResponse: {raw_response.text}") + raise ValueError(f"Error transforming ElevenLabs response: {e}") from None def get_complete_url( self, diff --git a/litellm/llms/gemini/audio_transcription/transformation.py b/litellm/llms/gemini/audio_transcription/transformation.py index 3457b22faa4..ca4953445ee 100644 --- a/litellm/llms/gemini/audio_transcription/transformation.py +++ b/litellm/llms/gemini/audio_transcription/transformation.py @@ -138,13 +138,13 @@ class GeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig): except ValueError: raise GeminiError( status_code=raw_response.status_code, - message=f"Received non-JSON response from Gemini Interactions API: {raw_response.text}", + message="Received non-JSON response from Gemini Interactions API", ) parsed: Final = GeminiTranscriptionInteractionResponse.model_validate(response_json) if parsed.status != "completed": raise GeminiError( status_code=raw_response.status_code, - message=f"Gemini transcription interaction did not complete (status={parsed.status}): {raw_response.text}", + message=f"Gemini transcription interaction did not complete (status={parsed.status})", ) text_contents: Final = tuple( content diff --git a/litellm/llms/mistral/audio_transcription/transformation.py b/litellm/llms/mistral/audio_transcription/transformation.py index 3eb9e122265..34200effb18 100644 --- a/litellm/llms/mistral/audio_transcription/transformation.py +++ b/litellm/llms/mistral/audio_transcription/transformation.py @@ -133,7 +133,7 @@ class MistralAudioTranscriptionConfig(BaseAudioTranscriptionConfig): response_json: Final = raw_response.json() except Exception: raise MistralAudioTranscriptionException( - message=raw_response.text, + message="Received non-JSON response from Mistral audio transcription", status_code=raw_response.status_code, headers=raw_response.headers, ) diff --git a/litellm/llms/openai/transcriptions/whisper_transformation.py b/litellm/llms/openai/transcriptions/whisper_transformation.py index eb4f1b6c9ce..947d6175122 100644 --- a/litellm/llms/openai/transcriptions/whisper_transformation.py +++ b/litellm/llms/openai/transcriptions/whisper_transformation.py @@ -141,7 +141,4 @@ class OpenAIWhisperAudioTranscriptionConfig(BaseAudioTranscriptionConfig): if any(key in raw_response_json for key in TranscriptionResponse.model_fields): return TranscriptionResponse(**raw_response_json) else: - raise ValueError( - "Invalid response format. Received response does not match the expected format. Got: ", - raw_response_json, - ) + raise ValueError("Invalid response format. Received response does not match the expected format.") diff --git a/litellm/llms/ovhcloud/audio_transcription/transformation.py b/litellm/llms/ovhcloud/audio_transcription/transformation.py index dd61bfb0c13..50bed1fef78 100644 --- a/litellm/llms/ovhcloud/audio_transcription/transformation.py +++ b/litellm/llms/ovhcloud/audio_transcription/transformation.py @@ -144,7 +144,7 @@ class OVHCloudAudioTranscriptionConfig(BaseAudioTranscriptionConfig): response_json: Final = raw_response.json() except Exception: raise OVHCloudException( - message=raw_response.text, + message="Received non-JSON response from OVHCloud audio transcription", status_code=raw_response.status_code, headers=raw_response.headers, ) diff --git a/litellm/llms/scaleway/audio_transcription/transformation.py b/litellm/llms/scaleway/audio_transcription/transformation.py index 8913cc362b3..2cc3ac17073 100644 --- a/litellm/llms/scaleway/audio_transcription/transformation.py +++ b/litellm/llms/scaleway/audio_transcription/transformation.py @@ -134,7 +134,7 @@ class ScalewayAudioTranscriptionConfig(BaseAudioTranscriptionConfig): response_json: Final = raw_response.json() except Exception: raise ScalewayAudioTranscriptionException( - message=raw_response.text, + message="Received non-JSON response from Scaleway audio transcription", status_code=raw_response.status_code, headers=raw_response.headers, ) diff --git a/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py b/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py index 26de755d5ca..09a09ef96dc 100644 --- a/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py +++ b/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py @@ -183,7 +183,7 @@ class VertexGeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexB except ValueError: raise VertexAIError( status_code=raw_response.status_code, - message=f"Received non-JSON response from Vertex AI Gemini transcription: {raw_response.text}", + message="Received non-JSON response from Vertex AI Gemini transcription", ) parsed: Final = VertexGeminiTranscriptionResponse.model_validate(response_json) texts: Final = tuple( diff --git a/litellm/llms/vertex_ai/audio_transcription/transformation.py b/litellm/llms/vertex_ai/audio_transcription/transformation.py index 4fdf93a6307..017d7ff7b2f 100644 --- a/litellm/llms/vertex_ai/audio_transcription/transformation.py +++ b/litellm/llms/vertex_ai/audio_transcription/transformation.py @@ -172,7 +172,7 @@ class VertexAIAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexBase) except ValueError: raise VertexAIError( status_code=raw_response.status_code, - message=f"Received non-JSON response from Google Speech-to-Text: {raw_response.text}", + message="Received non-JSON response from Google Speech-to-Text", ) parsed: Final = VertexSpeechToTextRecognizeResponse.model_validate(response_json) transcripts: Final = tuple( diff --git a/litellm/llms/watsonx/audio_transcription/transformation.py b/litellm/llms/watsonx/audio_transcription/transformation.py index 35fda05c028..b160ee65802 100644 --- a/litellm/llms/watsonx/audio_transcription/transformation.py +++ b/litellm/llms/watsonx/audio_transcription/transformation.py @@ -175,7 +175,7 @@ class IBMWatsonXAudioTranscriptionConfig(IBMWatsonXMixin, OpenAIWhisperAudioTran try: raw_response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: - raise ValueError(f"Error transforming response to json: {e}\nResponse: {raw_response.text}") + raise ValueError(f"Error transforming response to json: {e}") # Extract only valid fields for TranscriptionResponse.__init__() # TranscriptionResponse only accepts 'text' and 'usage' in __init__() @@ -191,10 +191,7 @@ class IBMWatsonXAudioTranscriptionConfig(IBMWatsonXMixin, OpenAIWhisperAudioTran response_kwargs["usage"] = usage if not response_kwargs: - raise ValueError( - "Invalid response format. Received response does not match the expected format. Got: ", - response_object, - ) + raise ValueError("Invalid response format. Received response does not match the expected format.") response: Final = TranscriptionResponse(**response_kwargs) diff --git a/litellm/llms/xai/audio_transcription/transformation.py b/litellm/llms/xai/audio_transcription/transformation.py index e29c3d9334a..c212dce26fe 100644 --- a/litellm/llms/xai/audio_transcription/transformation.py +++ b/litellm/llms/xai/audio_transcription/transformation.py @@ -39,7 +39,7 @@ class _XAISttWord(LiteLLMBaseModel): class _XAISttResponse(LiteLLMBaseModel): - model_config = ConfigDict(extra="allow") + model_config = ConfigDict(extra="allow", hide_input_in_errors=True) text: str = "" language: str = "unknown" duration: float | None = None diff --git a/litellm/types/llms/gemini_audio_transcription.py b/litellm/types/llms/gemini_audio_transcription.py index c1cf3452945..59dd15aa13e 100644 --- a/litellm/types/llms/gemini_audio_transcription.py +++ b/litellm/types/llms/gemini_audio_transcription.py @@ -75,7 +75,7 @@ class GeminiTranscriptionUsage(LiteLLMBaseModel): class GeminiTranscriptionInteractionResponse(LiteLLMBaseModel): - model_config = ConfigDict(extra="ignore") + model_config = ConfigDict(extra="ignore", hide_input_in_errors=True) id: str | None = None status: str | None = None diff --git a/litellm/types/llms/vertex_ai_gemini_transcription.py b/litellm/types/llms/vertex_ai_gemini_transcription.py index e78aafaca9f..d7bc136bcd1 100644 --- a/litellm/types/llms/vertex_ai_gemini_transcription.py +++ b/litellm/types/llms/vertex_ai_gemini_transcription.py @@ -68,7 +68,7 @@ class VertexGeminiTranscriptionUsageMetadata(LiteLLMBaseModel): class VertexGeminiTranscriptionResponse(LiteLLMBaseModel): - model_config = ConfigDict(extra="ignore") + model_config = ConfigDict(extra="ignore", hide_input_in_errors=True) candidates: tuple[VertexGeminiTranscriptionCandidate, ...] = () usageMetadata: VertexGeminiTranscriptionUsageMetadata | None = None diff --git a/litellm/types/llms/vertex_ai_speech_to_text.py b/litellm/types/llms/vertex_ai_speech_to_text.py index f3d343d13ce..3dea7610150 100644 --- a/litellm/types/llms/vertex_ai_speech_to_text.py +++ b/litellm/types/llms/vertex_ai_speech_to_text.py @@ -40,6 +40,8 @@ class VertexSpeechToTextResponseMetadata(LiteLLMBaseModel): class VertexSpeechToTextRecognizeResponse(LiteLLMBaseModel): + model_config = ConfigDict(hide_input_in_errors=True) + results: list[VertexSpeechToTextResult] = Field(default=[]) metadata: VertexSpeechToTextResponseMetadata | None = None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index ca434154f68..7c137b72c3e 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2989,6 +2989,8 @@ _DETECTED_LANGUAGES: Final = TypeAdapter(tuple[TranscriptionDetectedLanguage, .. class TranscriptionResponse(OpenAIObject): + model_config = ConfigDict(hide_input_in_errors=True) + text: str | None = None usage: TranscriptionUsageDurationObject | TranscriptionUsageTokensObject | None = None languages: Sequence[TranscriptionDetectedLanguage] | None = None diff --git a/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py b/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py index 3e197740e98..c41073e087f 100644 --- a/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py +++ b/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py @@ -2814,6 +2814,24 @@ class TestConvertToModelResponseObjectAudioTranscription: assert result.usage.output_tokens == 5 assert result.usage.input_token_details.audio_tokens == 4 + def test_parse_error_keeps_the_transcript_out_of_the_exception(self): + import traceback + + from litellm.types.utils import TranscriptionResponse + + transcript: Final = "CANARYTRANSCRIPT customer account ending 2750" + + with pytest.raises(Exception, match="Invalid response object") as exc_info: + convert_to_model_response_object( + response_object={"text": transcript, "usage": {"type": "duration", "seconds": "not-a-number"}}, + model_response_object=TranscriptionResponse(), + response_type="audio_transcription", + ) + + rendered: Final = "".join(traceback.format_exception(exc_info.value)) + assert "received_args=" in rendered + assert "CANARYTRANSCRIPT" not in rendered + class TestConvertToModelResponseObjectRerank: def test_basic_rerank(self): from litellm.types.utils import RerankResponse diff --git a/tests/unit/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py b/tests/unit/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py index 0951ba6ba92..d298142fe4b 100644 --- a/tests/unit/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py +++ b/tests/unit/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py @@ -1,11 +1,13 @@ import io import os import pathlib +import traceback from typing import Final from unittest.mock import MagicMock import httpx import pytest +from pydantic import ValidationError from litellm.llms.base_llm.audio_transcription.transformation import AudioTranscriptionRequestData from litellm.llms.deepgram.audio_transcription.transformation import ( @@ -588,31 +590,33 @@ def test_transform_response_maps_words_to_openai_word_timestamps(words: object, [{"word": "hello", "start": 0.0}], ], ) -def test_transform_response_wraps_malformed_words_with_the_raw_body(words: object): +def test_transform_response_keeps_the_transcript_out_of_malformed_words_errors(words: object): raw_response = httpx.Response(200, json=_deepgram_payload({"transcript": "hello", "words": words}, {})) with pytest.raises(ValueError, match="Error transforming Deepgram response: ") as exc_info: DeepgramAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) - assert str(exc_info.value).endswith(f"\nResponse: {raw_response.text}") + assert "hello" not in "".join(traceback.format_exception(exc_info.value)) @pytest.mark.parametrize( - ("payload", "error"), + ("payload", "failing_lookup_error"), [ - ({"results": None}, "'NoneType' object is not subscriptable"), - (_deepgram_payload({"transcript": "hello", "words": 5}, {}), "object of type 'int' has no len()"), - ( - {**_deepgram_payload({"transcript": "hello"}, {}), "metadata": None}, - "'NoneType' object is not subscriptable", - ), + ({"results": None}, TypeError), + (_deepgram_payload({"transcript": "hello", "words": 5}, {}), TypeError), + ({**_deepgram_payload({"transcript": "hello"}, {}), "metadata": None}, TypeError), + (_deepgram_payload({"transcript": ["hello"]}, {}), ValidationError), ], ) -def test_transform_response_wraps_the_error_of_the_failing_lookup_in_a_malformed_body(payload: object, error: str): +def test_transform_response_wraps_the_error_of_the_failing_lookup_without_the_body( + payload: object, failing_lookup_error: type[Exception] +): raw_response: Final = httpx.Response(200, json=payload) with pytest.raises(ValueError, match="Error transforming Deepgram response: ") as exc_info: DeepgramAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) - assert exc_info.type is ValueError - assert str(exc_info.value) == f"Error transforming Deepgram response: {error}\nResponse: {raw_response.text}" + failing_lookup: Final = exc_info.value.__context__ + assert isinstance(failing_lookup, failing_lookup_error) + assert str(exc_info.value) == f"Error transforming Deepgram response: {failing_lookup}" + assert "hello" not in "".join(traceback.format_exception(exc_info.value)) diff --git a/tests/unit/llms/elevenlabs/audio_transcription/test_transformation.py b/tests/unit/llms/elevenlabs/audio_transcription/test_transformation.py index f55644eccae..2f5754b3765 100644 --- a/tests/unit/llms/elevenlabs/audio_transcription/test_transformation.py +++ b/tests/unit/llms/elevenlabs/audio_transcription/test_transformation.py @@ -1,9 +1,11 @@ +import traceback from collections.abc import Iterator from typing import Final import httpx import pytest import respx +from pydantic import ValidationError import litellm from litellm.caching.llm_caching_handler import LLMClientCache @@ -109,24 +111,35 @@ def test_transform_audio_transcription_response_without_words_key_has_no_word_li @pytest.mark.parametrize( "payload", [ - ["not", "an", "object"], - "plain text", - 7, - {"text": ["not", "text"]}, - {"text": "t", "words": None}, - {"text": "t", "words": 7}, - {"text": "t", "words": "not a list"}, - {"text": "t", "words": ["not an object"]}, - {"text": "t", "words": [{"type": "word", "text": "ok"}, None]}, + ["sensitive", "an", "object"], + "sensitive text", + {"text": ["sensitive", "text"]}, + {"text": "sensitive", "words": None}, + {"text": "sensitive", "words": 7}, + {"text": "sensitive", "words": "not a list"}, + {"text": "sensitive", "words": ["not an object"]}, + {"text": "sensitive", "words": [{"type": "word", "text": "ok"}, None]}, ], ) -def test_transform_audio_transcription_response_wraps_malformed_payloads_with_the_raw_body(payload: object): +def test_transform_audio_transcription_response_keeps_the_body_out_of_malformed_payload_errors(payload: object): raw_response = httpx.Response(200, json=payload) with pytest.raises(ValueError, match="Error transforming ElevenLabs response: ") as exc_info: ElevenLabsAudioTranscriptionConfig().transform_audio_transcription_response(raw_response=raw_response) - assert str(exc_info.value).endswith(f"\nResponse: {raw_response.text}") + assert "sensitive" not in "".join(traceback.format_exception(exc_info.value)) + + +def test_transform_audio_transcription_response_names_the_failing_lookup_of_a_malformed_payload(): + raw_response = httpx.Response(200, json={"text": "sensitive", "words": [None]}) + + with pytest.raises(ValueError, match="Error transforming ElevenLabs response: ") as exc_info: + ElevenLabsAudioTranscriptionConfig().transform_audio_transcription_response(raw_response=raw_response) + + failing_lookup: Final = exc_info.value.__context__ + assert isinstance(failing_lookup, ValidationError) + assert str(exc_info.value) == f"Error transforming ElevenLabs response: {failing_lookup}" + assert "sensitive" not in "".join(traceback.format_exception(exc_info.value)) class TestElevenLabsAudioTranscription: diff --git a/tests/unit/llms/gemini/audio_transcription/test_gemini_audio_transcription_transformation.py b/tests/unit/llms/gemini/audio_transcription/test_gemini_audio_transcription_transformation.py index 08084c8fac0..a9ef468525f 100644 --- a/tests/unit/llms/gemini/audio_transcription/test_gemini_audio_transcription_transformation.py +++ b/tests/unit/llms/gemini/audio_transcription/test_gemini_audio_transcription_transformation.py @@ -1,8 +1,10 @@ import base64 import json +import traceback import httpx import pytest +import respx import litellm from litellm.llms.gemini.audio_transcription.transformation import ( @@ -241,6 +243,21 @@ class TestTransformResponse: with pytest.raises(GeminiError, match="non-JSON"): config.transform_audio_transcription_response(raw) + def test_non_completed_status_error_omits_response_body(self, config): + payload = json.loads(json.dumps(COMPLETED_RESPONSE)) + payload["status"] = "incomplete" + payload["steps"][0]["content"][0]["text"] = "sensitive transcript text" + with pytest.raises(GeminiError) as excinfo: + config.transform_audio_transcription_response(make_response(payload)) + assert "status=incomplete" in str(excinfo.value) + assert "sensitive transcript text" not in str(excinfo.value) + + def test_non_json_error_omits_response_body(self, config): + raw = httpx.Response(200, text="sensitive body", request=httpx.Request("POST", "https://example.test")) + with pytest.raises(GeminiError) as excinfo: + config.transform_audio_transcription_response(raw) + assert "sensitive body" not in str(excinfo.value) + def test_word_without_offsets_survives(self, config): payload = json.loads(json.dumps(COMPLETED_RESPONSE)) payload["steps"][0]["content"][0]["annotations"] = [{"type": "word_info", "text": "Hello"}] @@ -249,6 +266,66 @@ class TestTransformResponse: assert response.get("duration") is None + +class TestNonCompletedInteractionThroughRouter: + @pytest.fixture + def router(self, monkeypatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + return litellm.Router( + model_list=[ + { + "model_name": "transcribe", + "litellm_params": {"model": "gemini/gemini-3.5-transcribe", "api_key": "test-gemini-key"}, + } + ], + num_retries=2, + ) + + def _interaction(self, status): + payload = json.loads(json.dumps(COMPLETED_RESPONSE)) + payload["status"] = status + payload["steps"][0]["content"][0]["text"] = "sensitive transcript text" + return payload + + @pytest.mark.asyncio + @respx.mock + async def test_retry_recovers_an_incomplete_interaction(self, router): + route = respx.post("https://generativelanguage.googleapis.com/v1beta/interactions").mock( + side_effect=[ + httpx.Response(200, json=self._interaction("incomplete")), + httpx.Response(200, json=COMPLETED_RESPONSE), + ] + ) + response = await router.atranscription(model="transcribe", file=("call.wav", AUDIO_BYTES, "audio/wav")) + assert route.call_count == 2 + assert response.text == COMPLETED_RESPONSE["steps"][0]["content"][0]["text"] + + @pytest.mark.asyncio + @respx.mock + @pytest.mark.parametrize("status", ["incomplete", "failed", "cancelled", "in_progress", None]) + async def test_non_completed_interaction_retries_then_fails_without_transcript(self, router, status): + route = respx.post("https://generativelanguage.googleapis.com/v1beta/interactions").mock( + return_value=httpx.Response(200, json=self._interaction(status)) + ) + with pytest.raises(litellm.APIConnectionError) as excinfo: + await router.atranscription(model="transcribe", file=("call.wav", AUDIO_BYTES, "audio/wav")) + assert route.call_count == 3 + assert f"status={status}" in str(excinfo.value) + assert "sensitive transcript text" not in str(excinfo.value) + + @pytest.mark.asyncio + @respx.mock + async def test_response_field_of_unexpected_type_keeps_transcript_out_of_the_error(self, router): + payload = self._interaction("incomplete") + payload["steps"][0]["content"] = "sensitive transcript text" + respx.post("https://generativelanguage.googleapis.com/v1beta/interactions").mock( + return_value=httpx.Response(200, json=payload) + ) + with pytest.raises(litellm.APIConnectionError) as excinfo: + await router.atranscription(model="transcribe", file=("call.wav", AUDIO_BYTES, "audio/wav")) + assert "sensitive transcript text" not in "".join(traceback.format_exception(excinfo.value)) + + class TestSubtitleSynthesisThroughHandler: def _transform(self, config, response_format): from unittest.mock import Mock diff --git a/tests/unit/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py b/tests/unit/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py index b475cf7e3ff..494b1fe34ad 100644 --- a/tests/unit/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py +++ b/tests/unit/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py @@ -3,12 +3,15 @@ from unittest.mock import MagicMock import httpx import litellm +import pytest +from pydantic import ValidationError from litellm.llms.base_llm.audio_transcription.transformation import ( BaseAudioTranscriptionConfig, ) from litellm.llms.mistral.audio_transcription.transformation import ( MistralAudioTranscriptionConfig, + MistralAudioTranscriptionException, ) from litellm.types.utils import TranscriptionResponse from litellm.utils import ProviderConfigManager @@ -191,3 +194,22 @@ def test_mistral_audio_transcription_response_transform_empty(): assert isinstance(response, TranscriptionResponse) assert response.text == "" + + +def test_mistral_audio_transcription_response_transform_keeps_non_json_body_out_of_the_error(): + raw_response = httpx.Response(200, text='{"text": "sensitive') + + with pytest.raises(MistralAudioTranscriptionException, match="non-JSON") as exc_info: + MistralAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) + + assert "sensitive" not in str(exc_info.value) + + +@pytest.mark.parametrize("text", [["sensitive"], {"value": "sensitive"}]) +def test_mistral_audio_transcription_response_transform_keeps_a_malformed_text_field_out_of_the_error(text: object): + raw_response = httpx.Response(200, json={"text": text, "segments": [{"text": "sensitive"}]}) + + with pytest.raises(ValidationError) as exc_info: + MistralAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) + + assert "sensitive" not in str(exc_info.value) diff --git a/tests/unit/llms/openai/transcriptions/test_whisper_transformation.py b/tests/unit/llms/openai/transcriptions/test_whisper_transformation.py index 2dc24b1b313..339d62ef7ea 100644 --- a/tests/unit/llms/openai/transcriptions/test_whisper_transformation.py +++ b/tests/unit/llms/openai/transcriptions/test_whisper_transformation.py @@ -5,8 +5,10 @@ and transform_audio_transcription_response. import io import json +import traceback from unittest.mock import MagicMock +import httpx import pytest from litellm.llms.openai.transcriptions.whisper_transformation import ( @@ -104,3 +106,10 @@ class TestWhisperTransformResponse: is_json=False, ) ) + + def test_json_object_without_transcription_fields_keeps_the_body_out_of_the_error(self): + config = OpenAIWhisperAudioTranscriptionConfig() + raw_response = httpx.Response(200, json={"segments": [{"text": "sensitive"}]}) + with pytest.raises(ValueError, match="Invalid response format") as exc_info: + config.transform_audio_transcription_response(raw_response) + assert "sensitive" not in "".join(traceback.format_exception(exc_info.value)) diff --git a/tests/unit/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py b/tests/unit/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py index 422658932e6..ed8ec88e209 100644 --- a/tests/unit/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py +++ b/tests/unit/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py @@ -3,6 +3,7 @@ import pytest from pydantic import ValidationError from litellm.llms.ovhcloud.audio_transcription.transformation import OVHCloudAudioTranscriptionConfig +from litellm.llms.ovhcloud.utils import OVHCloudException from litellm.types.utils import TranscriptionResponse @@ -98,3 +99,12 @@ def test_transform_audio_transcription_response_rejects_non_object_bodies(payloa _transform(payload) assert "spoken secret" not in str(exc_info.value) + + +def test_transform_audio_transcription_response_keeps_a_non_json_body_out_of_the_error(): + raw_response = httpx.Response(200, text='{"text": "spoken secret') + + with pytest.raises(OVHCloudException, match="non-JSON") as exc_info: + OVHCloudAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) + + assert "spoken secret" not in str(exc_info.value) diff --git a/tests/unit/llms/scaleway/test_scaleway_audio_transcription_transformation.py b/tests/unit/llms/scaleway/test_scaleway_audio_transcription_transformation.py index f313f8bd873..16e60a15539 100644 --- a/tests/unit/llms/scaleway/test_scaleway_audio_transcription_transformation.py +++ b/tests/unit/llms/scaleway/test_scaleway_audio_transcription_transformation.py @@ -204,6 +204,17 @@ def test_scaleway_transform_response_raises_typed_exception_on_non_json(): ) +def test_scaleway_transform_response_keeps_non_json_body_out_of_the_error(): + raw_response = httpx.Response( + 200, text='{"text": "sensitive', headers={"content-type": "application/json"} + ) + + with pytest.raises(ScalewayAudioTranscriptionException, match="non-JSON") as exc_info: + ScalewayAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) + + assert "sensitive" not in str(exc_info.value) + + def test_scaleway_transform_response_returns_plain_text_for_non_json_content_type(): """When Scaleway responds with text/srt/vtt (response_format="text" etc.), the content-type is not application/json — return the body as plain text diff --git a/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_audio_transcription_transformation.py b/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_audio_transcription_transformation.py index 5898d933941..02b7c0ae71a 100644 --- a/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_audio_transcription_transformation.py +++ b/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_audio_transcription_transformation.py @@ -5,7 +5,7 @@ from urllib.parse import urlparse import httpx import pytest - +from pydantic import ValidationError import litellm from litellm.llms.vertex_ai.audio_transcription.transformation import ( @@ -208,6 +208,18 @@ class TestTransformResponse: response = config.transform_audio_transcription_response(raw_response) assert response.text == "" + def test_non_json_error_omits_response_body(self, config): + raw_response = httpx.Response(status_code=200, text='{"results": [{"alternatives": [{"transcript": "sensitive') + with pytest.raises(VertexAIError, match="non-JSON") as excinfo: + config.transform_audio_transcription_response(raw_response) + assert "sensitive" not in str(excinfo.value) + + def test_field_of_unexpected_type_omits_response_body(self, config): + raw_response = httpx.Response(status_code=200, json={"results": "sensitive transcript"}) + with pytest.raises(ValidationError) as excinfo: + config.transform_audio_transcription_response(raw_response) + assert "sensitive" not in str(excinfo.value) + def test_fractional_billed_duration(self, config): raw_response = httpx.Response( status_code=200, diff --git a/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py b/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py index 82ea034f91b..532e013c3d7 100644 --- a/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py +++ b/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py @@ -4,6 +4,7 @@ import os import httpx import pytest +from pydantic import ValidationError import litellm from litellm.llms.vertex_ai.audio_transcription.gemini_transcribe_transformation import ( @@ -255,6 +256,20 @@ class TestTransformResponse: with pytest.raises(VertexAIError, match="non-JSON"): config.transform_audio_transcription_response(raw_response) + def test_non_json_error_omits_response_body(self, config): + raw_response = httpx.Response(status_code=200, text='{"candidates": [{"content": {"parts": [{"text": "sensitive') + with pytest.raises(VertexAIError) as excinfo: + config.transform_audio_transcription_response(raw_response) + assert "sensitive" not in str(excinfo.value) + + def test_field_of_unexpected_type_omits_response_body(self, config): + raw_response = httpx.Response( + status_code=200, json={"candidates": [{"content": {"parts": "sensitive transcript"}}]} + ) + with pytest.raises(ValidationError) as excinfo: + config.transform_audio_transcription_response(raw_response) + assert "sensitive" not in str(excinfo.value) + class TestValidateEnvironment: def test_sets_oauth_headers(self): diff --git a/tests/unit/llms/watsonx/audio_transcription/test_watsonx_audio_transcription_transformation.py b/tests/unit/llms/watsonx/audio_transcription/test_watsonx_audio_transcription_transformation.py index b6e5007c456..c9f6f566ad6 100644 --- a/tests/unit/llms/watsonx/audio_transcription/test_watsonx_audio_transcription_transformation.py +++ b/tests/unit/llms/watsonx/audio_transcription/test_watsonx_audio_transcription_transformation.py @@ -4,6 +4,7 @@ Tests for IBM WatsonX Audio Transcription. Validates the WatsonX transcription response transformation. """ +import traceback from unittest.mock import MagicMock import httpx @@ -120,17 +121,21 @@ def test_transform_audio_transcription_response_copies_every_field_except_model( assert {key: response[key] for key in expected_extras} == expected_extras -@pytest.mark.parametrize("payload", [{}, {"model": "whisper"}, {"duration": 2.0}, {"text": None, "usage": None}]) -def test_transform_audio_transcription_response_without_text_or_usage_reports_the_body( +@pytest.mark.parametrize( + "payload", + [ + {"model": "sensitive"}, + {"duration": 2.0, "segments": [{"text": "sensitive"}]}, + {"text": None, "usage": None, "note": "sensitive"}, + ], +) +def test_transform_audio_transcription_response_without_text_or_usage_keeps_the_body_out_of_the_error( payload: dict[str, object], ) -> None: with pytest.raises(ValueError, match="Invalid response format") as exc_info: _transform_transcription_response(payload) - assert exc_info.value.args == ( - "Invalid response format. Received response does not match the expected format. Got: ", - payload, - ) + assert "sensitive" not in "".join(traceback.format_exception(exc_info.value)) @pytest.mark.parametrize("payload", [["not", "an", "object"], "plain text", 7, True]) @@ -141,3 +146,12 @@ def test_transform_audio_transcription_response_rejects_non_object_bodies_withou _transform_transcription_response(payload) assert "input_value" not in str(exc_info.value) + + +def test_transform_audio_transcription_response_keeps_a_non_json_body_out_of_the_error() -> None: + raw_response = httpx.Response(200, text='{"text": "sensitive') + + with pytest.raises(ValueError, match="Error transforming response to json") as exc_info: + IBMWatsonXAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) + + assert "sensitive" not in str(exc_info.value) diff --git a/tests/unit/llms/xai/test_xai_audio_transcription_transformation.py b/tests/unit/llms/xai/test_xai_audio_transcription_transformation.py index e2e3fc3d4dc..3505116819b 100644 --- a/tests/unit/llms/xai/test_xai_audio_transcription_transformation.py +++ b/tests/unit/llms/xai/test_xai_audio_transcription_transformation.py @@ -1,3 +1,5 @@ +import traceback + import httpx import pytest @@ -146,6 +148,23 @@ def test_transform_response_raises_on_error_status(): assert "Incorrect API key provided" in exc.value.message +@pytest.mark.parametrize( + "content", + [ + b'{"text": ["sensitive"]}', + b'{"text": "ok", "words": [{"text": ["sensitive"]}]}', + b'{"text": "sensitive', + ], +) +def test_transform_response_keeps_a_malformed_body_out_of_the_error(content): + raw = httpx.Response(200, content=content, request=httpx.Request("POST", "https://api.x.ai/v1/stt")) + + with pytest.raises(XAIAudioTranscriptionError, match="Error parsing xAI response") as exc: + CONFIG.transform_audio_transcription_response(raw_response=raw) + + assert "sensitive" not in "".join(traceback.format_exception(exc.value)) + + def test_transcription_routes_to_xai_stt(monkeypatch): monkeypatch.delenv("XAI_API_KEY", raising=False) monkeypatch.setattr(litellm, "xai_key", None)