mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-12 08:42:18 +00:00
fix(audio-transcription): keep transcripts out of transcription error messages (#50137)
* fix(gemini): omit raw response body from audio transcription error messages * fix(gemini): raise incomplete transcription interactions as non-retryable 400 and drop response bodies from Vertex transcription errors * fix(audio_transcription): keep response bodies out of transcription errors Keep the upstream status on a Gemini interaction that did not complete, so the router still retries it as it does on main. Gemini stops the same long clip on some tries and not on others, so a retry can recover it. Stop quoting the reply in the Deepgram, ElevenLabs, Watsonx, Mistral, OVHcloud and Scaleway parse errors, and hide pydantic input values on the Gemini and Vertex response models. * fix(audio-transcription): keep reply bodies out of watsonx, whisper, xai and validation errors Watsonx and whisper no longer print the parsed reply after Got: when it has no text field. TranscriptionResponse and the xAI response model hide input in validation errors, so a malformed field cannot quote transcript fragments. Deepgram and ElevenLabs keep the original error text, which is now safe. * fix(audio-transcription): keep transcripts out of the shared converter's parse errors * test(audio-transcription): compare wrapped lookup errors instead of pinning upstream wording --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
parent
8ac9da84f7
commit
5085022580
28 changed files with 276 additions and 53 deletions
|
|
@ -1072,9 +1072,10 @@ def convert_to_model_response_object(
|
|||
if isinstance(e, APIError):
|
||||
raise
|
||||
|
||||
hide_reply: Final = response_type == "audio_transcription"
|
||||
received_args: Final = dict(
|
||||
response_object=response_object,
|
||||
model_response_object=model_response_object,
|
||||
response_object="<hidden>" if hide_reply else response_object,
|
||||
model_response_object="<hidden>" if hide_reply else model_response_object,
|
||||
response_type=response_type,
|
||||
stream=stream,
|
||||
start_time=start_time,
|
||||
|
|
|
|||
|
|
@ -158,7 +158,7 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
|
|||
return response
|
||||
|
||||
except Exception as e:
|
||||
raise ValueError(f"Error transforming Deepgram response: {e}\nResponse: {raw_response.text}")
|
||||
raise ValueError(f"Error transforming Deepgram response: {e}") from None
|
||||
|
||||
def _reconstruct_diarized_transcript(self, words: list) -> str:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -160,7 +160,7 @@ class ElevenLabsAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
|
|||
return response
|
||||
|
||||
except Exception as e:
|
||||
raise ValueError(f"Error transforming ElevenLabs response: {e}\nResponse: {raw_response.text}")
|
||||
raise ValueError(f"Error transforming ElevenLabs response: {e}") from None
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -138,13 +138,13 @@ class GeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
|
|||
except ValueError:
|
||||
raise GeminiError(
|
||||
status_code=raw_response.status_code,
|
||||
message=f"Received non-JSON response from Gemini Interactions API: {raw_response.text}",
|
||||
message="Received non-JSON response from Gemini Interactions API",
|
||||
)
|
||||
parsed: Final = GeminiTranscriptionInteractionResponse.model_validate(response_json)
|
||||
if parsed.status != "completed":
|
||||
raise GeminiError(
|
||||
status_code=raw_response.status_code,
|
||||
message=f"Gemini transcription interaction did not complete (status={parsed.status}): {raw_response.text}",
|
||||
message=f"Gemini transcription interaction did not complete (status={parsed.status})",
|
||||
)
|
||||
text_contents: Final = tuple(
|
||||
content
|
||||
|
|
|
|||
|
|
@ -133,7 +133,7 @@ class MistralAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
|
|||
response_json: Final = raw_response.json()
|
||||
except Exception:
|
||||
raise MistralAudioTranscriptionException(
|
||||
message=raw_response.text,
|
||||
message="Received non-JSON response from Mistral audio transcription",
|
||||
status_code=raw_response.status_code,
|
||||
headers=raw_response.headers,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -141,7 +141,4 @@ class OpenAIWhisperAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
|
|||
if any(key in raw_response_json for key in TranscriptionResponse.model_fields):
|
||||
return TranscriptionResponse(**raw_response_json)
|
||||
else:
|
||||
raise ValueError(
|
||||
"Invalid response format. Received response does not match the expected format. Got: ",
|
||||
raw_response_json,
|
||||
)
|
||||
raise ValueError("Invalid response format. Received response does not match the expected format.")
|
||||
|
|
|
|||
|
|
@ -144,7 +144,7 @@ class OVHCloudAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
|
|||
response_json: Final = raw_response.json()
|
||||
except Exception:
|
||||
raise OVHCloudException(
|
||||
message=raw_response.text,
|
||||
message="Received non-JSON response from OVHCloud audio transcription",
|
||||
status_code=raw_response.status_code,
|
||||
headers=raw_response.headers,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -134,7 +134,7 @@ class ScalewayAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
|
|||
response_json: Final = raw_response.json()
|
||||
except Exception:
|
||||
raise ScalewayAudioTranscriptionException(
|
||||
message=raw_response.text,
|
||||
message="Received non-JSON response from Scaleway audio transcription",
|
||||
status_code=raw_response.status_code,
|
||||
headers=raw_response.headers,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -183,7 +183,7 @@ class VertexGeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexB
|
|||
except ValueError:
|
||||
raise VertexAIError(
|
||||
status_code=raw_response.status_code,
|
||||
message=f"Received non-JSON response from Vertex AI Gemini transcription: {raw_response.text}",
|
||||
message="Received non-JSON response from Vertex AI Gemini transcription",
|
||||
)
|
||||
parsed: Final = VertexGeminiTranscriptionResponse.model_validate(response_json)
|
||||
texts: Final = tuple(
|
||||
|
|
|
|||
|
|
@ -172,7 +172,7 @@ class VertexAIAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexBase)
|
|||
except ValueError:
|
||||
raise VertexAIError(
|
||||
status_code=raw_response.status_code,
|
||||
message=f"Received non-JSON response from Google Speech-to-Text: {raw_response.text}",
|
||||
message="Received non-JSON response from Google Speech-to-Text",
|
||||
)
|
||||
parsed: Final = VertexSpeechToTextRecognizeResponse.model_validate(response_json)
|
||||
transcripts: Final = tuple(
|
||||
|
|
|
|||
|
|
@ -175,7 +175,7 @@ class IBMWatsonXAudioTranscriptionConfig(IBMWatsonXMixin, OpenAIWhisperAudioTran
|
|||
try:
|
||||
raw_response_json: Final = _DECODED_JSON.validate_python(raw_response.json())
|
||||
except Exception as e:
|
||||
raise ValueError(f"Error transforming response to json: {e}\nResponse: {raw_response.text}")
|
||||
raise ValueError(f"Error transforming response to json: {e}")
|
||||
|
||||
# Extract only valid fields for TranscriptionResponse.__init__()
|
||||
# TranscriptionResponse only accepts 'text' and 'usage' in __init__()
|
||||
|
|
@ -191,10 +191,7 @@ class IBMWatsonXAudioTranscriptionConfig(IBMWatsonXMixin, OpenAIWhisperAudioTran
|
|||
response_kwargs["usage"] = usage
|
||||
|
||||
if not response_kwargs:
|
||||
raise ValueError(
|
||||
"Invalid response format. Received response does not match the expected format. Got: ",
|
||||
response_object,
|
||||
)
|
||||
raise ValueError("Invalid response format. Received response does not match the expected format.")
|
||||
|
||||
response: Final = TranscriptionResponse(**response_kwargs)
|
||||
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ class _XAISttWord(LiteLLMBaseModel):
|
|||
|
||||
|
||||
class _XAISttResponse(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
model_config = ConfigDict(extra="allow", hide_input_in_errors=True)
|
||||
text: str = ""
|
||||
language: str = "unknown"
|
||||
duration: float | None = None
|
||||
|
|
|
|||
|
|
@ -75,7 +75,7 @@ class GeminiTranscriptionUsage(LiteLLMBaseModel):
|
|||
|
||||
|
||||
class GeminiTranscriptionInteractionResponse(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
model_config = ConfigDict(extra="ignore", hide_input_in_errors=True)
|
||||
|
||||
id: str | None = None
|
||||
status: str | None = None
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ class VertexGeminiTranscriptionUsageMetadata(LiteLLMBaseModel):
|
|||
|
||||
|
||||
class VertexGeminiTranscriptionResponse(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
model_config = ConfigDict(extra="ignore", hide_input_in_errors=True)
|
||||
|
||||
candidates: tuple[VertexGeminiTranscriptionCandidate, ...] = ()
|
||||
usageMetadata: VertexGeminiTranscriptionUsageMetadata | None = None
|
||||
|
|
|
|||
|
|
@ -40,6 +40,8 @@ class VertexSpeechToTextResponseMetadata(LiteLLMBaseModel):
|
|||
|
||||
|
||||
class VertexSpeechToTextRecognizeResponse(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(hide_input_in_errors=True)
|
||||
|
||||
results: list[VertexSpeechToTextResult] = Field(default=[])
|
||||
metadata: VertexSpeechToTextResponseMetadata | None = None
|
||||
|
||||
|
|
|
|||
|
|
@ -2989,6 +2989,8 @@ _DETECTED_LANGUAGES: Final = TypeAdapter(tuple[TranscriptionDetectedLanguage, ..
|
|||
|
||||
|
||||
class TranscriptionResponse(OpenAIObject):
|
||||
model_config = ConfigDict(hide_input_in_errors=True)
|
||||
|
||||
text: str | None = None
|
||||
usage: TranscriptionUsageDurationObject | TranscriptionUsageTokensObject | None = None
|
||||
languages: Sequence[TranscriptionDetectedLanguage] | None = None
|
||||
|
|
|
|||
|
|
@ -2814,6 +2814,24 @@ class TestConvertToModelResponseObjectAudioTranscription:
|
|||
assert result.usage.output_tokens == 5
|
||||
assert result.usage.input_token_details.audio_tokens == 4
|
||||
|
||||
def test_parse_error_keeps_the_transcript_out_of_the_exception(self):
|
||||
import traceback
|
||||
|
||||
from litellm.types.utils import TranscriptionResponse
|
||||
|
||||
transcript: Final = "CANARYTRANSCRIPT customer account ending 2750"
|
||||
|
||||
with pytest.raises(Exception, match="Invalid response object") as exc_info:
|
||||
convert_to_model_response_object(
|
||||
response_object={"text": transcript, "usage": {"type": "duration", "seconds": "not-a-number"}},
|
||||
model_response_object=TranscriptionResponse(),
|
||||
response_type="audio_transcription",
|
||||
)
|
||||
|
||||
rendered: Final = "".join(traceback.format_exception(exc_info.value))
|
||||
assert "received_args=" in rendered
|
||||
assert "CANARYTRANSCRIPT" not in rendered
|
||||
|
||||
class TestConvertToModelResponseObjectRerank:
|
||||
def test_basic_rerank(self):
|
||||
from litellm.types.utils import RerankResponse
|
||||
|
|
|
|||
|
|
@ -1,11 +1,13 @@
|
|||
import io
|
||||
import os
|
||||
import pathlib
|
||||
import traceback
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from litellm.llms.base_llm.audio_transcription.transformation import AudioTranscriptionRequestData
|
||||
from litellm.llms.deepgram.audio_transcription.transformation import (
|
||||
|
|
@ -588,31 +590,33 @@ def test_transform_response_maps_words_to_openai_word_timestamps(words: object,
|
|||
[{"word": "hello", "start": 0.0}],
|
||||
],
|
||||
)
|
||||
def test_transform_response_wraps_malformed_words_with_the_raw_body(words: object):
|
||||
def test_transform_response_keeps_the_transcript_out_of_malformed_words_errors(words: object):
|
||||
raw_response = httpx.Response(200, json=_deepgram_payload({"transcript": "hello", "words": words}, {}))
|
||||
|
||||
with pytest.raises(ValueError, match="Error transforming Deepgram response: ") as exc_info:
|
||||
DeepgramAudioTranscriptionConfig().transform_audio_transcription_response(raw_response)
|
||||
|
||||
assert str(exc_info.value).endswith(f"\nResponse: {raw_response.text}")
|
||||
assert "hello" not in "".join(traceback.format_exception(exc_info.value))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("payload", "error"),
|
||||
("payload", "failing_lookup_error"),
|
||||
[
|
||||
({"results": None}, "'NoneType' object is not subscriptable"),
|
||||
(_deepgram_payload({"transcript": "hello", "words": 5}, {}), "object of type 'int' has no len()"),
|
||||
(
|
||||
{**_deepgram_payload({"transcript": "hello"}, {}), "metadata": None},
|
||||
"'NoneType' object is not subscriptable",
|
||||
),
|
||||
({"results": None}, TypeError),
|
||||
(_deepgram_payload({"transcript": "hello", "words": 5}, {}), TypeError),
|
||||
({**_deepgram_payload({"transcript": "hello"}, {}), "metadata": None}, TypeError),
|
||||
(_deepgram_payload({"transcript": ["hello"]}, {}), ValidationError),
|
||||
],
|
||||
)
|
||||
def test_transform_response_wraps_the_error_of_the_failing_lookup_in_a_malformed_body(payload: object, error: str):
|
||||
def test_transform_response_wraps_the_error_of_the_failing_lookup_without_the_body(
|
||||
payload: object, failing_lookup_error: type[Exception]
|
||||
):
|
||||
raw_response: Final = httpx.Response(200, json=payload)
|
||||
|
||||
with pytest.raises(ValueError, match="Error transforming Deepgram response: ") as exc_info:
|
||||
DeepgramAudioTranscriptionConfig().transform_audio_transcription_response(raw_response)
|
||||
|
||||
assert exc_info.type is ValueError
|
||||
assert str(exc_info.value) == f"Error transforming Deepgram response: {error}\nResponse: {raw_response.text}"
|
||||
failing_lookup: Final = exc_info.value.__context__
|
||||
assert isinstance(failing_lookup, failing_lookup_error)
|
||||
assert str(exc_info.value) == f"Error transforming Deepgram response: {failing_lookup}"
|
||||
assert "hello" not in "".join(traceback.format_exception(exc_info.value))
|
||||
|
|
|
|||
|
|
@ -1,9 +1,11 @@
|
|||
import traceback
|
||||
from collections.abc import Iterator
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from pydantic import ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm.caching.llm_caching_handler import LLMClientCache
|
||||
|
|
@ -109,24 +111,35 @@ def test_transform_audio_transcription_response_without_words_key_has_no_word_li
|
|||
@pytest.mark.parametrize(
|
||||
"payload",
|
||||
[
|
||||
["not", "an", "object"],
|
||||
"plain text",
|
||||
7,
|
||||
{"text": ["not", "text"]},
|
||||
{"text": "t", "words": None},
|
||||
{"text": "t", "words": 7},
|
||||
{"text": "t", "words": "not a list"},
|
||||
{"text": "t", "words": ["not an object"]},
|
||||
{"text": "t", "words": [{"type": "word", "text": "ok"}, None]},
|
||||
["sensitive", "an", "object"],
|
||||
"sensitive text",
|
||||
{"text": ["sensitive", "text"]},
|
||||
{"text": "sensitive", "words": None},
|
||||
{"text": "sensitive", "words": 7},
|
||||
{"text": "sensitive", "words": "not a list"},
|
||||
{"text": "sensitive", "words": ["not an object"]},
|
||||
{"text": "sensitive", "words": [{"type": "word", "text": "ok"}, None]},
|
||||
],
|
||||
)
|
||||
def test_transform_audio_transcription_response_wraps_malformed_payloads_with_the_raw_body(payload: object):
|
||||
def test_transform_audio_transcription_response_keeps_the_body_out_of_malformed_payload_errors(payload: object):
|
||||
raw_response = httpx.Response(200, json=payload)
|
||||
|
||||
with pytest.raises(ValueError, match="Error transforming ElevenLabs response: ") as exc_info:
|
||||
ElevenLabsAudioTranscriptionConfig().transform_audio_transcription_response(raw_response=raw_response)
|
||||
|
||||
assert str(exc_info.value).endswith(f"\nResponse: {raw_response.text}")
|
||||
assert "sensitive" not in "".join(traceback.format_exception(exc_info.value))
|
||||
|
||||
|
||||
def test_transform_audio_transcription_response_names_the_failing_lookup_of_a_malformed_payload():
|
||||
raw_response = httpx.Response(200, json={"text": "sensitive", "words": [None]})
|
||||
|
||||
with pytest.raises(ValueError, match="Error transforming ElevenLabs response: ") as exc_info:
|
||||
ElevenLabsAudioTranscriptionConfig().transform_audio_transcription_response(raw_response=raw_response)
|
||||
|
||||
failing_lookup: Final = exc_info.value.__context__
|
||||
assert isinstance(failing_lookup, ValidationError)
|
||||
assert str(exc_info.value) == f"Error transforming ElevenLabs response: {failing_lookup}"
|
||||
assert "sensitive" not in "".join(traceback.format_exception(exc_info.value))
|
||||
|
||||
|
||||
class TestElevenLabsAudioTranscription:
|
||||
|
|
|
|||
|
|
@ -1,8 +1,10 @@
|
|||
import base64
|
||||
import json
|
||||
import traceback
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm.llms.gemini.audio_transcription.transformation import (
|
||||
|
|
@ -241,6 +243,21 @@ class TestTransformResponse:
|
|||
with pytest.raises(GeminiError, match="non-JSON"):
|
||||
config.transform_audio_transcription_response(raw)
|
||||
|
||||
def test_non_completed_status_error_omits_response_body(self, config):
|
||||
payload = json.loads(json.dumps(COMPLETED_RESPONSE))
|
||||
payload["status"] = "incomplete"
|
||||
payload["steps"][0]["content"][0]["text"] = "sensitive transcript text"
|
||||
with pytest.raises(GeminiError) as excinfo:
|
||||
config.transform_audio_transcription_response(make_response(payload))
|
||||
assert "status=incomplete" in str(excinfo.value)
|
||||
assert "sensitive transcript text" not in str(excinfo.value)
|
||||
|
||||
def test_non_json_error_omits_response_body(self, config):
|
||||
raw = httpx.Response(200, text="sensitive body", request=httpx.Request("POST", "https://example.test"))
|
||||
with pytest.raises(GeminiError) as excinfo:
|
||||
config.transform_audio_transcription_response(raw)
|
||||
assert "sensitive body" not in str(excinfo.value)
|
||||
|
||||
def test_word_without_offsets_survives(self, config):
|
||||
payload = json.loads(json.dumps(COMPLETED_RESPONSE))
|
||||
payload["steps"][0]["content"][0]["annotations"] = [{"type": "word_info", "text": "Hello"}]
|
||||
|
|
@ -249,6 +266,66 @@ class TestTransformResponse:
|
|||
assert response.get("duration") is None
|
||||
|
||||
|
||||
|
||||
class TestNonCompletedInteractionThroughRouter:
|
||||
@pytest.fixture
|
||||
def router(self, monkeypatch):
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
return litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "transcribe",
|
||||
"litellm_params": {"model": "gemini/gemini-3.5-transcribe", "api_key": "test-gemini-key"},
|
||||
}
|
||||
],
|
||||
num_retries=2,
|
||||
)
|
||||
|
||||
def _interaction(self, status):
|
||||
payload = json.loads(json.dumps(COMPLETED_RESPONSE))
|
||||
payload["status"] = status
|
||||
payload["steps"][0]["content"][0]["text"] = "sensitive transcript text"
|
||||
return payload
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_retry_recovers_an_incomplete_interaction(self, router):
|
||||
route = respx.post("https://generativelanguage.googleapis.com/v1beta/interactions").mock(
|
||||
side_effect=[
|
||||
httpx.Response(200, json=self._interaction("incomplete")),
|
||||
httpx.Response(200, json=COMPLETED_RESPONSE),
|
||||
]
|
||||
)
|
||||
response = await router.atranscription(model="transcribe", file=("call.wav", AUDIO_BYTES, "audio/wav"))
|
||||
assert route.call_count == 2
|
||||
assert response.text == COMPLETED_RESPONSE["steps"][0]["content"][0]["text"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
@pytest.mark.parametrize("status", ["incomplete", "failed", "cancelled", "in_progress", None])
|
||||
async def test_non_completed_interaction_retries_then_fails_without_transcript(self, router, status):
|
||||
route = respx.post("https://generativelanguage.googleapis.com/v1beta/interactions").mock(
|
||||
return_value=httpx.Response(200, json=self._interaction(status))
|
||||
)
|
||||
with pytest.raises(litellm.APIConnectionError) as excinfo:
|
||||
await router.atranscription(model="transcribe", file=("call.wav", AUDIO_BYTES, "audio/wav"))
|
||||
assert route.call_count == 3
|
||||
assert f"status={status}" in str(excinfo.value)
|
||||
assert "sensitive transcript text" not in str(excinfo.value)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_response_field_of_unexpected_type_keeps_transcript_out_of_the_error(self, router):
|
||||
payload = self._interaction("incomplete")
|
||||
payload["steps"][0]["content"] = "sensitive transcript text"
|
||||
respx.post("https://generativelanguage.googleapis.com/v1beta/interactions").mock(
|
||||
return_value=httpx.Response(200, json=payload)
|
||||
)
|
||||
with pytest.raises(litellm.APIConnectionError) as excinfo:
|
||||
await router.atranscription(model="transcribe", file=("call.wav", AUDIO_BYTES, "audio/wav"))
|
||||
assert "sensitive transcript text" not in "".join(traceback.format_exception(excinfo.value))
|
||||
|
||||
|
||||
class TestSubtitleSynthesisThroughHandler:
|
||||
def _transform(self, config, response_format):
|
||||
from unittest.mock import Mock
|
||||
|
|
|
|||
|
|
@ -3,12 +3,15 @@ from unittest.mock import MagicMock
|
|||
|
||||
import httpx
|
||||
import litellm
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from litellm.llms.base_llm.audio_transcription.transformation import (
|
||||
BaseAudioTranscriptionConfig,
|
||||
)
|
||||
from litellm.llms.mistral.audio_transcription.transformation import (
|
||||
MistralAudioTranscriptionConfig,
|
||||
MistralAudioTranscriptionException,
|
||||
)
|
||||
from litellm.types.utils import TranscriptionResponse
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
|
@ -191,3 +194,22 @@ def test_mistral_audio_transcription_response_transform_empty():
|
|||
|
||||
assert isinstance(response, TranscriptionResponse)
|
||||
assert response.text == ""
|
||||
|
||||
|
||||
def test_mistral_audio_transcription_response_transform_keeps_non_json_body_out_of_the_error():
|
||||
raw_response = httpx.Response(200, text='{"text": "sensitive')
|
||||
|
||||
with pytest.raises(MistralAudioTranscriptionException, match="non-JSON") as exc_info:
|
||||
MistralAudioTranscriptionConfig().transform_audio_transcription_response(raw_response)
|
||||
|
||||
assert "sensitive" not in str(exc_info.value)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text", [["sensitive"], {"value": "sensitive"}])
|
||||
def test_mistral_audio_transcription_response_transform_keeps_a_malformed_text_field_out_of_the_error(text: object):
|
||||
raw_response = httpx.Response(200, json={"text": text, "segments": [{"text": "sensitive"}]})
|
||||
|
||||
with pytest.raises(ValidationError) as exc_info:
|
||||
MistralAudioTranscriptionConfig().transform_audio_transcription_response(raw_response)
|
||||
|
||||
assert "sensitive" not in str(exc_info.value)
|
||||
|
|
|
|||
|
|
@ -5,8 +5,10 @@ and transform_audio_transcription_response.
|
|||
|
||||
import io
|
||||
import json
|
||||
import traceback
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from litellm.llms.openai.transcriptions.whisper_transformation import (
|
||||
|
|
@ -104,3 +106,10 @@ class TestWhisperTransformResponse:
|
|||
is_json=False,
|
||||
)
|
||||
)
|
||||
|
||||
def test_json_object_without_transcription_fields_keeps_the_body_out_of_the_error(self):
|
||||
config = OpenAIWhisperAudioTranscriptionConfig()
|
||||
raw_response = httpx.Response(200, json={"segments": [{"text": "sensitive"}]})
|
||||
with pytest.raises(ValueError, match="Invalid response format") as exc_info:
|
||||
config.transform_audio_transcription_response(raw_response)
|
||||
assert "sensitive" not in "".join(traceback.format_exception(exc_info.value))
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import pytest
|
|||
from pydantic import ValidationError
|
||||
|
||||
from litellm.llms.ovhcloud.audio_transcription.transformation import OVHCloudAudioTranscriptionConfig
|
||||
from litellm.llms.ovhcloud.utils import OVHCloudException
|
||||
from litellm.types.utils import TranscriptionResponse
|
||||
|
||||
|
||||
|
|
@ -98,3 +99,12 @@ def test_transform_audio_transcription_response_rejects_non_object_bodies(payloa
|
|||
_transform(payload)
|
||||
|
||||
assert "spoken secret" not in str(exc_info.value)
|
||||
|
||||
|
||||
def test_transform_audio_transcription_response_keeps_a_non_json_body_out_of_the_error():
|
||||
raw_response = httpx.Response(200, text='{"text": "spoken secret')
|
||||
|
||||
with pytest.raises(OVHCloudException, match="non-JSON") as exc_info:
|
||||
OVHCloudAudioTranscriptionConfig().transform_audio_transcription_response(raw_response)
|
||||
|
||||
assert "spoken secret" not in str(exc_info.value)
|
||||
|
|
|
|||
|
|
@ -204,6 +204,17 @@ def test_scaleway_transform_response_raises_typed_exception_on_non_json():
|
|||
)
|
||||
|
||||
|
||||
def test_scaleway_transform_response_keeps_non_json_body_out_of_the_error():
|
||||
raw_response = httpx.Response(
|
||||
200, text='{"text": "sensitive', headers={"content-type": "application/json"}
|
||||
)
|
||||
|
||||
with pytest.raises(ScalewayAudioTranscriptionException, match="non-JSON") as exc_info:
|
||||
ScalewayAudioTranscriptionConfig().transform_audio_transcription_response(raw_response)
|
||||
|
||||
assert "sensitive" not in str(exc_info.value)
|
||||
|
||||
|
||||
def test_scaleway_transform_response_returns_plain_text_for_non_json_content_type():
|
||||
"""When Scaleway responds with text/srt/vtt (response_format="text" etc.),
|
||||
the content-type is not application/json — return the body as plain text
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ from urllib.parse import urlparse
|
|||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from pydantic import ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm.llms.vertex_ai.audio_transcription.transformation import (
|
||||
|
|
@ -208,6 +208,18 @@ class TestTransformResponse:
|
|||
response = config.transform_audio_transcription_response(raw_response)
|
||||
assert response.text == ""
|
||||
|
||||
def test_non_json_error_omits_response_body(self, config):
|
||||
raw_response = httpx.Response(status_code=200, text='{"results": [{"alternatives": [{"transcript": "sensitive')
|
||||
with pytest.raises(VertexAIError, match="non-JSON") as excinfo:
|
||||
config.transform_audio_transcription_response(raw_response)
|
||||
assert "sensitive" not in str(excinfo.value)
|
||||
|
||||
def test_field_of_unexpected_type_omits_response_body(self, config):
|
||||
raw_response = httpx.Response(status_code=200, json={"results": "sensitive transcript"})
|
||||
with pytest.raises(ValidationError) as excinfo:
|
||||
config.transform_audio_transcription_response(raw_response)
|
||||
assert "sensitive" not in str(excinfo.value)
|
||||
|
||||
def test_fractional_billed_duration(self, config):
|
||||
raw_response = httpx.Response(
|
||||
status_code=200,
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import os
|
|||
|
||||
import httpx
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm.llms.vertex_ai.audio_transcription.gemini_transcribe_transformation import (
|
||||
|
|
@ -255,6 +256,20 @@ class TestTransformResponse:
|
|||
with pytest.raises(VertexAIError, match="non-JSON"):
|
||||
config.transform_audio_transcription_response(raw_response)
|
||||
|
||||
def test_non_json_error_omits_response_body(self, config):
|
||||
raw_response = httpx.Response(status_code=200, text='{"candidates": [{"content": {"parts": [{"text": "sensitive')
|
||||
with pytest.raises(VertexAIError) as excinfo:
|
||||
config.transform_audio_transcription_response(raw_response)
|
||||
assert "sensitive" not in str(excinfo.value)
|
||||
|
||||
def test_field_of_unexpected_type_omits_response_body(self, config):
|
||||
raw_response = httpx.Response(
|
||||
status_code=200, json={"candidates": [{"content": {"parts": "sensitive transcript"}}]}
|
||||
)
|
||||
with pytest.raises(ValidationError) as excinfo:
|
||||
config.transform_audio_transcription_response(raw_response)
|
||||
assert "sensitive" not in str(excinfo.value)
|
||||
|
||||
|
||||
class TestValidateEnvironment:
|
||||
def test_sets_oauth_headers(self):
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ Tests for IBM WatsonX Audio Transcription.
|
|||
Validates the WatsonX transcription response transformation.
|
||||
"""
|
||||
|
||||
import traceback
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import httpx
|
||||
|
|
@ -120,17 +121,21 @@ def test_transform_audio_transcription_response_copies_every_field_except_model(
|
|||
assert {key: response[key] for key in expected_extras} == expected_extras
|
||||
|
||||
|
||||
@pytest.mark.parametrize("payload", [{}, {"model": "whisper"}, {"duration": 2.0}, {"text": None, "usage": None}])
|
||||
def test_transform_audio_transcription_response_without_text_or_usage_reports_the_body(
|
||||
@pytest.mark.parametrize(
|
||||
"payload",
|
||||
[
|
||||
{"model": "sensitive"},
|
||||
{"duration": 2.0, "segments": [{"text": "sensitive"}]},
|
||||
{"text": None, "usage": None, "note": "sensitive"},
|
||||
],
|
||||
)
|
||||
def test_transform_audio_transcription_response_without_text_or_usage_keeps_the_body_out_of_the_error(
|
||||
payload: dict[str, object],
|
||||
) -> None:
|
||||
with pytest.raises(ValueError, match="Invalid response format") as exc_info:
|
||||
_transform_transcription_response(payload)
|
||||
|
||||
assert exc_info.value.args == (
|
||||
"Invalid response format. Received response does not match the expected format. Got: ",
|
||||
payload,
|
||||
)
|
||||
assert "sensitive" not in "".join(traceback.format_exception(exc_info.value))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("payload", [["not", "an", "object"], "plain text", 7, True])
|
||||
|
|
@ -141,3 +146,12 @@ def test_transform_audio_transcription_response_rejects_non_object_bodies_withou
|
|||
_transform_transcription_response(payload)
|
||||
|
||||
assert "input_value" not in str(exc_info.value)
|
||||
|
||||
|
||||
def test_transform_audio_transcription_response_keeps_a_non_json_body_out_of_the_error() -> None:
|
||||
raw_response = httpx.Response(200, text='{"text": "sensitive')
|
||||
|
||||
with pytest.raises(ValueError, match="Error transforming response to json") as exc_info:
|
||||
IBMWatsonXAudioTranscriptionConfig().transform_audio_transcription_response(raw_response)
|
||||
|
||||
assert "sensitive" not in str(exc_info.value)
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
import traceback
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
|
|
@ -146,6 +148,23 @@ def test_transform_response_raises_on_error_status():
|
|||
assert "Incorrect API key provided" in exc.value.message
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"content",
|
||||
[
|
||||
b'{"text": ["sensitive"]}',
|
||||
b'{"text": "ok", "words": [{"text": ["sensitive"]}]}',
|
||||
b'{"text": "sensitive',
|
||||
],
|
||||
)
|
||||
def test_transform_response_keeps_a_malformed_body_out_of_the_error(content):
|
||||
raw = httpx.Response(200, content=content, request=httpx.Request("POST", "https://api.x.ai/v1/stt"))
|
||||
|
||||
with pytest.raises(XAIAudioTranscriptionError, match="Error parsing xAI response") as exc:
|
||||
CONFIG.transform_audio_transcription_response(raw_response=raw)
|
||||
|
||||
assert "sensitive" not in "".join(traceback.format_exception(exc.value))
|
||||
|
||||
|
||||
def test_transcription_routes_to_xai_stt(monkeypatch):
|
||||
monkeypatch.delenv("XAI_API_KEY", raising=False)
|
||||
monkeypatch.setattr(litellm, "xai_key", None)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue