From 086a58a06a3fcdac61d9ede4f6eb4f14b73c7c1f Mon Sep 17 00:00:00 2001 From: Chesars Date: Wed, 4 Mar 2026 16:20:05 -0300 Subject: [PATCH] feat(mistral): add Voxtral audio transcription support Add MistralAudioTranscriptionConfig for Mistral's /v1/audio/transcriptions endpoint, enabling litellm.transcription() with mistral/voxtral-mini-latest and other Voxtral models. Supports multipart form-data with OpenAI-compatible params (language, temperature, response_format, timestamp_granularities) plus Mistral-specific params like diarize. --- .../get_supported_openai_params.py | 8 + .../audio_transcription/transformation.py | 152 ++++++++++++++++ litellm/utils.py | 6 + ...tral_audio_transcription_transformation.py | 170 ++++++++++++++++++ 4 files changed, 336 insertions(+) create mode 100644 litellm/llms/mistral/audio_transcription/transformation.py create mode 100644 tests/test_litellm/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py diff --git a/litellm/litellm_core_utils/get_supported_openai_params.py b/litellm/litellm_core_utils/get_supported_openai_params.py index 4b40f44cbc4..534ec730b06 100644 --- a/litellm/litellm_core_utils/get_supported_openai_params.py +++ b/litellm/litellm_core_utils/get_supported_openai_params.py @@ -142,6 +142,14 @@ def get_supported_openai_params( # noqa: PLR0915 return litellm.MistralConfig().get_supported_openai_params(model=model) elif request_type == "embeddings": return litellm.MistralEmbeddingConfig().get_supported_openai_params() + elif request_type == "transcription": + from litellm.llms.mistral.audio_transcription.transformation import ( + MistralAudioTranscriptionConfig, + ) + + return MistralAudioTranscriptionConfig().get_supported_openai_params( + model=model + ) elif custom_llm_provider == "text-completion-codestral": return litellm.CodestralTextCompletionConfig().get_supported_openai_params( model=model diff --git a/litellm/llms/mistral/audio_transcription/transformation.py b/litellm/llms/mistral/audio_transcription/transformation.py new file mode 100644 index 00000000000..609035576e7 --- /dev/null +++ b/litellm/llms/mistral/audio_transcription/transformation.py @@ -0,0 +1,152 @@ +""" +Support for Mistral Voxtral audio transcription via ``/v1/audio/transcriptions``. + +API reference: https://docs.mistral.ai/api/#tag/audio/operation/audio_transcriptions_v1_audio_transcriptions_post +""" + +from typing import List, Optional, Union + +import httpx + +from litellm.litellm_core_utils.audio_utils.utils import process_audio_file +from litellm.llms.base_llm.audio_transcription.transformation import ( + AudioTranscriptionRequestData, + BaseAudioTranscriptionConfig, +) +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import ( + AllMessageValues, + OpenAIAudioTranscriptionOptionalParams, +) +from litellm.types.utils import FileTypes, TranscriptionResponse + + +class MistralAudioTranscriptionException(BaseLLMException): + pass + + +class MistralAudioTranscriptionConfig(BaseAudioTranscriptionConfig): + def get_supported_openai_params( + self, model: str + ) -> List[OpenAIAudioTranscriptionOptionalParams]: + return [ + "language", + "temperature", + "timestamp_granularities", + "response_format", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + supported_params = self.get_supported_openai_params(model) + for k, v in non_default_params.items(): + if k in supported_params: + optional_params[k] = v + return optional_params + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + api_base = ( + "https://api.mistral.ai/v1" + if api_base is None + else api_base.rstrip("/") + ) + return f"{api_base}/audio/transcriptions" + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> BaseLLMException: + return MistralAudioTranscriptionException( + message=error_message, + status_code=status_code, + headers=headers, + ) + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + if api_key is None: + api_key = get_secret_str("MISTRAL_API_KEY") + + default_headers = { + "Authorization": f"Bearer {api_key}", + "accept": "application/json", + } + default_headers.update(headers or {}) + return default_headers + + def transform_audio_transcription_request( + self, + model: str, + audio_file: FileTypes, + optional_params: dict, + litellm_params: dict, + ) -> AudioTranscriptionRequestData: + processed_audio = process_audio_file(audio_file) + + form_fields: dict = { + "model": model, + } + + # OpenAI-compatible params + for key in self.get_supported_openai_params(model): + value = optional_params.get(key) + if value is not None: + form_fields[key] = value + + # Mistral-specific params (e.g. diarize) + provider_specific_params = self.get_provider_specific_params( + model=model, + optional_params=optional_params, + openai_params=self.get_supported_openai_params(model), + ) + for key, value in provider_specific_params.items(): + form_fields[key] = str(value) + + files = { + "file": ( + processed_audio.filename, + processed_audio.file_content, + processed_audio.content_type, + ) + } + + return AudioTranscriptionRequestData(data=form_fields, files=files) + + def transform_audio_transcription_response( + self, + raw_response: httpx.Response, + ) -> TranscriptionResponse: + try: + response_json = raw_response.json() + except Exception: + raise MistralAudioTranscriptionException( + message=raw_response.text, + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + text = response_json.get("text") or "" + response = TranscriptionResponse(text=text) + response._hidden_params = response_json + return response diff --git a/litellm/utils.py b/litellm/utils.py index d192609eead..09351af81b0 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8265,6 +8265,12 @@ class ProviderConfigManager: ) return OVHCloudAudioTranscriptionConfig() + elif litellm.LlmProviders.MISTRAL == provider: + from litellm.llms.mistral.audio_transcription.transformation import ( + MistralAudioTranscriptionConfig, + ) + + return MistralAudioTranscriptionConfig() return None @staticmethod diff --git a/tests/test_litellm/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py b/tests/test_litellm/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py new file mode 100644 index 00000000000..6dc5133f665 --- /dev/null +++ b/tests/test_litellm/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py @@ -0,0 +1,170 @@ +import os +from typing import Dict +from unittest.mock import MagicMock + +import httpx +import litellm +import pytest + +from litellm.llms.base_llm.audio_transcription.transformation import ( + BaseAudioTranscriptionConfig, +) +from litellm.llms.mistral.audio_transcription.transformation import ( + MistralAudioTranscriptionConfig, +) +from litellm.types.utils import TranscriptionResponse +from litellm.utils import ProviderConfigManager +from tests.llm_translation.base_audio_transcription_unit_tests import ( + BaseLLMAudioTranscriptionTest, +) + + +@pytest.mark.skipif( + not os.getenv("MISTRAL_API_KEY"), + reason="MISTRAL_API_KEY not set, skipping Mistral audio transcription tests", +) +class TestMistralAudioTranscription(BaseLLMAudioTranscriptionTest): + def get_base_audio_transcription_call_args(self) -> Dict: + return { + "model": "mistral/voxtral-mini-latest", + } + + def get_custom_llm_provider(self) -> litellm.LlmProviders: + return litellm.LlmProviders.MISTRAL + + def test_audio_transcription_async(self): # type: ignore[override] + pytest.skip( + "Async audio transcription test for Mistral is skipped in this suite; " + "async test plugins (e.g. pytest-asyncio/anyio) are not configured here." + ) + + +def test_mistral_audio_transcription_config_installed(): + """Ensure Mistral audio transcription config is registered with ProviderConfigManager.""" + config = ProviderConfigManager.get_provider_audio_transcription_config( + model="mistral/voxtral-mini-latest", + provider=litellm.LlmProviders.MISTRAL, + ) + assert config is not None + assert isinstance(config, BaseAudioTranscriptionConfig) + assert isinstance(config, MistralAudioTranscriptionConfig) + + +def test_mistral_audio_transcription_get_complete_url(): + config = MistralAudioTranscriptionConfig() + url = config.get_complete_url( + api_base=None, + api_key="fake-key", + model="voxtral-mini-latest", + optional_params={}, + litellm_params={}, + ) + assert url == "https://api.mistral.ai/v1/audio/transcriptions" + + +def test_mistral_audio_transcription_get_complete_url_custom_base(): + config = MistralAudioTranscriptionConfig() + url = config.get_complete_url( + api_base="https://custom.api.example.com/v1/", + api_key="fake-key", + model="voxtral-mini-latest", + optional_params={}, + litellm_params={}, + ) + assert url == "https://custom.api.example.com/v1/audio/transcriptions" + + +def test_mistral_audio_transcription_validate_environment(): + config = MistralAudioTranscriptionConfig() + headers = config.validate_environment( + headers={}, + model="voxtral-mini-latest", + messages=[], + optional_params={}, + litellm_params={}, + api_key="test-key-123", + ) + assert headers["Authorization"] == "Bearer test-key-123" + assert headers["accept"] == "application/json" + + +def test_mistral_audio_transcription_supported_params(): + config = MistralAudioTranscriptionConfig() + params = config.get_supported_openai_params("voxtral-mini-latest") + assert "language" in params + assert "temperature" in params + assert "response_format" in params + assert "timestamp_granularities" in params + + +def test_mistral_audio_transcription_request_transform(): + config = MistralAudioTranscriptionConfig() + + wav_path = os.path.join( + os.path.dirname(__file__), "../../../../..", "tests", "llm_translation", "gettysburg.wav" + ) + audio_file = open(wav_path, "rb") + + result = config.transform_audio_transcription_request( + model="voxtral-mini-latest", + audio_file=audio_file, + optional_params={"language": "en", "temperature": 0.0}, + litellm_params={}, + ) + + audio_file.close() + + assert isinstance(result.data, dict) + assert result.data["model"] == "voxtral-mini-latest" + assert result.data["language"] == "en" + assert result.data["temperature"] == 0.0 + assert result.files is not None + assert "file" in result.files + + +def test_mistral_audio_transcription_request_with_diarize(): + """Test that Mistral-specific params like diarize are passed through.""" + config = MistralAudioTranscriptionConfig() + + wav_path = os.path.join( + os.path.dirname(__file__), "../../../../..", "tests", "llm_translation", "gettysburg.wav" + ) + audio_file = open(wav_path, "rb") + + result = config.transform_audio_transcription_request( + model="voxtral-mini-latest", + audio_file=audio_file, + optional_params={"diarize": True}, + litellm_params={}, + ) + + audio_file.close() + + assert isinstance(result.data, dict) + assert result.data["diarize"] == "True" + + +def test_mistral_audio_transcription_response_transform(): + config = MistralAudioTranscriptionConfig() + + mock_response = MagicMock(spec=httpx.Response) + mock_response.json.return_value = { + "text": "Four score and seven years ago..." + } + + response = config.transform_audio_transcription_response(mock_response) + + assert isinstance(response, TranscriptionResponse) + assert response.text == "Four score and seven years ago..." + + +def test_mistral_audio_transcription_response_transform_empty(): + config = MistralAudioTranscriptionConfig() + + mock_response = MagicMock(spec=httpx.Response) + mock_response.json.return_value = {} + + response = config.transform_audio_transcription_response(mock_response) + + assert isinstance(response, TranscriptionResponse) + assert response.text == ""