From 21edacd9a525c4fef6938e1cb2bf45a1ab9c480b Mon Sep 17 00:00:00 2001 From: RealBhupesh Date: Tue, 1 Sep 2026 11:34:20 +0530 Subject: [PATCH] fix(vertex): send audio/webm for Gemini transcription of webm --- .../gemini_transcribe_transformation.py | 14 ++++++++++- ...tex_ai_gemini_transcribe_transformation.py | 25 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py b/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py index f4db5eb110c..a923830c780 100644 --- a/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py +++ b/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py @@ -44,6 +44,7 @@ from litellm.types.utils import ( DEFAULT_GEMINI_TRANSCRIBE_LOCATION: Final = "global" AUDIO_MODALITY: Final = "AUDIO" +AMBIGUOUS_WEBM_MIME_TYPES: Final = frozenset({"application/webm", "application/x-webm", "video/webm", "video/x-webm"}) class VertexGeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexBase): @@ -151,6 +152,7 @@ class VertexGeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexB litellm_params: Mapping[str, object], ) -> AudioTranscriptionRequestData: processed_audio: Final = process_audio_file(audio_file) + mime_type: Final = _audio_transcription_mime_type(audio_file, processed_audio.content_type) request_body: Final = VertexGeminiTranscriptionRequest( contents=( VertexGeminiTranscriptionContent( @@ -158,7 +160,7 @@ class VertexGeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexB parts=( VertexGeminiTranscriptionPart( inlineData=VertexGeminiTranscriptionInlineData( - mimeType=processed_audio.content_type, + mimeType=mime_type, data=base64.b64encode(processed_audio.file_content).decode("utf-8"), ) ), @@ -210,6 +212,16 @@ class VertexGeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexB return response +def _audio_transcription_mime_type(audio_file: FileTypes, processed_content_type: str) -> str: + incoming_content_type: Final = audio_file[2] if isinstance(audio_file, tuple) and len(audio_file) >= 3 else None + if isinstance(incoming_content_type, str) and incoming_content_type.split(";", 1)[0].strip().lower().startswith( + "audio/" + ): + return incoming_content_type + processed_media_type: Final = processed_content_type.split(";", 1)[0].strip().lower() + return "audio/webm" if processed_media_type in AMBIGUOUS_WEBM_MIME_TYPES else processed_content_type + + def _audio_transcription_config(language: object) -> VertexGeminiTranscriptionAudioConfig: if not isinstance(language, str) or not language: return VertexGeminiTranscriptionAudioConfig() diff --git a/tests/test_litellm/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py b/tests/test_litellm/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py index eadd87d9c92..63a44457b02 100644 --- a/tests/test_litellm/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py @@ -1,6 +1,7 @@ import base64 import json import os +from unittest.mock import Mock import httpx import pytest @@ -217,6 +218,30 @@ class TestTransformRequest: assert round_tripped["generationConfig"] == {"audioTranscriptionConfig": {"languageCodes": ["en-US"]}} assert round_tripped["contents"][0]["role"] == "user" + def test_webm_audio_uses_audio_mime_type(self, config): + audio_file = Mock(spec=["name", "read", "seek"]) + audio_file.name = "speech.webm" + audio_file.read.return_value = AUDIO_BYTES + + request_data = config.transform_audio_transcription_request( + model="gemini-3.5-transcribe-preview", + audio_file=audio_file, + optional_params={}, + litellm_params={}, + ) + + assert request_data.data["contents"][0]["parts"][0]["inlineData"]["mimeType"] == "audio/webm" + + def test_incoming_audio_mime_type_is_preserved(self, config): + request_data = config.transform_audio_transcription_request( + model="gemini-3.5-transcribe-preview", + audio_file=("speech.webm", AUDIO_BYTES, "audio/webm; codecs=opus"), + optional_params={}, + litellm_params={}, + ) + + assert request_data.data["contents"][0]["parts"][0]["inlineData"]["mimeType"] == "audio/webm; codecs=opus" + class TestTransformResponse: def test_generate_content_response(self, config):