scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-28 01:32:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 43

fix(vertex): send audio/webm for Gemini transcription of webm

Browse source
This commit is contained in:
RealBhupesh 2026-09-01 11:34:20 +05:30
parent c78f405473
commit 21edacd9a5
2 changed files with 38 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py
View file

@ -44,6 +44,7 @@ from litellm.types.utils import (
DEFAULT_GEMINI_TRANSCRIBE_LOCATION: Final = "global"
AUDIO_MODALITY: Final = "AUDIO"
AMBIGUOUS_WEBM_MIME_TYPES: Final = frozenset({"application/webm", "application/x-webm", "video/webm", "video/x-webm"})
class VertexGeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexBase):
@ -151,6 +152,7 @@ class VertexGeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexB
litellm_params: Mapping[str, object],
) -> AudioTranscriptionRequestData:
processed_audio: Final = process_audio_file(audio_file)
mime_type: Final = _audio_transcription_mime_type(audio_file, processed_audio.content_type)
request_body: Final = VertexGeminiTranscriptionRequest(
contents=(
VertexGeminiTranscriptionContent(
@ -158,7 +160,7 @@ class VertexGeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexB
parts=(
VertexGeminiTranscriptionPart(
inlineData=VertexGeminiTranscriptionInlineData(
mimeType=processed_audio.content_type,
mimeType=mime_type,
data=base64.b64encode(processed_audio.file_content).decode("utf-8"),
)
),
@ -210,6 +212,16 @@ class VertexGeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexB
return response
def _audio_transcription_mime_type(audio_file: FileTypes, processed_content_type: str) -> str:
incoming_content_type: Final = audio_file[2] if isinstance(audio_file, tuple) and len(audio_file) >= 3 else None
if isinstance(incoming_content_type, str) and incoming_content_type.split(";", 1)[0].strip().lower().startswith(
"audio/"
):
return incoming_content_type
processed_media_type: Final = processed_content_type.split(";", 1)[0].strip().lower()
return "audio/webm" if processed_media_type in AMBIGUOUS_WEBM_MIME_TYPES else processed_content_type
def _audio_transcription_config(language: object) -> VertexGeminiTranscriptionAudioConfig:
if not isinstance(language, str) or not language:
return VertexGeminiTranscriptionAudioConfig()

25
tests/test_litellm/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py
View file

@ -1,6 +1,7 @@
import base64
import json
import os
from unittest.mock import Mock
import httpx
import pytest
@ -217,6 +218,30 @@ class TestTransformRequest:
assert round_tripped["generationConfig"] == {"audioTranscriptionConfig": {"languageCodes": ["en-US"]}}
assert round_tripped["contents"][0]["role"] == "user"
def test_webm_audio_uses_audio_mime_type(self, config):
audio_file = Mock(spec=["name", "read", "seek"])
audio_file.name = "speech.webm"
audio_file.read.return_value = AUDIO_BYTES
request_data = config.transform_audio_transcription_request(
model="gemini-3.5-transcribe-preview",
audio_file=audio_file,
optional_params={},
litellm_params={},
)
assert request_data.data["contents"][0]["parts"][0]["inlineData"]["mimeType"] == "audio/webm"
def test_incoming_audio_mime_type_is_preserved(self, config):
request_data = config.transform_audio_transcription_request(
model="gemini-3.5-transcribe-preview",
audio_file=("speech.webm", AUDIO_BYTES, "audio/webm; codecs=opus"),
optional_params={},
litellm_params={},
)
assert request_data.data["contents"][0]["parts"][0]["inlineData"]["mimeType"] == "audio/webm; codecs=opus"
class TestTransformResponse:
def test_generate_content_response(self, config):