feat(audio): add Gemini transcription vocabulary support

This commit is contained in:
Marcin Juraszek 2026-09-22 21:36:26 +02:00
parent 2ea43214c9
commit fd023cc46a
14 changed files with 239 additions and 10 deletions

View file

@ -825,6 +825,7 @@ OPENAI_CHAT_COMPLETION_PARAMS: Final = [
OPENAI_TRANSCRIPTION_PARAMS: Final = [
"language",
"keywords",
"response_format",
"timestamp_granularities",
]

View file

@ -4,6 +4,7 @@ Utils used for litellm.transcription() and litellm.atranscription()
import hashlib
import os
from collections.abc import Sequence
from dataclasses import dataclass
from typing import Final
@ -142,6 +143,7 @@ BARE_ISO_639_1_TO_BCP47: Final = {
"ru": "ru-RU",
"hi": "hi-IN",
"ar": "ar-SA",
"pl": "pl-PL",
}
@ -157,6 +159,12 @@ def normalize_transcription_language_to_bcp47(language: str) -> str:
return BARE_ISO_639_1_TO_BCP47.get(language.lower(), language)
def normalize_transcription_keywords(keywords: object) -> tuple[str, ...]:
if not isinstance(keywords, Sequence) or isinstance(keywords, (str, bytes)):
return ()
return tuple(keyword for keyword in keywords if isinstance(keyword, str) and keyword)
def get_audio_file_name(file_obj: FileTypes) -> str:
"""
Safely get the name of a file-like object or return its string representation.

View file

@ -183,7 +183,17 @@ def get_supported_openai_params(
return litellm.DatabricksConfig().get_supported_openai_params(model=model)
elif request_type == "embeddings":
return litellm.DatabricksEmbeddingConfig().get_supported_openai_params()
elif custom_llm_provider == "palm" or custom_llm_provider == "gemini":
elif custom_llm_provider == "palm":
return litellm.GoogleAIStudioGeminiConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "gemini":
if request_type == "transcription":
transcription_provider_config = litellm.ProviderConfigManager.get_provider_audio_transcription_config(
model=model,
provider=LlmProviders.GEMINI,
)
if transcription_provider_config:
return transcription_provider_config.get_supported_openai_params(model=model)
return None
return litellm.GoogleAIStudioGeminiConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "novita":
return litellm.NovitaConfig().get_supported_openai_params(model=model)

View file

@ -4,8 +4,11 @@ from typing import Final
from httpx import Headers, Response
import litellm
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.audio_utils.subtitle_utils import SUBTITLE_RESPONSE_FORMATS
from litellm.litellm_core_utils.audio_utils.utils import (
normalize_transcription_keywords,
normalize_transcription_language_to_bcp47,
process_audio_file,
)
@ -47,7 +50,12 @@ class GeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
def get_supported_openai_params(
self, model: str
) -> list[OpenAIAudioTranscriptionOptionalParams]: # mutable-ok: BaseAudioTranscriptionConfig signature
return ["language", "response_format", "timestamp_granularities"] # mutable-ok: base contract returns a list
return [ # mutable-ok: BaseAudioTranscriptionConfig requires a list
"language",
"keywords",
"response_format",
"timestamp_granularities",
]
@property
def supports_subtitle_synthesis(self) -> bool:
@ -123,7 +131,10 @@ class GeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
request: Final = _build_interaction_request(
model=model,
audio_input=audio_input,
transcription_config=_build_transcription_config(optional_params),
transcription_config=_build_transcription_config(
optional_params,
drop_params=litellm_params.get("drop_params") is True or bool(litellm.drop_params),
),
)
return AudioTranscriptionRequestData(data=dict(request)) # mutable-ok: AudioTranscriptionRequestData wants dict
@ -227,10 +238,32 @@ def _timestamp_config(timestamp_granularities: object, response_format: object)
return _WORD_TIMESTAMP_CONFIG if wants_word_timestamps else _EMPTY_TRANSCRIPTION_CONFIG
def _build_transcription_config(optional_params: Mapping[str, object]) -> GeminiTranscriptionConfig:
def _custom_vocabulary_config(keywords: object) -> GeminiTranscriptionConfig:
vocabulary: Final = normalize_transcription_keywords(keywords)
vocabulary_config: Final[GeminiTranscriptionConfig] = {"custom_vocabulary": vocabulary}
return vocabulary_config if vocabulary else _EMPTY_TRANSCRIPTION_CONFIG
def _build_transcription_config(
optional_params: Mapping[str, object],
drop_params: bool = False,
) -> GeminiTranscriptionConfig:
vocabulary_config: Final = _custom_vocabulary_config(optional_params.get("keywords"))
timestamp_config: Final = _timestamp_config(
optional_params.get("timestamp_granularities"),
optional_params.get("response_format"),
)
has_conflicting_params: Final = bool(vocabulary_config and timestamp_config)
if has_conflicting_params and not drop_params:
raise UnsupportedParamsError(
status_code=400,
message="Gemini transcription does not support custom vocabulary with word timestamps.",
)
resolved_vocabulary_config: Final = _EMPTY_TRANSCRIPTION_CONFIG if has_conflicting_params else vocabulary_config
transcription_config: Final[GeminiTranscriptionConfig] = {
**_language_config(optional_params.get("language")),
**_timestamp_config(optional_params.get("timestamp_granularities"), optional_params.get("response_format")),
**resolved_vocabulary_config,
**timestamp_config,
}
return transcription_config

View file

@ -12,6 +12,10 @@ from typing_extensions import ReadOnly, Required, TypedDict
import litellm
from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.audio_utils.utils import (
normalize_transcription_keywords,
normalize_transcription_language_to_bcp47,
)
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
@ -109,12 +113,33 @@ class _OpenAIRealtimeClientEvent(TypedDict, total=False):
item: ReadOnly[dict[str, object]]
class _GeminiLiveInputAudioTranscription(TypedDict, total=False):
languageCodes: ReadOnly[tuple[str, ...]]
customVocabulary: ReadOnly[tuple[str, ...]]
def _parse_setup(session_configuration_request: str) -> BidiGenerateContentSetup:
envelope: Final[_GeminiLiveSetupEnvelope] = json.loads(session_configuration_request)
empty_setup: Final[BidiGenerateContentSetup] = {}
return envelope.get("setup", empty_setup)
def _map_input_audio_transcription(value: object) -> _GeminiLiveInputAudioTranscription:
empty_config: Final[_GeminiLiveInputAudioTranscription] = {}
if not isinstance(value, Mapping):
return empty_config
language: Final = value.get("language")
language_codes: Final = (
(normalize_transcription_language_to_bcp47(language),) if isinstance(language, str) and language else ()
)
custom_vocabulary: Final = normalize_transcription_keywords(value.get("keywords"))
transcription_config: Final[_GeminiLiveInputAudioTranscription] = {
**({"languageCodes": language_codes} if language_codes else {}),
**({"customVocabulary": custom_vocabulary} if custom_vocabulary else {}),
}
return transcription_config
def _grounding_metadata_from_frame(frame: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
"""Read ``serverContent.groundingMetadata`` off the frame that carries the turn's usage.
@ -335,7 +360,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
value=value, optional_params=optional_params
)
elif key == "input_audio_transcription" and value is not None:
optional_params["inputAudioTranscription"] = {}
optional_params["inputAudioTranscription"] = _map_input_audio_transcription(value)
elif key == "turn_detection" and value is not None:
value_typed = cast(OpenAIRealtimeTurnDetection, value)
if (

View file

@ -7846,6 +7846,7 @@ def transcription(
response_format: Literal["json", "text", "srt", "verbose_json", "vtt"] | None = None,
timestamp_granularities: list[Literal["word", "segment"]] | None = None,
temperature: int | None = None, # openai defaults this to 0
keywords: list[str] | None = None,
## LITELLM PARAMS ##
user: str | None = None,
timeout=600, # default to 10 minutes
@ -7896,6 +7897,7 @@ def transcription(
optional_params: Final = get_optional_params_transcription(
model=model,
language=language,
keywords=keywords,
prompt=prompt,
response_format=response_format,
timestamp_granularities=timestamp_granularities,

View file

@ -62715,6 +62715,7 @@
"tpm": 250000
},
"gemini/gemini-3.5-transcribe": {
"audio_transcription_config": "gemini_transcribe",
"input_cost_per_audio_token": 2e-06,
"input_cost_per_token": 2e-06,
"litellm_provider": "gemini",

View file

@ -17,6 +17,7 @@ class GeminiTranscriptionVerbatimMode(TypedDict, total=False):
class GeminiTranscriptionConfig(TypedDict, total=False):
custom_vocabulary: ReadOnly[tuple[str, ...]]
language_codes: ReadOnly[tuple[str, ...]]
mode: ReadOnly[GeminiTranscriptionVerbatimMode]

View file

@ -1154,6 +1154,7 @@ AllEmbeddingInputValues = str | list[str] | list[int] | list[list[int]]
OpenAIAudioTranscriptionOptionalParams = Literal[
"language",
"keywords",
"prompt",
"temperature",
"response_format",

View file

@ -3428,6 +3428,7 @@ def get_optional_params_transcription(
temperature: int | None = None,
timestamp_granularities: list[Literal["word", "segment"]] | None = None,
drop_params: bool | None = None,
keywords: list[str] | None = None,
**kwargs,
):
from litellm.constants import OPENAI_TRANSCRIPTION_PARAMS
@ -3444,6 +3445,7 @@ def get_optional_params_transcription(
default_params: Final = {
"language": None,
"keywords": None,
"prompt": None,
"response_format": None,
"temperature": None, # openai defaults this to 0
@ -6256,8 +6258,6 @@ def _build_model_info(
api_base: str | None = None,
api_key: str | None = None,
) -> ModelInfo:
supported_openai_params = litellm.get_supported_openai_params(model=model, custom_llm_provider=custom_llm_provider)
_model_info: Final = _get_model_info_helper(
model=model,
custom_llm_provider=custom_llm_provider,
@ -6271,6 +6271,18 @@ def _build_model_info(
if value is not None:
_model_info[key] = value
request_type_by_mode: Final = MappingProxyType(
{
"embedding": "embeddings",
"audio_transcription": "transcription",
}
)
supported_openai_params = litellm.get_supported_openai_params(
model=model,
custom_llm_provider=custom_llm_provider,
request_type=request_type_by_mode.get(_model_info.get("mode"), "chat_completion"),
)
# if verbose_logger.isEnabledFor(logging.DEBUG):
# verbose_logger.debug(f"model_info: {_model_info}")
@ -8939,7 +8951,10 @@ class ProviderConfigManager:
)
return VertexAIAudioTranscriptionConfig()
elif litellm.LlmProviders.GEMINI == provider:
elif (
litellm.LlmProviders.GEMINI == provider
and model_cost_entry.get("audio_transcription_config") == "gemini_transcribe"
):
from litellm.llms.gemini.audio_transcription.transformation import (
GeminiAudioTranscriptionConfig,
)

View file

@ -62715,6 +62715,7 @@
"tpm": 250000
},
"gemini/gemini-3.5-transcribe": {
"audio_transcription_config": "gemini_transcribe",
"input_cost_per_audio_token": 2e-06,
"input_cost_per_token": 2e-06,
"litellm_provider": "gemini",

View file

@ -2110,6 +2110,44 @@ def test_get_provider_audio_transcription_config():
)
def test_get_supported_openai_params_uses_gemini_transcription_config():
from litellm.litellm_core_utils.get_supported_openai_params import (
get_supported_openai_params,
)
params = get_supported_openai_params(
model="gemini-3.5-transcribe",
custom_llm_provider="gemini",
request_type="transcription",
)
assert params is not None
assert "language" in params
assert "keywords" in params
assert "max_completion_tokens" not in params
def test_gemini_model_info_reports_transcription_params(monkeypatch):
from litellm.utils import _build_model_info
monkeypatch.setitem(
litellm.model_cost,
"gemini/gemini-3.5-transcribe",
{
"audio_transcription_config": "gemini_transcribe",
"mode": "audio_transcription",
"supported_endpoints": ["/v1/audio/transcriptions"],
},
)
model_info = _build_model_info(
model="gemini-3.5-transcribe",
custom_llm_provider="gemini",
)
assert "keywords" in model_info["supported_openai_params"]
@pytest.mark.parametrize(
"model, expected_bool",
[

View file

@ -10,7 +10,7 @@ from litellm.llms.gemini.audio_transcription.transformation import (
)
from litellm.llms.gemini.common_utils import GeminiError
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager
from litellm.utils import ProviderConfigManager, get_optional_params_transcription
AUDIO_BYTES = b"RIFF....WAVEfmt fake-wav-bytes"
@ -72,6 +72,23 @@ def test_provider_config_manager_returns_gemini_config():
assert isinstance(provider_config, GeminiAudioTranscriptionConfig)
def test_provider_config_manager_does_not_use_batch_config_for_gemini_live():
provider_config = ProviderConfigManager.get_provider_audio_transcription_config(
model="gemini-3.5-transcribe-live", provider=LlmProviders.GEMINI
)
assert provider_config is None
class TestSupportedParams:
def test_keywords_are_advertised(self, config):
assert config.get_supported_openai_params("gemini-3.5-transcribe") == [
"language",
"keywords",
"response_format",
"timestamp_granularities",
]
class TestValidateEnvironment:
def test_sets_api_key_and_revision_headers(self, config):
headers = config.validate_environment(
@ -152,6 +169,57 @@ class TestTransformRequest:
transcription_config = request_data.data["generation_config"]["transcription_config"]
assert json.loads(json.dumps(transcription_config)) == {"language_codes": ["en-US"]}
def test_keywords_map_to_custom_vocabulary(self, config):
optional_params = get_optional_params_transcription(
model="gemini-3.5-transcribe",
custom_llm_provider="gemini",
keywords=["alpha", "beta"],
)
request_data = config.transform_audio_transcription_request(
model="gemini-3.5-transcribe",
audio_file=("sample.wav", AUDIO_BYTES, "audio/wav"),
optional_params=optional_params,
litellm_params={},
)
transcription_config = request_data.data["generation_config"]["transcription_config"]
assert json.loads(json.dumps(transcription_config)) == {
"custom_vocabulary": ["alpha", "beta"]
}
def test_keywords_with_word_timestamps_raise(self, config, monkeypatch):
monkeypatch.setattr(litellm, "drop_params", False)
with pytest.raises(litellm.UnsupportedParamsError, match="custom vocabulary with word timestamps"):
config.transform_audio_transcription_request(
model="gemini-3.5-transcribe",
audio_file=("sample.wav", AUDIO_BYTES, "audio/wav"),
optional_params={
"keywords": ["alpha"],
"timestamp_granularities": ["word"],
},
litellm_params={},
)
def test_drop_params_removes_keywords_with_word_timestamps(self, config, monkeypatch):
monkeypatch.setattr(litellm, "drop_params", False)
request_data = config.transform_audio_transcription_request(
model="gemini-3.5-transcribe",
audio_file=("sample.wav", AUDIO_BYTES, "audio/wav"),
optional_params={
"keywords": ["alpha"],
"timestamp_granularities": ["word"],
},
litellm_params={"drop_params": True},
)
transcription_config = request_data.data["generation_config"]["transcription_config"]
assert "custom_vocabulary" not in transcription_config
assert json.loads(json.dumps(transcription_config["mode"])) == {
"type": "verbatim",
"timestamp_granularities": ["word"],
"diarization_mode": "speaker",
}
def test_word_timestamp_granularity_maps_to_verbatim_diarization_mode(self, config):
request_data = config.transform_audio_transcription_request(
model="gemini-3.5-transcribe",

View file

@ -839,6 +839,31 @@ def test_gemini_realtime_multi_tool_calls_have_unique_item_ids():
assert responses[1]["output_index"] == 1
def test_gemini_session_update_maps_input_audio_transcription_settings():
config = GeminiRealtimeConfig()
session_update = {
"type": "session.update",
"session": {
"input_audio_transcription": {
"language": "pl",
"keywords": ["alpha", "beta"],
},
},
}
result = config.transform_realtime_request(
json.dumps(session_update),
"gemini-3.5-transcribe-live",
session_configuration_request=None,
)
setup = json.loads(result[0])
assert setup["setup"]["inputAudioTranscription"] == {
"languageCodes": ["pl-PL"],
"customVocabulary": ["alpha", "beta"],
}
def test_gemini_session_update_includes_input_audio_transcription_default():
"""Verify _handle_session_update includes inputAudioTranscription default."""
config = GeminiRealtimeConfig()