From fd023cc46af2f70f8db88f156aa9629d288e8ec5 Mon Sep 17 00:00:00 2001 From: Marcin Juraszek <10106352+korasino@users.noreply.github.com> Date: Tue, 22 Sep 2026 21:36:26 +0200 Subject: [PATCH] feat(audio): add Gemini transcription vocabulary support --- litellm/constants.py | 1 + .../litellm_core_utils/audio_utils/utils.py | 8 +++ .../get_supported_openai_params.py | 12 +++- .../audio_transcription/transformation.py | 41 +++++++++-- .../llms/gemini/realtime/transformation.py | 27 ++++++- litellm/main.py | 2 + ...odel_prices_and_context_window_backup.json | 1 + .../types/llms/gemini_audio_transcription.py | 1 + litellm/types/llms/openai.py | 1 + litellm/utils.py | 21 +++++- model_prices_and_context_window.json | 1 + tests/litellm_utils_tests/test_utils.py | 38 ++++++++++ ...mini_audio_transcription_transformation.py | 70 ++++++++++++++++++- .../test_gemini_realtime_transformation.py | 25 +++++++ 14 files changed, 239 insertions(+), 10 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 1012ca831f2..2bf00dc19e1 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -825,6 +825,7 @@ OPENAI_CHAT_COMPLETION_PARAMS: Final = [ OPENAI_TRANSCRIPTION_PARAMS: Final = [ "language", + "keywords", "response_format", "timestamp_granularities", ] diff --git a/litellm/litellm_core_utils/audio_utils/utils.py b/litellm/litellm_core_utils/audio_utils/utils.py index dab3e48f91a..87d4fb0c986 100644 --- a/litellm/litellm_core_utils/audio_utils/utils.py +++ b/litellm/litellm_core_utils/audio_utils/utils.py @@ -4,6 +4,7 @@ Utils used for litellm.transcription() and litellm.atranscription() import hashlib import os +from collections.abc import Sequence from dataclasses import dataclass from typing import Final @@ -142,6 +143,7 @@ BARE_ISO_639_1_TO_BCP47: Final = { "ru": "ru-RU", "hi": "hi-IN", "ar": "ar-SA", + "pl": "pl-PL", } @@ -157,6 +159,12 @@ def normalize_transcription_language_to_bcp47(language: str) -> str: return BARE_ISO_639_1_TO_BCP47.get(language.lower(), language) +def normalize_transcription_keywords(keywords: object) -> tuple[str, ...]: + if not isinstance(keywords, Sequence) or isinstance(keywords, (str, bytes)): + return () + return tuple(keyword for keyword in keywords if isinstance(keyword, str) and keyword) + + def get_audio_file_name(file_obj: FileTypes) -> str: """ Safely get the name of a file-like object or return its string representation. diff --git a/litellm/litellm_core_utils/get_supported_openai_params.py b/litellm/litellm_core_utils/get_supported_openai_params.py index 08b8816e17d..3020718cc30 100644 --- a/litellm/litellm_core_utils/get_supported_openai_params.py +++ b/litellm/litellm_core_utils/get_supported_openai_params.py @@ -183,7 +183,17 @@ def get_supported_openai_params( return litellm.DatabricksConfig().get_supported_openai_params(model=model) elif request_type == "embeddings": return litellm.DatabricksEmbeddingConfig().get_supported_openai_params() - elif custom_llm_provider == "palm" or custom_llm_provider == "gemini": + elif custom_llm_provider == "palm": + return litellm.GoogleAIStudioGeminiConfig().get_supported_openai_params(model=model) + elif custom_llm_provider == "gemini": + if request_type == "transcription": + transcription_provider_config = litellm.ProviderConfigManager.get_provider_audio_transcription_config( + model=model, + provider=LlmProviders.GEMINI, + ) + if transcription_provider_config: + return transcription_provider_config.get_supported_openai_params(model=model) + return None return litellm.GoogleAIStudioGeminiConfig().get_supported_openai_params(model=model) elif custom_llm_provider == "novita": return litellm.NovitaConfig().get_supported_openai_params(model=model) diff --git a/litellm/llms/gemini/audio_transcription/transformation.py b/litellm/llms/gemini/audio_transcription/transformation.py index c8dd7a9a5ff..79e8603be50 100644 --- a/litellm/llms/gemini/audio_transcription/transformation.py +++ b/litellm/llms/gemini/audio_transcription/transformation.py @@ -4,8 +4,11 @@ from typing import Final from httpx import Headers, Response +import litellm +from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.audio_utils.subtitle_utils import SUBTITLE_RESPONSE_FORMATS from litellm.litellm_core_utils.audio_utils.utils import ( + normalize_transcription_keywords, normalize_transcription_language_to_bcp47, process_audio_file, ) @@ -47,7 +50,12 @@ class GeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig): def get_supported_openai_params( self, model: str ) -> list[OpenAIAudioTranscriptionOptionalParams]: # mutable-ok: BaseAudioTranscriptionConfig signature - return ["language", "response_format", "timestamp_granularities"] # mutable-ok: base contract returns a list + return [ # mutable-ok: BaseAudioTranscriptionConfig requires a list + "language", + "keywords", + "response_format", + "timestamp_granularities", + ] @property def supports_subtitle_synthesis(self) -> bool: @@ -123,7 +131,10 @@ class GeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig): request: Final = _build_interaction_request( model=model, audio_input=audio_input, - transcription_config=_build_transcription_config(optional_params), + transcription_config=_build_transcription_config( + optional_params, + drop_params=litellm_params.get("drop_params") is True or bool(litellm.drop_params), + ), ) return AudioTranscriptionRequestData(data=dict(request)) # mutable-ok: AudioTranscriptionRequestData wants dict @@ -227,10 +238,32 @@ def _timestamp_config(timestamp_granularities: object, response_format: object) return _WORD_TIMESTAMP_CONFIG if wants_word_timestamps else _EMPTY_TRANSCRIPTION_CONFIG -def _build_transcription_config(optional_params: Mapping[str, object]) -> GeminiTranscriptionConfig: +def _custom_vocabulary_config(keywords: object) -> GeminiTranscriptionConfig: + vocabulary: Final = normalize_transcription_keywords(keywords) + vocabulary_config: Final[GeminiTranscriptionConfig] = {"custom_vocabulary": vocabulary} + return vocabulary_config if vocabulary else _EMPTY_TRANSCRIPTION_CONFIG + + +def _build_transcription_config( + optional_params: Mapping[str, object], + drop_params: bool = False, +) -> GeminiTranscriptionConfig: + vocabulary_config: Final = _custom_vocabulary_config(optional_params.get("keywords")) + timestamp_config: Final = _timestamp_config( + optional_params.get("timestamp_granularities"), + optional_params.get("response_format"), + ) + has_conflicting_params: Final = bool(vocabulary_config and timestamp_config) + if has_conflicting_params and not drop_params: + raise UnsupportedParamsError( + status_code=400, + message="Gemini transcription does not support custom vocabulary with word timestamps.", + ) + resolved_vocabulary_config: Final = _EMPTY_TRANSCRIPTION_CONFIG if has_conflicting_params else vocabulary_config transcription_config: Final[GeminiTranscriptionConfig] = { **_language_config(optional_params.get("language")), - **_timestamp_config(optional_params.get("timestamp_granularities"), optional_params.get("response_format")), + **resolved_vocabulary_config, + **timestamp_config, } return transcription_config diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index e64cbf88d95..aafb9dc3946 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -12,6 +12,10 @@ from typing_extensions import ReadOnly, Required, TypedDict import litellm from litellm import verbose_logger from litellm._uuid import uuid +from litellm.litellm_core_utils.audio_utils.utils import ( + normalize_transcription_keywords, + normalize_transcription_language_to_bcp47, +) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( @@ -109,12 +113,33 @@ class _OpenAIRealtimeClientEvent(TypedDict, total=False): item: ReadOnly[dict[str, object]] +class _GeminiLiveInputAudioTranscription(TypedDict, total=False): + languageCodes: ReadOnly[tuple[str, ...]] + customVocabulary: ReadOnly[tuple[str, ...]] + + def _parse_setup(session_configuration_request: str) -> BidiGenerateContentSetup: envelope: Final[_GeminiLiveSetupEnvelope] = json.loads(session_configuration_request) empty_setup: Final[BidiGenerateContentSetup] = {} return envelope.get("setup", empty_setup) +def _map_input_audio_transcription(value: object) -> _GeminiLiveInputAudioTranscription: + empty_config: Final[_GeminiLiveInputAudioTranscription] = {} + if not isinstance(value, Mapping): + return empty_config + language: Final = value.get("language") + language_codes: Final = ( + (normalize_transcription_language_to_bcp47(language),) if isinstance(language, str) and language else () + ) + custom_vocabulary: Final = normalize_transcription_keywords(value.get("keywords")) + transcription_config: Final[_GeminiLiveInputAudioTranscription] = { + **({"languageCodes": language_codes} if language_codes else {}), + **({"customVocabulary": custom_vocabulary} if custom_vocabulary else {}), + } + return transcription_config + + def _grounding_metadata_from_frame(frame: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: """Read ``serverContent.groundingMetadata`` off the frame that carries the turn's usage. @@ -335,7 +360,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): value=value, optional_params=optional_params ) elif key == "input_audio_transcription" and value is not None: - optional_params["inputAudioTranscription"] = {} + optional_params["inputAudioTranscription"] = _map_input_audio_transcription(value) elif key == "turn_detection" and value is not None: value_typed = cast(OpenAIRealtimeTurnDetection, value) if ( diff --git a/litellm/main.py b/litellm/main.py index 7d231a1bf7a..54e7aac0227 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -7846,6 +7846,7 @@ def transcription( response_format: Literal["json", "text", "srt", "verbose_json", "vtt"] | None = None, timestamp_granularities: list[Literal["word", "segment"]] | None = None, temperature: int | None = None, # openai defaults this to 0 + keywords: list[str] | None = None, ## LITELLM PARAMS ## user: str | None = None, timeout=600, # default to 10 minutes @@ -7896,6 +7897,7 @@ def transcription( optional_params: Final = get_optional_params_transcription( model=model, language=language, + keywords=keywords, prompt=prompt, response_format=response_format, timestamp_granularities=timestamp_granularities, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 8d68e325198..67b851a4093 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -62715,6 +62715,7 @@ "tpm": 250000 }, "gemini/gemini-3.5-transcribe": { + "audio_transcription_config": "gemini_transcribe", "input_cost_per_audio_token": 2e-06, "input_cost_per_token": 2e-06, "litellm_provider": "gemini", diff --git a/litellm/types/llms/gemini_audio_transcription.py b/litellm/types/llms/gemini_audio_transcription.py index f7e74ba4bf8..f663ee5063e 100644 --- a/litellm/types/llms/gemini_audio_transcription.py +++ b/litellm/types/llms/gemini_audio_transcription.py @@ -17,6 +17,7 @@ class GeminiTranscriptionVerbatimMode(TypedDict, total=False): class GeminiTranscriptionConfig(TypedDict, total=False): + custom_vocabulary: ReadOnly[tuple[str, ...]] language_codes: ReadOnly[tuple[str, ...]] mode: ReadOnly[GeminiTranscriptionVerbatimMode] diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 599b1a76249..5ecc5a0583f 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1154,6 +1154,7 @@ AllEmbeddingInputValues = str | list[str] | list[int] | list[list[int]] OpenAIAudioTranscriptionOptionalParams = Literal[ "language", + "keywords", "prompt", "temperature", "response_format", diff --git a/litellm/utils.py b/litellm/utils.py index b9d56b25653..6f77ce1a5bf 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3428,6 +3428,7 @@ def get_optional_params_transcription( temperature: int | None = None, timestamp_granularities: list[Literal["word", "segment"]] | None = None, drop_params: bool | None = None, + keywords: list[str] | None = None, **kwargs, ): from litellm.constants import OPENAI_TRANSCRIPTION_PARAMS @@ -3444,6 +3445,7 @@ def get_optional_params_transcription( default_params: Final = { "language": None, + "keywords": None, "prompt": None, "response_format": None, "temperature": None, # openai defaults this to 0 @@ -6256,8 +6258,6 @@ def _build_model_info( api_base: str | None = None, api_key: str | None = None, ) -> ModelInfo: - supported_openai_params = litellm.get_supported_openai_params(model=model, custom_llm_provider=custom_llm_provider) - _model_info: Final = _get_model_info_helper( model=model, custom_llm_provider=custom_llm_provider, @@ -6271,6 +6271,18 @@ def _build_model_info( if value is not None: _model_info[key] = value + request_type_by_mode: Final = MappingProxyType( + { + "embedding": "embeddings", + "audio_transcription": "transcription", + } + ) + supported_openai_params = litellm.get_supported_openai_params( + model=model, + custom_llm_provider=custom_llm_provider, + request_type=request_type_by_mode.get(_model_info.get("mode"), "chat_completion"), + ) + # if verbose_logger.isEnabledFor(logging.DEBUG): # verbose_logger.debug(f"model_info: {_model_info}") @@ -8939,7 +8951,10 @@ class ProviderConfigManager: ) return VertexAIAudioTranscriptionConfig() - elif litellm.LlmProviders.GEMINI == provider: + elif ( + litellm.LlmProviders.GEMINI == provider + and model_cost_entry.get("audio_transcription_config") == "gemini_transcribe" + ): from litellm.llms.gemini.audio_transcription.transformation import ( GeminiAudioTranscriptionConfig, ) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 8d68e325198..67b851a4093 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -62715,6 +62715,7 @@ "tpm": 250000 }, "gemini/gemini-3.5-transcribe": { + "audio_transcription_config": "gemini_transcribe", "input_cost_per_audio_token": 2e-06, "input_cost_per_token": 2e-06, "litellm_provider": "gemini", diff --git a/tests/litellm_utils_tests/test_utils.py b/tests/litellm_utils_tests/test_utils.py index f7575b969c4..2017e603af1 100644 --- a/tests/litellm_utils_tests/test_utils.py +++ b/tests/litellm_utils_tests/test_utils.py @@ -2110,6 +2110,44 @@ def test_get_provider_audio_transcription_config(): ) +def test_get_supported_openai_params_uses_gemini_transcription_config(): + from litellm.litellm_core_utils.get_supported_openai_params import ( + get_supported_openai_params, + ) + + params = get_supported_openai_params( + model="gemini-3.5-transcribe", + custom_llm_provider="gemini", + request_type="transcription", + ) + + assert params is not None + assert "language" in params + assert "keywords" in params + assert "max_completion_tokens" not in params + + +def test_gemini_model_info_reports_transcription_params(monkeypatch): + from litellm.utils import _build_model_info + + monkeypatch.setitem( + litellm.model_cost, + "gemini/gemini-3.5-transcribe", + { + "audio_transcription_config": "gemini_transcribe", + "mode": "audio_transcription", + "supported_endpoints": ["/v1/audio/transcriptions"], + }, + ) + + model_info = _build_model_info( + model="gemini-3.5-transcribe", + custom_llm_provider="gemini", + ) + + assert "keywords" in model_info["supported_openai_params"] + + @pytest.mark.parametrize( "model, expected_bool", [ diff --git a/tests/unit/llms/gemini/audio_transcription/test_gemini_audio_transcription_transformation.py b/tests/unit/llms/gemini/audio_transcription/test_gemini_audio_transcription_transformation.py index 08084c8fac0..ab2ee82b604 100644 --- a/tests/unit/llms/gemini/audio_transcription/test_gemini_audio_transcription_transformation.py +++ b/tests/unit/llms/gemini/audio_transcription/test_gemini_audio_transcription_transformation.py @@ -10,7 +10,7 @@ from litellm.llms.gemini.audio_transcription.transformation import ( ) from litellm.llms.gemini.common_utils import GeminiError from litellm.types.utils import LlmProviders -from litellm.utils import ProviderConfigManager +from litellm.utils import ProviderConfigManager, get_optional_params_transcription AUDIO_BYTES = b"RIFF....WAVEfmt fake-wav-bytes" @@ -72,6 +72,23 @@ def test_provider_config_manager_returns_gemini_config(): assert isinstance(provider_config, GeminiAudioTranscriptionConfig) +def test_provider_config_manager_does_not_use_batch_config_for_gemini_live(): + provider_config = ProviderConfigManager.get_provider_audio_transcription_config( + model="gemini-3.5-transcribe-live", provider=LlmProviders.GEMINI + ) + assert provider_config is None + + +class TestSupportedParams: + def test_keywords_are_advertised(self, config): + assert config.get_supported_openai_params("gemini-3.5-transcribe") == [ + "language", + "keywords", + "response_format", + "timestamp_granularities", + ] + + class TestValidateEnvironment: def test_sets_api_key_and_revision_headers(self, config): headers = config.validate_environment( @@ -152,6 +169,57 @@ class TestTransformRequest: transcription_config = request_data.data["generation_config"]["transcription_config"] assert json.loads(json.dumps(transcription_config)) == {"language_codes": ["en-US"]} + def test_keywords_map_to_custom_vocabulary(self, config): + optional_params = get_optional_params_transcription( + model="gemini-3.5-transcribe", + custom_llm_provider="gemini", + keywords=["alpha", "beta"], + ) + request_data = config.transform_audio_transcription_request( + model="gemini-3.5-transcribe", + audio_file=("sample.wav", AUDIO_BYTES, "audio/wav"), + optional_params=optional_params, + litellm_params={}, + ) + transcription_config = request_data.data["generation_config"]["transcription_config"] + assert json.loads(json.dumps(transcription_config)) == { + "custom_vocabulary": ["alpha", "beta"] + } + + def test_keywords_with_word_timestamps_raise(self, config, monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + + with pytest.raises(litellm.UnsupportedParamsError, match="custom vocabulary with word timestamps"): + config.transform_audio_transcription_request( + model="gemini-3.5-transcribe", + audio_file=("sample.wav", AUDIO_BYTES, "audio/wav"), + optional_params={ + "keywords": ["alpha"], + "timestamp_granularities": ["word"], + }, + litellm_params={}, + ) + + def test_drop_params_removes_keywords_with_word_timestamps(self, config, monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + + request_data = config.transform_audio_transcription_request( + model="gemini-3.5-transcribe", + audio_file=("sample.wav", AUDIO_BYTES, "audio/wav"), + optional_params={ + "keywords": ["alpha"], + "timestamp_granularities": ["word"], + }, + litellm_params={"drop_params": True}, + ) + transcription_config = request_data.data["generation_config"]["transcription_config"] + assert "custom_vocabulary" not in transcription_config + assert json.loads(json.dumps(transcription_config["mode"])) == { + "type": "verbatim", + "timestamp_granularities": ["word"], + "diarization_mode": "speaker", + } + def test_word_timestamp_granularity_maps_to_verbatim_diarization_mode(self, config): request_data = config.transform_audio_transcription_request( model="gemini-3.5-transcribe", diff --git a/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py b/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py index bcd5f3d8d19..fde06dfbc50 100644 --- a/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py +++ b/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py @@ -839,6 +839,31 @@ def test_gemini_realtime_multi_tool_calls_have_unique_item_ids(): assert responses[1]["output_index"] == 1 +def test_gemini_session_update_maps_input_audio_transcription_settings(): + config = GeminiRealtimeConfig() + session_update = { + "type": "session.update", + "session": { + "input_audio_transcription": { + "language": "pl", + "keywords": ["alpha", "beta"], + }, + }, + } + + result = config.transform_realtime_request( + json.dumps(session_update), + "gemini-3.5-transcribe-live", + session_configuration_request=None, + ) + + setup = json.loads(result[0]) + assert setup["setup"]["inputAudioTranscription"] == { + "languageCodes": ["pl-PL"], + "customVocabulary": ["alpha", "beta"], + } + + def test_gemini_session_update_includes_input_audio_transcription_default(): """Verify _handle_session_update includes inputAudioTranscription default.""" config = GeminiRealtimeConfig()