From 50dc4ce39735fb67fa15b1efa90c1eb40abdc64e Mon Sep 17 00:00:00 2001 From: Rakshit Jain <63701425+the-quantum-nargle@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:37:04 +0800 Subject: [PATCH 1/4] feat(gemini): support multiple transcription language hints --- .../llms/gemini/audio_transcription/README.md | 30 ++++++++ .../audio_transcription/transformation.py | 25 ++++++- ...mini_audio_transcription_transformation.py | 74 ++++++++++++++++++- 3 files changed, 125 insertions(+), 4 deletions(-) create mode 100644 litellm/llms/gemini/audio_transcription/README.md diff --git a/litellm/llms/gemini/audio_transcription/README.md b/litellm/llms/gemini/audio_transcription/README.md new file mode 100644 index 00000000000..e4dd90bc6e0 --- /dev/null +++ b/litellm/llms/gemini/audio_transcription/README.md @@ -0,0 +1,30 @@ +# Gemini transcription language hints + +Gemini transcription accepts multiple language hints through the provider-specific `language_codes` parameter. The existing OpenAI-compatible `language="en"` parameter still works for a single hint + +```python +import litellm + +with open("sample.wav", "rb") as audio: + transcript = litellm.transcription( + model="gemini/gemini-3.5-transcribe", + file=audio, + language_codes=["en-US", "es-ES"], + ) +``` + +Through the LiteLLM proxy, send repeated `language_codes[]` multipart fields or a JSON-encoded list in `language_codes`: + +```bash +curl http://localhost:4000/v1/audio/transcriptions \ + -H "Authorization: Bearer $LITELLM_API_KEY" \ + -F 'model=gemini/gemini-3.5-transcribe' \ + -F 'file=@sample.wav' \ + -F 'language_codes=["en-US", "es-ES"]' +``` + +`language_codes` takes precedence over `language` when supplied. An empty list enables automatic language detection, even if `language` is also set. Omit both parameters to use automatic detection as before + +Bare language codes use the same BCP-47 normalization as `language`; explicit script and region variants are preserved. Invalid lists return an error instead of silently discarding hints. Word timestamps and speaker diarization can be used with multiple hints + +These hints guide recognition; they do not restrict the transcript to the listed languages. See the [Gemini transcription documentation](https://ai.google.dev/gemini-api/docs/transcribe#language-detection-and-hints) for provider behavior diff --git a/litellm/llms/gemini/audio_transcription/transformation.py b/litellm/llms/gemini/audio_transcription/transformation.py index 48a345a4940..3f8206858e0 100644 --- a/litellm/llms/gemini/audio_transcription/transformation.py +++ b/litellm/llms/gemini/audio_transcription/transformation.py @@ -1,8 +1,9 @@ import base64 from collections.abc import Mapping, Sequence -from typing import Final +from typing import Annotated, Final from httpx import Headers, Response +from pydantic import StringConstraints, TypeAdapter, ValidationError from litellm.litellm_core_utils.audio_utils.subtitle_utils import SUBTITLE_RESPONSE_FORMATS from litellm.litellm_core_utils.audio_utils.utils import ( @@ -211,7 +212,25 @@ def _build_interaction_request( return configured_request -def _language_config(language: object) -> GeminiTranscriptionConfig: +_LANGUAGE_CODES_ADAPTER: Final = TypeAdapter( + tuple[Annotated[str, StringConstraints(strict=True, strip_whitespace=True, min_length=1)], ...] +) + + +def _language_config(language: object, language_codes: object) -> GeminiTranscriptionConfig: + if language_codes is not None: + try: + codes: Final = ( + _LANGUAGE_CODES_ADAPTER.validate_json(language_codes) + if isinstance(language_codes, str) + else _LANGUAGE_CODES_ADAPTER.validate_python(language_codes) + ) + except ValidationError: + raise GeminiError(status_code=400, message="language_codes must be a list of non-empty language strings") + multiple_language_config: Final[GeminiTranscriptionConfig] = { + "language_codes": tuple(normalize_transcription_language_to_bcp47(code) for code in codes), + } + return multiple_language_config if not isinstance(language, str) or not language: return _EMPTY_TRANSCRIPTION_CONFIG language_config: Final[GeminiTranscriptionConfig] = { @@ -229,7 +248,7 @@ def _timestamp_config(timestamp_granularities: object, response_format: object) def _build_transcription_config(optional_params: Mapping[str, object]) -> GeminiTranscriptionConfig: transcription_config: Final[GeminiTranscriptionConfig] = { - **_language_config(optional_params.get("language")), + **_language_config(optional_params.get("language"), optional_params.get("language_codes")), **_timestamp_config(optional_params.get("timestamp_granularities"), optional_params.get("response_format")), } return transcription_config diff --git a/tests/unit/llms/gemini/audio_transcription/test_gemini_audio_transcription_transformation.py b/tests/unit/llms/gemini/audio_transcription/test_gemini_audio_transcription_transformation.py index 08084c8fac0..2dd36c46891 100644 --- a/tests/unit/llms/gemini/audio_transcription/test_gemini_audio_transcription_transformation.py +++ b/tests/unit/llms/gemini/audio_transcription/test_gemini_audio_transcription_transformation.py @@ -1,5 +1,7 @@ import base64 import json +from collections.abc import Mapping +from typing import Final import httpx import pytest @@ -10,7 +12,7 @@ from litellm.llms.gemini.audio_transcription.transformation import ( ) from litellm.llms.gemini.common_utils import GeminiError from litellm.types.utils import LlmProviders -from litellm.utils import ProviderConfigManager +from litellm.utils import ProviderConfigManager, get_optional_params_transcription AUDIO_BYTES = b"RIFF....WAVEfmt fake-wav-bytes" @@ -123,6 +125,76 @@ class TestGetCompleteUrl: class TestTransformRequest: + @pytest.mark.parametrize( + ("params", "expected_codes"), + [ + ({"language_codes": ["en", "es-ES"]}, ["en-US", "es-ES"]), + ({"language_codes": '["en", "es-ES"]'}, ["en-US", "es-ES"]), + ({"language_codes": [" en ", " es-ES "]}, ["en-US", "es-ES"]), + ({"language_codes": ["zh-Hant-TW", "zh-Hans-CN"]}, ["zh-Hant-TW", "zh-Hans-CN"]), + ({"language": "fr", "language_codes": ["en", "es"]}, ["en-US", "es-ES"]), + ({"language": "fr", "language_codes": []}, []), + ({"language": "fr", "language_codes": "[]"}, []), + ({"language": "fr", "language_codes": None}, ["fr-FR"]), + ], + ) + def test_language_codes_reach_gemini_through_provider_params( + self, + config: GeminiAudioTranscriptionConfig, + params: Mapping[str, object], + expected_codes: list[str], + ) -> None: + optional_params: Final = get_optional_params_transcription( + model="gemini-3.5-transcribe", custom_llm_provider="gemini", **params + ) + request_data: Final = config.transform_audio_transcription_request( + model="gemini-3.5-transcribe", + audio_file=("sample.wav", AUDIO_BYTES, "audio/wav"), + optional_params=optional_params, + litellm_params={}, + ) + assert json.loads(json.dumps(request_data.data)) == { + "model": "gemini-3.5-transcribe", + "input": [ + { + "type": "audio", + "data": base64.b64encode(AUDIO_BYTES).decode("utf-8"), + "mime_type": "audio/wav", + } + ], + "generation_config": {"transcription_config": {"language_codes": expected_codes}}, + } + + @pytest.mark.parametrize( + "language_codes", + ["en", '"en"', "[", "null", 1, {}, ["en", 1], ["en", ""], [" "], '["en", null]'], + ) + def test_invalid_language_codes_raise_instead_of_silently_dropping_hints( + self, config: GeminiAudioTranscriptionConfig, language_codes: object + ) -> None: + with pytest.raises(GeminiError, match="language_codes must be a list of non-empty language strings") as excinfo: + config.transform_audio_transcription_request( + model="gemini-3.5-transcribe", + audio_file=("sample.wav", AUDIO_BYTES, "audio/wav"), + optional_params={"language_codes": language_codes}, + litellm_params={}, + ) + assert excinfo.value.status_code == 400 + + def test_multiple_language_codes_preserve_word_timestamps(self, config: GeminiAudioTranscriptionConfig) -> None: + request_data: Final = config.transform_audio_transcription_request( + model="gemini-3.5-transcribe", + audio_file=("sample.wav", AUDIO_BYTES, "audio/wav"), + optional_params={"language_codes": ["en-US", "es-ES"], "timestamp_granularities": ["word"]}, + litellm_params={}, + ) + assert json.loads(json.dumps(request_data.data["generation_config"])) == { + "transcription_config": { + "language_codes": ["en-US", "es-ES"], + "mode": {"type": "verbatim", "timestamp_granularities": ["word"], "diarization_mode": "speaker"}, + } + } + def test_builds_json_interaction_request(self, config): request_data = config.transform_audio_transcription_request( model="gemini/gemini-3.5-transcribe", From cebc09791e6c6d32948f9c3117b7bc7dbe93444a Mon Sep 17 00:00:00 2001 From: Rakshit Jain <63701425+the-quantum-nargle@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:47:26 +0800 Subject: [PATCH 2/4] ci: retry checks after Codecov recovery From 9a90a86d1d4960308a3db6099870118333e0d7e4 Mon Sep 17 00:00:00 2001 From: Rakshit Jain <63701425+the-quantum-nargle@users.noreply.github.com> Date: Tue, 6 Oct 2026 11:02:29 +0800 Subject: [PATCH 3/4] ci: retry coverage uploads after Codecov incident resolution From d483d4fbf28dbb29ef549d8f939a257907fe8ac0 Mon Sep 17 00:00:00 2001 From: Rakshit Jain <63701425+the-quantum-nargle@users.noreply.github.com> Date: Tue, 6 Oct 2026 12:19:06 +0800 Subject: [PATCH 4/4] test(gemini): cover repeated transcription hints through proxy --- tests/unit/proxy/proxy_server/conftest.py | 41 ++++++++++++++++++- .../proxy/proxy_server/test_routes_audio.py | 36 ++++++++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/tests/unit/proxy/proxy_server/conftest.py b/tests/unit/proxy/proxy_server/conftest.py index 9baf3206fc4..f47c625093c 100644 --- a/tests/unit/proxy/proxy_server/conftest.py +++ b/tests/unit/proxy/proxy_server/conftest.py @@ -11,9 +11,10 @@ import contextlib import os import sys from pathlib import Path -from typing import Any, AsyncIterator, Callable, Dict, Iterator, List, Optional +from typing import Any, AsyncIterator, Callable, Dict, Final, Iterator, List, Optional from unittest.mock import AsyncMock, MagicMock +import httpx import pytest # Repo root, anchored to this file (not CWD) so the path is correct no @@ -536,3 +537,41 @@ def reset_login_throttle(monkeypatch): _drop_throttle_keys() yield _drop_throttle_keys _drop_throttle_keys() + + +@pytest.fixture +async def gemini_transcription_requests(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[list[httpx.Request]]: + import litellm + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + from litellm.proxy import proxy_server + + requests: Final[list[httpx.Request]] = [] # mutable-ok: captures requests to the injected provider transport + + def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response( + 200, + json={ + "id": "test-interaction", + "status": "completed", + "steps": [{"type": "model_generation", "content": [{"type": "text", "text": "Hello world."}]}], + }, + ) + + handler: Final = AsyncHTTPHandler(transport=httpx.MockTransport(respond)) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "gemini-transcribe", + "litellm_params": {"model": "gemini/gemini-3.5-transcribe", "api_key": "test-key"}, + } + ], + num_retries=0, + default_litellm_params={"client": handler}, + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + try: + yield requests + finally: + router.discard() + await handler.close() diff --git a/tests/unit/proxy/proxy_server/test_routes_audio.py b/tests/unit/proxy/proxy_server/test_routes_audio.py index de76c7257cf..4edf9653833 100644 --- a/tests/unit/proxy/proxy_server/test_routes_audio.py +++ b/tests/unit/proxy/proxy_server/test_routes_audio.py @@ -10,12 +10,18 @@ Pins (PR2): from __future__ import annotations import io +import json +from collections.abc import Callable +from contextlib import AbstractContextManager +from typing import Final from unittest.mock import AsyncMock, MagicMock import httpx import pytest +from fastapi.testclient import TestClient from litellm.proxy import proxy_server +from litellm.proxy._types import UserAPIKeyAuth from litellm.types.llms.openai import HttpxBinaryResponseContent @@ -254,3 +260,33 @@ def test_audio_transcription_error(client, auth_as, patched_transcription_error, response = client.post(path, files=files, data=data) assert response.status_code == 500 assert len(response.content) > 0 + + +@pytest.mark.parametrize("path", ["/v1/audio/transcriptions", "/audio/transcriptions"]) +def test_repeated_language_fields_reach_gemini_through_proxy( + client: TestClient, + auth_as: Callable[..., AbstractContextManager[UserAPIKeyAuth]], + gemini_transcription_requests: list[httpx.Request], + path: str, +) -> None: + with auth_as(): + response: Final = client.post( + path, + files=[ + ("file", ("audio.wav", b"RIFF....WAVEfmt synthetic-audio", "audio/wav")), + ("model", (None, "gemini-transcribe")), + ("language", (None, "fr")), + ("language_codes[]", (None, "en-US")), + ("language_codes[]", (None, "es-ES")), + ("timestamp_granularities[]", (None, "word")), + ], + ) + assert response.status_code == 200, response.text + assert response.json()["text"] == "Hello world." + assert len(gemini_transcription_requests) == 1 + assert json.loads(gemini_transcription_requests[0].content)["generation_config"] == { + "transcription_config": { + "language_codes": ["en-US", "es-ES"], + "mode": {"type": "verbatim", "timestamp_granularities": ["word"], "diarization_mode": "speaker"}, + } + }