This commit is contained in:
Rakshit Jain 2026-10-24 08:31:31 +00:00 • committed by GitHub
commit 032e0ce4ba
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 201 additions and 5 deletions

View file

@ -0,0 +1,30 @@
# Gemini transcription language hints
Gemini transcription accepts multiple language hints through the provider-specific `language_codes` parameter. The existing OpenAI-compatible `language="en"` parameter still works for a single hint
```python
import litellm
with open("sample.wav", "rb") as audio:
transcript = litellm.transcription(
model="gemini/gemini-3.5-transcribe",
file=audio,
language_codes=["en-US", "es-ES"],
)
```
Through the LiteLLM proxy, send repeated `language_codes[]` multipart fields or a JSON-encoded list in `language_codes`:
```bash
curl http://localhost:4000/v1/audio/transcriptions \
-H "Authorization: Bearer $LITELLM_API_KEY" \
-F 'model=gemini/gemini-3.5-transcribe' \
-F 'file=@sample.wav' \
-F 'language_codes=["en-US", "es-ES"]'
```
`language_codes` takes precedence over `language` when supplied. An empty list enables automatic language detection, even if `language` is also set. Omit both parameters to use automatic detection as before
Bare language codes use the same BCP-47 normalization as `language`; explicit script and region variants are preserved. Invalid lists return an error instead of silently discarding hints. Word timestamps and speaker diarization can be used with multiple hints
These hints guide recognition; they do not restrict the transcript to the listed languages. See the [Gemini transcription documentation](https://ai.google.dev/gemini-api/docs/transcribe#language-detection-and-hints) for provider behavior

View file

@ -1,8 +1,9 @@
import base64
from collections.abc import Mapping, Sequence
from typing import Final
from typing import Annotated, Final
from httpx import Headers, Response
from pydantic import StringConstraints, TypeAdapter, ValidationError
from litellm.litellm_core_utils.audio_utils.subtitle_utils import SUBTITLE_RESPONSE_FORMATS
from litellm.litellm_core_utils.audio_utils.utils import (
@ -211,7 +212,25 @@ def _build_interaction_request(
return configured_request
def _language_config(language: object) -> GeminiTranscriptionConfig:
_LANGUAGE_CODES_ADAPTER: Final = TypeAdapter(
tuple[Annotated[str, StringConstraints(strict=True, strip_whitespace=True, min_length=1)], ...]
)
def _language_config(language: object, language_codes: object) -> GeminiTranscriptionConfig:
if language_codes is not None:
try:
codes: Final = (
_LANGUAGE_CODES_ADAPTER.validate_json(language_codes)
if isinstance(language_codes, str)
else _LANGUAGE_CODES_ADAPTER.validate_python(language_codes)
)
except ValidationError:
raise GeminiError(status_code=400, message="language_codes must be a list of non-empty language strings")
multiple_language_config: Final[GeminiTranscriptionConfig] = {
"language_codes": tuple(normalize_transcription_language_to_bcp47(code) for code in codes),
}
return multiple_language_config
if not isinstance(language, str) or not language:
return _EMPTY_TRANSCRIPTION_CONFIG
language_config: Final[GeminiTranscriptionConfig] = {
@ -229,7 +248,7 @@ def _timestamp_config(timestamp_granularities: object, response_format: object)
def _build_transcription_config(optional_params: Mapping[str, object]) -> GeminiTranscriptionConfig:
transcription_config: Final[GeminiTranscriptionConfig] = {
**_language_config(optional_params.get("language")),
**_language_config(optional_params.get("language"), optional_params.get("language_codes")),
**_timestamp_config(optional_params.get("timestamp_granularities"), optional_params.get("response_format")),
}
return transcription_config

View file

@ -1,5 +1,7 @@
import base64
import json
from collections.abc import Mapping
from typing import Final
import httpx
import pytest
@ -10,7 +12,7 @@ from litellm.llms.gemini.audio_transcription.transformation import (
)
from litellm.llms.gemini.common_utils import GeminiError
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager
from litellm.utils import ProviderConfigManager, get_optional_params_transcription
AUDIO_BYTES = b"RIFF....WAVEfmt fake-wav-bytes"
@ -123,6 +125,76 @@ class TestGetCompleteUrl:
class TestTransformRequest:
@pytest.mark.parametrize(
("params", "expected_codes"),
[
({"language_codes": ["en", "es-ES"]}, ["en-US", "es-ES"]),
({"language_codes": '["en", "es-ES"]'}, ["en-US", "es-ES"]),
({"language_codes": [" en ", " es-ES "]}, ["en-US", "es-ES"]),
({"language_codes": ["zh-Hant-TW", "zh-Hans-CN"]}, ["zh-Hant-TW", "zh-Hans-CN"]),
({"language": "fr", "language_codes": ["en", "es"]}, ["en-US", "es-ES"]),
({"language": "fr", "language_codes": []}, []),
({"language": "fr", "language_codes": "[]"}, []),
({"language": "fr", "language_codes": None}, ["fr-FR"]),
],
)
def test_language_codes_reach_gemini_through_provider_params(
self,
config: GeminiAudioTranscriptionConfig,
params: Mapping[str, object],
expected_codes: list[str],
) -> None:
optional_params: Final = get_optional_params_transcription(
model="gemini-3.5-transcribe", custom_llm_provider="gemini", **params
)
request_data: Final = config.transform_audio_transcription_request(
model="gemini-3.5-transcribe",
audio_file=("sample.wav", AUDIO_BYTES, "audio/wav"),
optional_params=optional_params,
litellm_params={},
)
assert json.loads(json.dumps(request_data.data)) == {
"model": "gemini-3.5-transcribe",
"input": [
{
"type": "audio",
"data": base64.b64encode(AUDIO_BYTES).decode("utf-8"),
"mime_type": "audio/wav",
}
],
"generation_config": {"transcription_config": {"language_codes": expected_codes}},
}
@pytest.mark.parametrize(
"language_codes",
["en", '"en"', "[", "null", 1, {}, ["en", 1], ["en", ""], [" "], '["en", null]'],
)
def test_invalid_language_codes_raise_instead_of_silently_dropping_hints(
self, config: GeminiAudioTranscriptionConfig, language_codes: object
) -> None:
with pytest.raises(GeminiError, match="language_codes must be a list of non-empty language strings") as excinfo:
config.transform_audio_transcription_request(
model="gemini-3.5-transcribe",
audio_file=("sample.wav", AUDIO_BYTES, "audio/wav"),
optional_params={"language_codes": language_codes},
litellm_params={},
)
assert excinfo.value.status_code == 400
def test_multiple_language_codes_preserve_word_timestamps(self, config: GeminiAudioTranscriptionConfig) -> None:
request_data: Final = config.transform_audio_transcription_request(
model="gemini-3.5-transcribe",
audio_file=("sample.wav", AUDIO_BYTES, "audio/wav"),
optional_params={"language_codes": ["en-US", "es-ES"], "timestamp_granularities": ["word"]},
litellm_params={},
)
assert json.loads(json.dumps(request_data.data["generation_config"])) == {
"transcription_config": {
"language_codes": ["en-US", "es-ES"],
"mode": {"type": "verbatim", "timestamp_granularities": ["word"], "diarization_mode": "speaker"},
}
}
def test_builds_json_interaction_request(self, config):
request_data = config.transform_audio_transcription_request(
model="gemini/gemini-3.5-transcribe",

View file

@ -11,9 +11,10 @@ import contextlib
import os
import sys
from pathlib import Path
from typing import Any, AsyncIterator, Callable, Dict, Iterator, List, Optional
from typing import Any, AsyncIterator, Callable, Dict, Final, Iterator, List, Optional
from unittest.mock import AsyncMock, MagicMock
import httpx
import pytest
# Repo root, anchored to this file (not CWD) so the path is correct no
@ -536,3 +537,41 @@ def reset_login_throttle(monkeypatch):
_drop_throttle_keys()
yield _drop_throttle_keys
_drop_throttle_keys()
@pytest.fixture
async def gemini_transcription_requests(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[list[httpx.Request]]:
import litellm
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.proxy import proxy_server
requests: Final[list[httpx.Request]] = [] # mutable-ok: captures requests to the injected provider transport
def respond(request: httpx.Request) -> httpx.Response:
requests.append(request)
return httpx.Response(
200,
json={
"id": "test-interaction",
"status": "completed",
"steps": [{"type": "model_generation", "content": [{"type": "text", "text": "Hello world."}]}],
},
)
handler: Final = AsyncHTTPHandler(transport=httpx.MockTransport(respond))
router: Final = litellm.Router(
model_list=[
{
"model_name": "gemini-transcribe",
"litellm_params": {"model": "gemini/gemini-3.5-transcribe", "api_key": "test-key"},
}
],
num_retries=0,
default_litellm_params={"client": handler},
)
monkeypatch.setattr(proxy_server, "llm_router", router)
try:
yield requests
finally:
router.discard()
await handler.close()

View file

@ -10,12 +10,18 @@ Pins (PR2):
from __future__ import annotations
import io
import json
from collections.abc import Callable
from contextlib import AbstractContextManager
from typing import Final
from unittest.mock import AsyncMock, MagicMock
import httpx
import pytest
from fastapi.testclient import TestClient
from litellm.proxy import proxy_server
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.llms.openai import HttpxBinaryResponseContent
@ -254,3 +260,33 @@ def test_audio_transcription_error(client, auth_as, patched_transcription_error,
response = client.post(path, files=files, data=data)
assert response.status_code == 500
assert len(response.content) > 0
@pytest.mark.parametrize("path", ["/v1/audio/transcriptions", "/audio/transcriptions"])
def test_repeated_language_fields_reach_gemini_through_proxy(
client: TestClient,
auth_as: Callable[..., AbstractContextManager[UserAPIKeyAuth]],
gemini_transcription_requests: list[httpx.Request],
path: str,
) -> None:
with auth_as():
response: Final = client.post(
path,
files=[
("file", ("audio.wav", b"RIFF....WAVEfmt synthetic-audio", "audio/wav")),
("model", (None, "gemini-transcribe")),
("language", (None, "fr")),
("language_codes[]", (None, "en-US")),
("language_codes[]", (None, "es-ES")),
("timestamp_granularities[]", (None, "word")),
],
)
assert response.status_code == 200, response.text
assert response.json()["text"] == "Hello world."
assert len(gemini_transcription_requests) == 1
assert json.loads(gemini_transcription_requests[0].content)["generation_config"] == {
"transcription_config": {
"language_codes": ["en-US", "es-ES"],
"mode": {"type": "verbatim", "timestamp_granularities": ["word"], "diarization_mode": "speaker"},
}
}