mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-22 10:40:46 +00:00
Merge d483d4fbf2 into e04e89c701
This commit is contained in:
commit
032e0ce4ba
5 changed files with 201 additions and 5 deletions
30
litellm/llms/gemini/audio_transcription/README.md
Normal file
30
litellm/llms/gemini/audio_transcription/README.md
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
# Gemini transcription language hints
|
||||
|
||||
Gemini transcription accepts multiple language hints through the provider-specific `language_codes` parameter. The existing OpenAI-compatible `language="en"` parameter still works for a single hint
|
||||
|
||||
```python
|
||||
import litellm
|
||||
|
||||
with open("sample.wav", "rb") as audio:
|
||||
transcript = litellm.transcription(
|
||||
model="gemini/gemini-3.5-transcribe",
|
||||
file=audio,
|
||||
language_codes=["en-US", "es-ES"],
|
||||
)
|
||||
```
|
||||
|
||||
Through the LiteLLM proxy, send repeated `language_codes[]` multipart fields or a JSON-encoded list in `language_codes`:
|
||||
|
||||
```bash
|
||||
curl http://localhost:4000/v1/audio/transcriptions \
|
||||
-H "Authorization: Bearer $LITELLM_API_KEY" \
|
||||
-F 'model=gemini/gemini-3.5-transcribe' \
|
||||
-F 'file=@sample.wav' \
|
||||
-F 'language_codes=["en-US", "es-ES"]'
|
||||
```
|
||||
|
||||
`language_codes` takes precedence over `language` when supplied. An empty list enables automatic language detection, even if `language` is also set. Omit both parameters to use automatic detection as before
|
||||
|
||||
Bare language codes use the same BCP-47 normalization as `language`; explicit script and region variants are preserved. Invalid lists return an error instead of silently discarding hints. Word timestamps and speaker diarization can be used with multiple hints
|
||||
|
||||
These hints guide recognition; they do not restrict the transcript to the listed languages. See the [Gemini transcription documentation](https://ai.google.dev/gemini-api/docs/transcribe#language-detection-and-hints) for provider behavior
|
||||
|
|
@ -1,8 +1,9 @@
|
|||
import base64
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final
|
||||
from typing import Annotated, Final
|
||||
|
||||
from httpx import Headers, Response
|
||||
from pydantic import StringConstraints, TypeAdapter, ValidationError
|
||||
|
||||
from litellm.litellm_core_utils.audio_utils.subtitle_utils import SUBTITLE_RESPONSE_FORMATS
|
||||
from litellm.litellm_core_utils.audio_utils.utils import (
|
||||
|
|
@ -211,7 +212,25 @@ def _build_interaction_request(
|
|||
return configured_request
|
||||
|
||||
|
||||
def _language_config(language: object) -> GeminiTranscriptionConfig:
|
||||
_LANGUAGE_CODES_ADAPTER: Final = TypeAdapter(
|
||||
tuple[Annotated[str, StringConstraints(strict=True, strip_whitespace=True, min_length=1)], ...]
|
||||
)
|
||||
|
||||
|
||||
def _language_config(language: object, language_codes: object) -> GeminiTranscriptionConfig:
|
||||
if language_codes is not None:
|
||||
try:
|
||||
codes: Final = (
|
||||
_LANGUAGE_CODES_ADAPTER.validate_json(language_codes)
|
||||
if isinstance(language_codes, str)
|
||||
else _LANGUAGE_CODES_ADAPTER.validate_python(language_codes)
|
||||
)
|
||||
except ValidationError:
|
||||
raise GeminiError(status_code=400, message="language_codes must be a list of non-empty language strings")
|
||||
multiple_language_config: Final[GeminiTranscriptionConfig] = {
|
||||
"language_codes": tuple(normalize_transcription_language_to_bcp47(code) for code in codes),
|
||||
}
|
||||
return multiple_language_config
|
||||
if not isinstance(language, str) or not language:
|
||||
return _EMPTY_TRANSCRIPTION_CONFIG
|
||||
language_config: Final[GeminiTranscriptionConfig] = {
|
||||
|
|
@ -229,7 +248,7 @@ def _timestamp_config(timestamp_granularities: object, response_format: object)
|
|||
|
||||
def _build_transcription_config(optional_params: Mapping[str, object]) -> GeminiTranscriptionConfig:
|
||||
transcription_config: Final[GeminiTranscriptionConfig] = {
|
||||
**_language_config(optional_params.get("language")),
|
||||
**_language_config(optional_params.get("language"), optional_params.get("language_codes")),
|
||||
**_timestamp_config(optional_params.get("timestamp_granularities"), optional_params.get("response_format")),
|
||||
}
|
||||
return transcription_config
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
import base64
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
|
@ -10,7 +12,7 @@ from litellm.llms.gemini.audio_transcription.transformation import (
|
|||
)
|
||||
from litellm.llms.gemini.common_utils import GeminiError
|
||||
from litellm.types.utils import LlmProviders
|
||||
from litellm.utils import ProviderConfigManager
|
||||
from litellm.utils import ProviderConfigManager, get_optional_params_transcription
|
||||
|
||||
AUDIO_BYTES = b"RIFF....WAVEfmt fake-wav-bytes"
|
||||
|
||||
|
|
@ -123,6 +125,76 @@ class TestGetCompleteUrl:
|
|||
|
||||
|
||||
class TestTransformRequest:
|
||||
@pytest.mark.parametrize(
|
||||
("params", "expected_codes"),
|
||||
[
|
||||
({"language_codes": ["en", "es-ES"]}, ["en-US", "es-ES"]),
|
||||
({"language_codes": '["en", "es-ES"]'}, ["en-US", "es-ES"]),
|
||||
({"language_codes": [" en ", " es-ES "]}, ["en-US", "es-ES"]),
|
||||
({"language_codes": ["zh-Hant-TW", "zh-Hans-CN"]}, ["zh-Hant-TW", "zh-Hans-CN"]),
|
||||
({"language": "fr", "language_codes": ["en", "es"]}, ["en-US", "es-ES"]),
|
||||
({"language": "fr", "language_codes": []}, []),
|
||||
({"language": "fr", "language_codes": "[]"}, []),
|
||||
({"language": "fr", "language_codes": None}, ["fr-FR"]),
|
||||
],
|
||||
)
|
||||
def test_language_codes_reach_gemini_through_provider_params(
|
||||
self,
|
||||
config: GeminiAudioTranscriptionConfig,
|
||||
params: Mapping[str, object],
|
||||
expected_codes: list[str],
|
||||
) -> None:
|
||||
optional_params: Final = get_optional_params_transcription(
|
||||
model="gemini-3.5-transcribe", custom_llm_provider="gemini", **params
|
||||
)
|
||||
request_data: Final = config.transform_audio_transcription_request(
|
||||
model="gemini-3.5-transcribe",
|
||||
audio_file=("sample.wav", AUDIO_BYTES, "audio/wav"),
|
||||
optional_params=optional_params,
|
||||
litellm_params={},
|
||||
)
|
||||
assert json.loads(json.dumps(request_data.data)) == {
|
||||
"model": "gemini-3.5-transcribe",
|
||||
"input": [
|
||||
{
|
||||
"type": "audio",
|
||||
"data": base64.b64encode(AUDIO_BYTES).decode("utf-8"),
|
||||
"mime_type": "audio/wav",
|
||||
}
|
||||
],
|
||||
"generation_config": {"transcription_config": {"language_codes": expected_codes}},
|
||||
}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"language_codes",
|
||||
["en", '"en"', "[", "null", 1, {}, ["en", 1], ["en", ""], [" "], '["en", null]'],
|
||||
)
|
||||
def test_invalid_language_codes_raise_instead_of_silently_dropping_hints(
|
||||
self, config: GeminiAudioTranscriptionConfig, language_codes: object
|
||||
) -> None:
|
||||
with pytest.raises(GeminiError, match="language_codes must be a list of non-empty language strings") as excinfo:
|
||||
config.transform_audio_transcription_request(
|
||||
model="gemini-3.5-transcribe",
|
||||
audio_file=("sample.wav", AUDIO_BYTES, "audio/wav"),
|
||||
optional_params={"language_codes": language_codes},
|
||||
litellm_params={},
|
||||
)
|
||||
assert excinfo.value.status_code == 400
|
||||
|
||||
def test_multiple_language_codes_preserve_word_timestamps(self, config: GeminiAudioTranscriptionConfig) -> None:
|
||||
request_data: Final = config.transform_audio_transcription_request(
|
||||
model="gemini-3.5-transcribe",
|
||||
audio_file=("sample.wav", AUDIO_BYTES, "audio/wav"),
|
||||
optional_params={"language_codes": ["en-US", "es-ES"], "timestamp_granularities": ["word"]},
|
||||
litellm_params={},
|
||||
)
|
||||
assert json.loads(json.dumps(request_data.data["generation_config"])) == {
|
||||
"transcription_config": {
|
||||
"language_codes": ["en-US", "es-ES"],
|
||||
"mode": {"type": "verbatim", "timestamp_granularities": ["word"], "diarization_mode": "speaker"},
|
||||
}
|
||||
}
|
||||
|
||||
def test_builds_json_interaction_request(self, config):
|
||||
request_data = config.transform_audio_transcription_request(
|
||||
model="gemini/gemini-3.5-transcribe",
|
||||
|
|
|
|||
|
|
@ -11,9 +11,10 @@ import contextlib
|
|||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, AsyncIterator, Callable, Dict, Iterator, List, Optional
|
||||
from typing import Any, AsyncIterator, Callable, Dict, Final, Iterator, List, Optional
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
# Repo root, anchored to this file (not CWD) so the path is correct no
|
||||
|
|
@ -536,3 +537,41 @@ def reset_login_throttle(monkeypatch):
|
|||
_drop_throttle_keys()
|
||||
yield _drop_throttle_keys
|
||||
_drop_throttle_keys()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def gemini_transcription_requests(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[list[httpx.Request]]:
|
||||
import litellm
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
requests: Final[list[httpx.Request]] = [] # mutable-ok: captures requests to the injected provider transport
|
||||
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
requests.append(request)
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "test-interaction",
|
||||
"status": "completed",
|
||||
"steps": [{"type": "model_generation", "content": [{"type": "text", "text": "Hello world."}]}],
|
||||
},
|
||||
)
|
||||
|
||||
handler: Final = AsyncHTTPHandler(transport=httpx.MockTransport(respond))
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gemini-transcribe",
|
||||
"litellm_params": {"model": "gemini/gemini-3.5-transcribe", "api_key": "test-key"},
|
||||
}
|
||||
],
|
||||
num_retries=0,
|
||||
default_litellm_params={"client": handler},
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
try:
|
||||
yield requests
|
||||
finally:
|
||||
router.discard()
|
||||
await handler.close()
|
||||
|
|
|
|||
|
|
@ -10,12 +10,18 @@ Pins (PR2):
|
|||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from contextlib import AbstractContextManager
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.types.llms.openai import HttpxBinaryResponseContent
|
||||
|
||||
|
||||
|
|
@ -254,3 +260,33 @@ def test_audio_transcription_error(client, auth_as, patched_transcription_error,
|
|||
response = client.post(path, files=files, data=data)
|
||||
assert response.status_code == 500
|
||||
assert len(response.content) > 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["/v1/audio/transcriptions", "/audio/transcriptions"])
|
||||
def test_repeated_language_fields_reach_gemini_through_proxy(
|
||||
client: TestClient,
|
||||
auth_as: Callable[..., AbstractContextManager[UserAPIKeyAuth]],
|
||||
gemini_transcription_requests: list[httpx.Request],
|
||||
path: str,
|
||||
) -> None:
|
||||
with auth_as():
|
||||
response: Final = client.post(
|
||||
path,
|
||||
files=[
|
||||
("file", ("audio.wav", b"RIFF....WAVEfmt synthetic-audio", "audio/wav")),
|
||||
("model", (None, "gemini-transcribe")),
|
||||
("language", (None, "fr")),
|
||||
("language_codes[]", (None, "en-US")),
|
||||
("language_codes[]", (None, "es-ES")),
|
||||
("timestamp_granularities[]", (None, "word")),
|
||||
],
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["text"] == "Hello world."
|
||||
assert len(gemini_transcription_requests) == 1
|
||||
assert json.loads(gemini_transcription_requests[0].content)["generation_config"] == {
|
||||
"transcription_config": {
|
||||
"language_codes": ["en-US", "es-ES"],
|
||||
"mode": {"type": "verbatim", "timestamp_granularities": ["word"], "diarization_mode": "speaker"},
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue