From 33584bcc0dd1c6c7a191be4e5ad41ed07929ade4 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:24:02 -0800 Subject: [PATCH] fix(speech): send the deployment's extra_headers and chat's caching rule through the TTS bridge Gemini and Vertex AI speech now forward the deployment's extra_headers to the inner chat call, pass caching as sent so chat's own rule decides, and keep the router's deployment timeout so speech waits as long as chat --- .../speech_to_completion_bridge/handler.py | 10 ++++- litellm/main.py | 4 ++ litellm/types/utils.py | 7 +++- .../text_to_speech/test_transformation.py | 2 + tests/unit/test_main.py | 39 ++++++++++++++++++- 5 files changed, 57 insertions(+), 5 deletions(-) diff --git a/litellm/endpoints/speech/speech_to_completion_bridge/handler.py b/litellm/endpoints/speech/speech_to_completion_bridge/handler.py index f0c06669639..93ddc503248 100644 --- a/litellm/endpoints/speech/speech_to_completion_bridge/handler.py +++ b/litellm/endpoints/speech/speech_to_completion_bridge/handler.py @@ -2,6 +2,7 @@ Handler for transforming /chat/completions api requests to litellm.responses requests """ +from collections.abc import Mapping from typing import TYPE_CHECKING, Final from typing_extensions import TypedDict @@ -91,6 +92,7 @@ class SpeechToCompletionBridgeHandler: logging_obj: "LiteLLMLoggingObj", custom_llm_provider: str, api_base: str | None, + extra_headers: Mapping[str, str] | None, timeout: "float | httpx.Timeout | None", cache_controls: "CacheControlsHint", ) -> "HttpxBinaryResponseContent": @@ -112,7 +114,13 @@ class SpeechToCompletionBridgeHandler: model=model, input=input, optional_params=optional_params, - litellm_params={**litellm_params, "api_base": api_base, "timeout": timeout, **cache_controls}, + litellm_params={ + **litellm_params, + "api_base": api_base, + "extra_headers": extra_headers, + "timeout": timeout, + **cache_controls, + }, headers=headers, litellm_logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, diff --git a/litellm/main.py b/litellm/main.py index 1b9c71f04f6..10032fa37e6 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -174,6 +174,7 @@ from litellm.types.utils import ( AgenticLoopParams, BaseUrlRoutingHint, CacheControlsHint, + DeploymentHeadersHint, ModelResponseStream, ProviderRoutingHints, RawRequestTypedDict, @@ -4362,6 +4363,7 @@ _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints) _BASE_URL_HINT: Final = TypeAdapter(BaseUrlRoutingHint) _CACHE_CONTROLS_HINT: Final = TypeAdapter(CacheControlsHint) +_DEPLOYMENT_HEADERS_HINT: Final = TypeAdapter(DeploymentHeadersHint) _OPAQUE_VALUE_ADAPTER: Final = TypeAdapter(object) @@ -9237,6 +9239,7 @@ def speech( logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, api_base=api_base, + extra_headers=_DEPLOYMENT_HEADERS_HINT.validate_python(kwargs).get("extra_headers"), timeout=timeout, cache_controls=_CACHE_CONTROLS_HINT.validate_python(kwargs), ) @@ -9291,6 +9294,7 @@ def speech( logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, api_base=api_base, + extra_headers=_DEPLOYMENT_HEADERS_HINT.validate_python(kwargs).get("extra_headers"), timeout=timeout, cache_controls=_CACHE_CONTROLS_HINT.validate_python(kwargs), ) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 39491e6c8bd..69ee616b693 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -49,7 +49,6 @@ from pydantic import ( OnErrorOmit, PrivateAttr, SkipValidation, - StrictBool, TypeAdapter, ValidationError, field_serializer, @@ -5402,4 +5401,8 @@ class BaseUrlRoutingHint(TypedDict, total=False): class CacheControlsHint(TypedDict, total=False): cache: ReadOnly[OnErrorOmit[Mapping[str, object] | None]] - caching: ReadOnly[OnErrorOmit[StrictBool | None]] + caching: ReadOnly[OnErrorOmit[JsonValue]] + + +class DeploymentHeadersHint(TypedDict, total=False): + extra_headers: ReadOnly[OnErrorOmit[Mapping[str, str] | None]] diff --git a/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py b/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py index e9994184201..a1a1f4fb271 100644 --- a/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py +++ b/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py @@ -1774,10 +1774,12 @@ async def test_aspeech_vertex_ai_gemini_tts_routes_to_configured_api_base( vertex_credentials=_AUTHORIZED_USER, vertex_location="global", api_base="https://vertex.gateway.internal", + extra_headers={"x-gateway-key": "gateway-secret"}, ) assert gateway_route.call_count == 1 assert gateway_route.calls.last.request.headers["authorization"] == "Bearer minted-google-token" + assert gateway_route.calls.last.request.headers["x-gateway-key"] == "gateway-secret" assert _PCM_AUDIO in response.content diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 90598d8b3c0..d116cebc69d 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -6221,6 +6221,7 @@ async def test_speech_gemini_tts_routes_to_configured_api_base( "input": "hello from litellm", "voice": "Kore", "api_key": "fake-gemini-key", + "extra_headers": {"x-gateway-key": "gateway-secret"}, endpoint_param: GEMINI_GATEWAY_BASE, } @@ -6228,6 +6229,7 @@ async def test_speech_gemini_tts_routes_to_configured_api_base( assert gateway_route.call_count == 1 assert gateway_route.calls.last.request.headers["x-goog-api-key"] == "fake-gemini-key" + assert gateway_route.calls.last.request.headers["x-gateway-key"] == "gateway-secret" assert b"pcm-audio-bytes" in response.content @@ -6261,6 +6263,34 @@ def test_speech_gemini_tts_without_a_timeout_waits_as_long_as_the_same_chat_comp assert speech_call.request.extensions["timeout"] == chat_call.request.extensions["timeout"] +async def test_speech_gemini_tts_behind_the_router_waits_as_long_as_the_same_chat_completion( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.post(url__regex=GEMINI_DEFAULT_GENERATE_CONTENT_URL).mock( + return_value=httpx.Response(200, json=_gemini_tts_generate_content_response()) + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "gemini-tts", + "litellm_params": {"model": f"gemini/{GEMINI_TTS_MODEL}", "api_key": "fake-gemini-key"}, + } + ] + ) + + await router.aspeech(model="gemini-tts", input="hello", voice="Kore") + await router.acompletion( + model="gemini-tts", + messages=[{"role": "user", "content": "hello"}], + modalities=["audio"], + audio={"voice": "Kore", "format": "pcm16"}, + ) + + speech_call, chat_call = route.calls + assert speech_call.request.extensions["timeout"] == chat_call.request.extensions["timeout"] + + @pytest.mark.parametrize( ("first_controls", "second_controls", "expected_upstream_calls"), [ @@ -6268,6 +6298,8 @@ def test_speech_gemini_tts_without_a_timeout_waits_as_long_as_the_same_chat_comp ({}, {"cache": {"no-cache": True}}, 2), ({"cache": {"no-store": True}}, {}, 2), ({}, {"caching": False}, 2), + ({}, {"caching": "yes"}, 2), + ({}, {"caching": 0}, 2), ], ) def test_speech_gemini_tts_honors_the_request_cache_controls( @@ -6294,8 +6326,11 @@ def test_speech_gemini_tts_honors_the_request_cache_controls( assert route.call_count == expected_upstream_calls -@pytest.mark.parametrize("malformed_controls", [{"caching": "yes"}, {"caching": 1}, {"cache": {1: True}}]) -def test_speech_gemini_tts_without_a_cache_ignores_malformed_cache_controls( +@pytest.mark.parametrize( + "malformed_controls", + [{"caching": "yes"}, {"caching": 1}, {"cache": {1: True}}, {"extra_headers": {"x-gateway-key": 5}}], +) +def test_speech_gemini_tts_without_a_cache_ignores_malformed_controls( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, malformed_controls: Mapping[str, object],