From fdc7648f43cbd4867665d77dbda18dedd152bb10 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:18:53 -0800 Subject: [PATCH] fix(speech): honor request cache controls on Gemini and Vertex AI TTS --- .../speech_to_completion_bridge/handler.py | 4 ++- litellm/main.py | 4 +++ litellm/types/utils.py | 4 +++ ...st_gemini_tts_voice_mapping_owned_proxy.py | 3 +- .../text_to_speech/test_transformation.py | 23 +++++++++++++ tests/unit/test_main.py | 32 +++++++++++++++++++ 6 files changed, 67 insertions(+), 3 deletions(-) diff --git a/litellm/endpoints/speech/speech_to_completion_bridge/handler.py b/litellm/endpoints/speech/speech_to_completion_bridge/handler.py index 961c209fc5b..2af2b9fb52c 100644 --- a/litellm/endpoints/speech/speech_to_completion_bridge/handler.py +++ b/litellm/endpoints/speech/speech_to_completion_bridge/handler.py @@ -2,6 +2,7 @@ Handler for transforming /chat/completions api requests to litellm.responses requests """ +from collections.abc import Mapping from typing import TYPE_CHECKING, Final from typing_extensions import TypedDict @@ -91,6 +92,7 @@ class SpeechToCompletionBridgeHandler: custom_llm_provider: str, api_base: str | None, timeout: "float | httpx.Timeout | None", + cache: Mapping[str, object] | None, ) -> "HttpxBinaryResponseContent": received_args: Final = locals() from litellm import completion @@ -110,7 +112,7 @@ class SpeechToCompletionBridgeHandler: model=model, input=input, optional_params=optional_params, - litellm_params={**litellm_params, "api_base": api_base, "timeout": timeout}, + litellm_params={**litellm_params, "api_base": api_base, "timeout": timeout, "cache": cache}, headers=headers, litellm_logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, diff --git a/litellm/main.py b/litellm/main.py index e62ca56fe24..5736f18c2ae 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -156,6 +156,7 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( BaseUrlRoutingHint, + CacheControlsHint, CustomPricingLiteLLMParams, ModelResponseStream, ProviderRoutingHints, @@ -4234,6 +4235,7 @@ _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints) _BASE_URL_HINT: Final = TypeAdapter(BaseUrlRoutingHint) +_CACHE_CONTROLS_HINT: Final = TypeAdapter(CacheControlsHint) _OPAQUE_VALUE_ADAPTER: Final = TypeAdapter(object) @@ -8939,6 +8941,7 @@ def speech( custom_llm_provider=custom_llm_provider, api_base=api_base, timeout=timeout, + cache=_CACHE_CONTROLS_HINT.validate_python(kwargs).get("cache"), ) # Vertex AI Text-to-Speech (Google Cloud TTS) @@ -8992,6 +8995,7 @@ def speech( custom_llm_provider=custom_llm_provider, api_base=api_base, timeout=timeout, + cache=_CACHE_CONTROLS_HINT.validate_python(kwargs).get("cache"), ) elif custom_llm_provider == "runwayml": from litellm.llms.runwayml.text_to_speech.transformation import ( diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 70ca05904ef..f79afcf5c8e 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -5017,3 +5017,7 @@ class ProviderRoutingHints(TypedDict, total=False): class BaseUrlRoutingHint(TypedDict, total=False): base_url: ReadOnly[str | None] + + +class CacheControlsHint(TypedDict, total=False): + cache: ReadOnly[Mapping[str, object] | None] diff --git a/tests/integration/providers/test_gemini_tts_voice_mapping_owned_proxy.py b/tests/integration/providers/test_gemini_tts_voice_mapping_owned_proxy.py index 08ac73cd8e7..43c5d56a8c6 100644 --- a/tests/integration/providers/test_gemini_tts_voice_mapping_owned_proxy.py +++ b/tests/integration/providers/test_gemini_tts_voice_mapping_owned_proxy.py @@ -1,7 +1,6 @@ """The speech route, the audio_speech health check and deferred Vertex Live setups map OpenAI voices. -The speech bridge calls the Gemini chat path without the deployment's ``api_base`` (tracked separately), -so these rows boot their own proxy with ``GEMINI_API_BASE`` pointed at the scripted peer, and with +These rows boot their own proxy with ``GEMINI_API_BASE`` pointed at the scripted peer, and with ``LITELLM_GEMINI_LIVE_DEFER_SETUP`` so a client ``session.update`` voice builds the Vertex Live setup. """ diff --git a/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py b/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py index 118ee3b0929..01e75804379 100644 --- a/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py +++ b/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py @@ -17,6 +17,7 @@ import responses from pydantic import ValidationError import litellm +from litellm.caching.caching import Cache from litellm.litellm_core_utils.exception_mapping_utils import exception_type from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import OPENAI_TO_GEMINI_TTS_VOICES from litellm.llms.vertex_ai.text_to_speech.transformation import ( @@ -1726,6 +1727,28 @@ async def test_aspeech_vertex_ai_gemini_tts_routes_to_configured_api_base( assert _PCM_AUDIO in response.content +@pytest.mark.asyncio +async def test_aspeech_vertex_ai_gemini_tts_no_cache_request_reaches_vertex( + respx_mock: respx.MockRouter, google_token_endpoint: responses.RequestsMock, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "cache", Cache(type="local")) + vertex_route: Final = respx_mock.post( + url__regex=r"https://aiplatform\.googleapis\.com/.*/models/gemini-3\.8-flash-tts:generateContent$" + ).mock(return_value=httpx.Response(200, json=_GENERATE_CONTENT_RESPONSE)) + speech_kwargs: Final = { + "model": "vertex_ai/gemini-3.8-flash-tts", + "input": "Say hello", + "voice": "Kore", + "vertex_credentials": _AUTHORIZED_USER, + "vertex_location": "global", + } + + await litellm.aspeech(**speech_kwargs) + await litellm.aspeech(**speech_kwargs, cache={"no-cache": True}) + + assert vertex_route.call_count == 2 + + @pytest.mark.asyncio async def test_aspeech_vertex_ai_gemini_tts_mp3_reaches_cloud_tts_for_any_gemini_tts_model( respx_mock: respx.MockRouter, google_token_endpoint: responses.RequestsMock diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 7d01a215de9..243a4bfcb13 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -5515,6 +5515,38 @@ def test_speech_gemini_tts_without_a_timeout_waits_as_long_as_the_same_chat_comp assert speech_call.request.extensions["timeout"] == chat_call.request.extensions["timeout"] +@pytest.mark.parametrize( + ("first_controls", "second_controls", "expected_upstream_calls"), + [ + ({}, {}, 1), + ({}, {"cache": {"no-cache": True}}, 2), + ({"cache": {"no-store": True}}, {}, 2), + ], +) +def test_speech_gemini_tts_honors_the_request_cache_controls( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + first_controls: Mapping[str, Mapping[str, bool]], + second_controls: Mapping[str, Mapping[str, bool]], + expected_upstream_calls: int, +): + monkeypatch.setattr(litellm, "cache", Cache(type="local")) + route: Final = respx_mock.post(url__regex=GEMINI_DEFAULT_GENERATE_CONTENT_URL).mock( + return_value=httpx.Response(200, json=_gemini_tts_generate_content_response()) + ) + speech_kwargs: Final = { + "model": f"gemini/{GEMINI_TTS_MODEL}", + "input": "hello", + "voice": "Kore", + "api_key": "fake-gemini-key", + } + + litellm.speech(**speech_kwargs, **first_controls) + litellm.speech(**speech_kwargs, **second_controls) + + assert route.call_count == expected_upstream_calls + + FOUNDRY_HOST: Final = "https://my-project.services.ai.azure.com"