From d2396358e67936ccc800fdee2daab04d1ee14c34 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:39:48 -0700 Subject: [PATCH] fix(vertex_ai): check the cache token minimum against the api_base's model The cache is created for the model a publisher-model api_base names, so the minimum prompt size now comes from that model too. A prompt below its minimum is sent uncached instead of failing at cache create. --- .../vertex_ai_context_caching.py | 8 ++--- .../test_vertex_ai_context_caching.py | 35 +++++++++++++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py index d629ebb4eb1..91d8b35e778 100644 --- a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py +++ b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py @@ -422,14 +422,14 @@ class ContextCachingEndpoints(VertexBase): if len(cached_messages) == 0: return messages, optional_params, None + target: Final = _cache_target(custom_llm_provider, api_base, model, vertex_project, vertex_location) skip_reason: Final = _context_caching_skip_reason( - cached_messages, optional_params.get("tools"), model, custom_llm_provider, api_base + cached_messages, optional_params.get("tools"), target.model, custom_llm_provider, api_base ) if skip_reason is not None: verbose_logger.debug("%s Skipping context caching.", skip_reason) return messages, optional_params, None - target: Final = _cache_target(custom_llm_provider, api_base, model, vertex_project, vertex_location) tools: Final = optional_params.pop("tools", None) tool_choice: Final = optional_params.pop("tool_choice", None) @@ -573,14 +573,14 @@ class ContextCachingEndpoints(VertexBase): if len(cached_messages) == 0: return messages, optional_params, None + target: Final = _cache_target(custom_llm_provider, api_base, model, vertex_project, vertex_location) skip_reason: Final = _context_caching_skip_reason( - cached_messages, optional_params.get("tools"), model, custom_llm_provider, api_base + cached_messages, optional_params.get("tools"), target.model, custom_llm_provider, api_base ) if skip_reason is not None: verbose_logger.debug("%s Skipping context caching.", skip_reason) return messages, optional_params, None - target: Final = _cache_target(custom_llm_provider, api_base, model, vertex_project, vertex_location) tools: Final = optional_params.pop("tools", None) tool_choice: Final = optional_params.pop("tool_choice", None) diff --git a/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py b/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py index a6240ca8c91..83377323e65 100644 --- a/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py +++ b/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py @@ -1430,6 +1430,41 @@ class TestContextCachingEndpoints: self._token_check_patcher.start() + @pytest.mark.parametrize("is_async", [False, True]) + @pytest.mark.asyncio + async def test_check_and_create_cache_applies_the_api_base_model_minimum( + self, local_model_cost_map, is_async: bool + ): + self._token_check_patcher.stop() + messages: Final = [ + {"role": "system", "content": " ".join(["word"] * 1500), "cache_control": {"type": "ephemeral"}}, + {"role": "user", "content": "Hello"}, + ] + call_args: Final = dict( + messages=messages, + optional_params={}, + api_key="unused", + api_base="https://gateway.example.com/v1/projects/base-project/locations/global/publishers/google/models/gemini-2.5-pro", + model="gemini-2.5-flash", + timeout=30.0, + logging_obj=self.mock_logging, + custom_llm_provider="vertex_ai", + vertex_project="config-project", + vertex_location="us-central1", + vertex_auth_header="vertex-token", + ) + + result: Final = ( + await self.context_caching.async_check_and_create_cache(client=self.mock_async_client, **call_args) + if is_async + else self.context_caching.check_and_create_cache(client=self.mock_client, **call_args) + ) + + self._token_check_patcher.start() + assert result == (messages, {}, None) + assert not self.mock_client.mock_calls + assert not self.mock_async_client.mock_calls + @pytest.mark.parametrize("custom_llm_provider", ["gemini", "vertex_ai", "vertex_ai_beta"]) @patch("litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.separate_cached_messages") @pytest.mark.asyncio