diff --git a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py index 91d8b35e778..884c82fe331 100644 --- a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py +++ b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py @@ -9,7 +9,6 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger from litellm.caching.caching import Cache, LiteLLMCacheType -from litellm.constants import MINIMUM_PROMPT_CACHE_TOKEN_COUNT from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, @@ -18,7 +17,7 @@ from litellm.llms.custom_httpx.http_handler import ( ) from litellm.llms.openai.openai import AllMessageValues from litellm.types.llms.openai import ChatCompletionToolParam -from litellm.utils import is_prompt_caching_valid_prompt +from litellm.utils import get_prompt_cache_min_tokens, is_prompt_caching_valid_prompt from ..common_utils import VertexAIError, get_vertex_base_url from ..vertex_llm_base import VertexBase, publisher_model_location @@ -67,6 +66,19 @@ class _CreatedCachedContentRoot(RootModel[_CreatedCachedContent]): model_config = ConfigDict(frozen=True, hide_input_in_errors=True) +_TOOL_PARAMS: Final = ("tools", "tool_choice") + + +def _sync_cache_client(timeout: float | httpx.Timeout | None) -> HTTPHandler: + if timeout is None: + return HTTPHandler() + return HTTPHandler(timeout=httpx.Timeout(timeout) if isinstance(timeout, (float, int)) else timeout) + + +class _CachedContentsNotServedError(VertexAIError): + pass + + def _record_cache_creation_tokens(logging_obj: Logging, created: _CreatedCachedContent) -> None: tokens: Final = (created.get("usageMetadata") or {}).get("totalTokenCount") or 0 logging_obj.vertex_ai_cache_creation_input_tokens = max(tokens, 0) # rebind-ok: set provider count @@ -102,30 +114,36 @@ def _cache_target( return _CacheTarget(model=model_location.model, project=model_location.project, location=model_location.location) +def _prompt_cache_min_tokens(models: Sequence[str], custom_llm_provider: str) -> int: + return max( + get_prompt_cache_min_tokens( + model if model.startswith(custom_llm_provider) else f"{custom_llm_provider}/{model}" + ) + for model in models + ) + + def _context_caching_skip_reason( cached_messages: Sequence[AllMessageValues], tools: Sequence[ChatCompletionToolParam] | None, - model: str, + deployment_model: str, + cache_model: str, custom_llm_provider: Literal["vertex_ai", "vertex_ai_beta", "gemini"], - api_base: str | None, ) -> str | None: if not cached_messages_end_on_supported_turn(cached_messages): return ( "Vertex AI context caching: cached message block ends on a model turn once " "system messages are extracted, which the cachedContents API rejects." ) + min_tokens: Final = _prompt_cache_min_tokens((deployment_model, cache_model), custom_llm_provider) if not is_prompt_caching_valid_prompt( - model=model, + model=cache_model, messages=list(cached_messages), tools=None if tools is None else list(tools), custom_llm_provider=custom_llm_provider, + min_token_count=min_tokens, ): - return ( - "Vertex AI context caching: cached content is below minimum token " - f"count ({MINIMUM_PROMPT_CACHE_TOKEN_COUNT})." - ) - if custom_llm_provider == "gemini" and api_base is not None and api_base.rstrip("/").endswith("/v1"): - return "Gemini context caching: the Gemini API serves cachedContents only under v1beta, and this api_base names v1." + return f"Vertex AI context caching: cached content is below minimum token count ({min_tokens})." return None @@ -258,6 +276,8 @@ class ContextCachingEndpoints(VertexBase): except httpx.HTTPStatusError as e: if e.response.status_code == 403: return None + if e.response.status_code == 404 and custom_llm_provider == "gemini": + raise _CachedContentsNotServedError(status_code=404, message=e.response.text) raise VertexAIError(status_code=e.response.status_code, message=e.response.text) except Exception as e: raise VertexAIError(status_code=500, message=str(e)) @@ -357,6 +377,8 @@ class ContextCachingEndpoints(VertexBase): except httpx.HTTPStatusError as e: if e.response.status_code == 403: return None + if e.response.status_code == 404 and custom_llm_provider == "gemini": + raise _CachedContentsNotServedError(status_code=404, message=e.response.text) raise VertexAIError(status_code=e.response.status_code, message=e.response.text) except Exception as e: raise VertexAIError(status_code=500, message=str(e)) @@ -423,16 +445,15 @@ class ContextCachingEndpoints(VertexBase): return messages, optional_params, None target: Final = _cache_target(custom_llm_provider, api_base, model, vertex_project, vertex_location) + tools: Final = optional_params.get("tools") + tool_choice: Final = optional_params.get("tool_choice") skip_reason: Final = _context_caching_skip_reason( - cached_messages, optional_params.get("tools"), target.model, custom_llm_provider, api_base + cached_messages, tools, model, target.model, custom_llm_provider ) if skip_reason is not None: verbose_logger.debug("%s Skipping context caching.", skip_reason) return messages, optional_params, None - tools: Final = optional_params.pop("tools", None) - tool_choice: Final = optional_params.pop("tool_choice", None) - ## AUTHORIZATION ## token, url = self._get_token_and_url_context_caching( gemini_api_key=api_key, @@ -454,33 +475,31 @@ class ContextCachingEndpoints(VertexBase): if extra_headers is not None: headers.update(extra_headers) - if client is None or not isinstance(client, HTTPHandler): - _params: Final = {} - if timeout is not None: - if isinstance(timeout, float) or isinstance(timeout, int): - timeout = httpx.Timeout(timeout) - _params["timeout"] = timeout - client = HTTPHandler(**_params) - else: - client = client + http_client: Final = client if isinstance(client, HTTPHandler) else _sync_cache_client(timeout) ## CHECK IF CACHED ALREADY generated_cache_key: Final = local_cache_obj.get_cache_key( messages=cached_messages, tools=tools, tool_choice=tool_choice, model=target.model ) - google_cache_name: Final = self.check_cache( - cache_key=generated_cache_key, - client=client, - headers=headers, - api_key=api_key, - api_base=api_base, - logging_obj=logging_obj, - custom_llm_provider=custom_llm_provider, - vertex_project=target.project, - vertex_location=target.location, - vertex_auth_header=vertex_auth_header, - model=target.model, - ) + try: + google_cache_name: Final = self.check_cache( + cache_key=generated_cache_key, + client=http_client, + headers=headers, + api_key=api_key, + api_base=api_base, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + vertex_project=target.project, + vertex_location=target.location, + vertex_auth_header=vertex_auth_header, + model=target.model, + ) + except _CachedContentsNotServedError: + verbose_logger.debug("Gemini context caching: %s answers 404. Skipping context caching.", url) + return messages, optional_params, None + for tool_param in _TOOL_PARAMS: + optional_params.pop(tool_param, None) if google_cache_name: return non_cached_messages, optional_params, google_cache_name @@ -510,7 +529,7 @@ class ContextCachingEndpoints(VertexBase): ) try: - response: Final = client.post( + response: Final = http_client.post( url=url, headers=headers, json=cached_content_request_body, @@ -574,16 +593,15 @@ class ContextCachingEndpoints(VertexBase): return messages, optional_params, None target: Final = _cache_target(custom_llm_provider, api_base, model, vertex_project, vertex_location) + tools: Final = optional_params.get("tools") + tool_choice: Final = optional_params.get("tool_choice") skip_reason: Final = _context_caching_skip_reason( - cached_messages, optional_params.get("tools"), target.model, custom_llm_provider, api_base + cached_messages, tools, model, target.model, custom_llm_provider ) if skip_reason is not None: verbose_logger.debug("%s Skipping context caching.", skip_reason) return messages, optional_params, None - tools: Final = optional_params.pop("tools", None) - tool_choice: Final = optional_params.pop("tool_choice", None) - ## AUTHORIZATION ## token, url = self._get_token_and_url_context_caching( gemini_api_key=api_key, @@ -681,9 +699,15 @@ class ContextCachingEndpoints(VertexBase): ) return raw_response_cached.get("name") - cache_name: Final = await self._known_caches.single_flight( - scope, generated_cache_key, lookup_or_create, _wait_seconds(timeout) - ) + try: + cache_name: Final = await self._known_caches.single_flight( + scope, generated_cache_key, lookup_or_create, _wait_seconds(timeout) + ) + except _CachedContentsNotServedError: + verbose_logger.debug("Gemini context caching: %s answers 404. Skipping context caching.", url) + return messages, optional_params, None + for tool_param in _TOOL_PARAMS: + optional_params.pop(tool_param, None) return non_cached_messages, optional_params, cache_name def get_cache(self): diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index f1176f777c7..7cb5432d009 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -376,7 +376,7 @@ class RequestBody(TypedDict, total=False): class CachedContentRequestBody(TypedDict, total=False): contents: Required[list[ContentType]] system_instruction: SystemInstructions - tools: Tools + tools: Tools | None toolConfig: ToolConfig model: Required[str] # Format: models/{model} ttl: str # ending in 's' - Example: "3.5s". diff --git a/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py b/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py index 83377323e65..78b9e930001 100644 --- a/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py +++ b/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py @@ -304,10 +304,13 @@ class TestContextCachingEndpoints: ] assert cache_for("gemini-2.5-pro") == pro_cache - @pytest.mark.parametrize("api_base", ["https://gateway.example.com/v1", "https://gateway.example.com/v1/"]) - @pytest.mark.parametrize("is_async", [False, True]) - @pytest.mark.asyncio - async def test_check_and_create_cache_skips_a_gemini_v1_api_base(self, api_base: str, is_async: bool): + def _gemini_v1_gateway_call_args(self, list_response: MagicMock): + self.mock_client.get.return_value = list_response + self.mock_async_client.get = AsyncMock(return_value=list_response) + created: Final = MagicMock() + created.json.return_value = {"name": "cachedContents/gateway-cache"} + self.mock_client.post.return_value = created + self.mock_async_client.post = AsyncMock(return_value=created) messages: Final = [ { "role": "user", @@ -319,7 +322,7 @@ class TestContextCachingEndpoints: messages=messages, optional_params={"tools": self.sample_tools}, api_key="unused", - api_base=api_base, + api_base="https://gateway.example.com/gemini/v1", model="gemini-2.5-flash", timeout=30.0, logging_obj=self.mock_logging, @@ -328,6 +331,19 @@ class TestContextCachingEndpoints: vertex_location=None, vertex_auth_header=None, ) + return call_args + + @pytest.mark.parametrize("is_async", [False, True]) + @pytest.mark.asyncio + async def test_check_and_create_cache_sends_uncached_when_the_gemini_api_base_serves_no_cached_contents( + self, is_async: bool + ): + not_served: Final = MagicMock() + not_served.raise_for_status.side_effect = httpx.HTTPStatusError( + "Not Found", request=MagicMock(), response=MagicMock(status_code=404, text="") + ) + + call_args: Final = self._gemini_v1_gateway_call_args(not_served) result: Final = ( await self.context_caching.async_check_and_create_cache(client=self.mock_async_client, **call_args) @@ -335,9 +351,31 @@ class TestContextCachingEndpoints: else self.context_caching.check_and_create_cache(client=self.mock_client, **call_args) ) - assert result == (messages, {"tools": self.sample_tools}, None) - assert not self.mock_client.mock_calls - assert not self.mock_async_client.mock_calls + assert result == (call_args["messages"], {"tools": self.sample_tools}, None) + self.mock_client.post.assert_not_called() + self.mock_async_client.post.assert_not_called() + + @pytest.mark.parametrize("is_async", [False, True]) + @pytest.mark.asyncio + async def test_check_and_create_cache_caches_through_a_gemini_v1_gateway_that_serves_cached_contents( + self, is_async: bool + ): + empty_list: Final = MagicMock() + empty_list.json.return_value = {} + + call_args: Final = self._gemini_v1_gateway_call_args(empty_list) + + _, params, cache_name = ( + await self.context_caching.async_check_and_create_cache(client=self.mock_async_client, **call_args) + if is_async + else self.context_caching.check_and_create_cache(client=self.mock_client, **call_args) + ) + client: Final = self.mock_async_client if is_async else self.mock_client + + assert params == {} + assert cache_name == "cachedContents/gateway-cache" + assert client.get.call_args.kwargs["url"] == "https://gateway.example.com/gemini/v1/cachedContents" + assert client.post.call_args.kwargs["url"].startswith("https://gateway.example.com/gemini/v1/cachedContents") @pytest.mark.parametrize("custom_llm_provider", ["gemini", "vertex_ai", "vertex_ai_beta"]) @patch("litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.separate_cached_messages") @@ -1430,10 +1468,14 @@ class TestContextCachingEndpoints: self._token_check_patcher.start() + @pytest.mark.parametrize( + ("api_base_model", "deployment_model"), + [("gemini-2.5-pro", "gemini-2.5-flash"), ("gemini-not-in-the-cost-map", "gemini-2.5-pro")], + ) @pytest.mark.parametrize("is_async", [False, True]) @pytest.mark.asyncio - async def test_check_and_create_cache_applies_the_api_base_model_minimum( - self, local_model_cost_map, is_async: bool + async def test_check_and_create_cache_applies_the_stricter_model_minimum( + self, local_model_cost_map, is_async: bool, api_base_model: str, deployment_model: str ): self._token_check_patcher.stop() messages: Final = [ @@ -1444,8 +1486,8 @@ class TestContextCachingEndpoints: messages=messages, optional_params={}, api_key="unused", - api_base="https://gateway.example.com/v1/projects/base-project/locations/global/publishers/google/models/gemini-2.5-pro", - model="gemini-2.5-flash", + api_base=f"https://gateway.example.com/v1/projects/base-project/locations/global/publishers/google/models/{api_base_model}", + model=deployment_model, timeout=30.0, logging_obj=self.mock_logging, custom_llm_provider="vertex_ai",