fix(vertex_ai): use the stricter cache minimum and send a Gemini base's 404 uncached

This commit is contained in:
mateo-berri 2026-10-28 23:45:01 -07:00
parent d2396358e6
commit b80fd9bc93
3 changed files with 124 additions and 58 deletions

View file

@ -9,7 +9,6 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
from litellm.caching.caching import Cache, LiteLLMCacheType
from litellm.constants import MINIMUM_PROMPT_CACHE_TOKEN_COUNT
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
@ -18,7 +17,7 @@ from litellm.llms.custom_httpx.http_handler import (
)
from litellm.llms.openai.openai import AllMessageValues
from litellm.types.llms.openai import ChatCompletionToolParam
from litellm.utils import is_prompt_caching_valid_prompt
from litellm.utils import get_prompt_cache_min_tokens, is_prompt_caching_valid_prompt
from ..common_utils import VertexAIError, get_vertex_base_url
from ..vertex_llm_base import VertexBase, publisher_model_location
@ -67,6 +66,19 @@ class _CreatedCachedContentRoot(RootModel[_CreatedCachedContent]):
model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
_TOOL_PARAMS: Final = ("tools", "tool_choice")
def _sync_cache_client(timeout: float | httpx.Timeout | None) -> HTTPHandler:
if timeout is None:
return HTTPHandler()
return HTTPHandler(timeout=httpx.Timeout(timeout) if isinstance(timeout, (float, int)) else timeout)
class _CachedContentsNotServedError(VertexAIError):
pass
def _record_cache_creation_tokens(logging_obj: Logging, created: _CreatedCachedContent) -> None:
tokens: Final = (created.get("usageMetadata") or {}).get("totalTokenCount") or 0
logging_obj.vertex_ai_cache_creation_input_tokens = max(tokens, 0) # rebind-ok: set provider count
@ -102,30 +114,36 @@ def _cache_target(
return _CacheTarget(model=model_location.model, project=model_location.project, location=model_location.location)
def _prompt_cache_min_tokens(models: Sequence[str], custom_llm_provider: str) -> int:
return max(
get_prompt_cache_min_tokens(
model if model.startswith(custom_llm_provider) else f"{custom_llm_provider}/{model}"
)
for model in models
)
def _context_caching_skip_reason(
cached_messages: Sequence[AllMessageValues],
tools: Sequence[ChatCompletionToolParam] | None,
model: str,
deployment_model: str,
cache_model: str,
custom_llm_provider: Literal["vertex_ai", "vertex_ai_beta", "gemini"],
api_base: str | None,
) -> str | None:
if not cached_messages_end_on_supported_turn(cached_messages):
return (
"Vertex AI context caching: cached message block ends on a model turn once "
"system messages are extracted, which the cachedContents API rejects."
)
min_tokens: Final = _prompt_cache_min_tokens((deployment_model, cache_model), custom_llm_provider)
if not is_prompt_caching_valid_prompt(
model=model,
model=cache_model,
messages=list(cached_messages),
tools=None if tools is None else list(tools),
custom_llm_provider=custom_llm_provider,
min_token_count=min_tokens,
):
return (
"Vertex AI context caching: cached content is below minimum token "
f"count ({MINIMUM_PROMPT_CACHE_TOKEN_COUNT})."
)
if custom_llm_provider == "gemini" and api_base is not None and api_base.rstrip("/").endswith("/v1"):
return "Gemini context caching: the Gemini API serves cachedContents only under v1beta, and this api_base names v1."
return f"Vertex AI context caching: cached content is below minimum token count ({min_tokens})."
return None
@ -258,6 +276,8 @@ class ContextCachingEndpoints(VertexBase):
except httpx.HTTPStatusError as e:
if e.response.status_code == 403:
return None
if e.response.status_code == 404 and custom_llm_provider == "gemini":
raise _CachedContentsNotServedError(status_code=404, message=e.response.text)
raise VertexAIError(status_code=e.response.status_code, message=e.response.text)
except Exception as e:
raise VertexAIError(status_code=500, message=str(e))
@ -357,6 +377,8 @@ class ContextCachingEndpoints(VertexBase):
except httpx.HTTPStatusError as e:
if e.response.status_code == 403:
return None
if e.response.status_code == 404 and custom_llm_provider == "gemini":
raise _CachedContentsNotServedError(status_code=404, message=e.response.text)
raise VertexAIError(status_code=e.response.status_code, message=e.response.text)
except Exception as e:
raise VertexAIError(status_code=500, message=str(e))
@ -423,16 +445,15 @@ class ContextCachingEndpoints(VertexBase):
return messages, optional_params, None
target: Final = _cache_target(custom_llm_provider, api_base, model, vertex_project, vertex_location)
tools: Final = optional_params.get("tools")
tool_choice: Final = optional_params.get("tool_choice")
skip_reason: Final = _context_caching_skip_reason(
cached_messages, optional_params.get("tools"), target.model, custom_llm_provider, api_base
cached_messages, tools, model, target.model, custom_llm_provider
)
if skip_reason is not None:
verbose_logger.debug("%s Skipping context caching.", skip_reason)
return messages, optional_params, None
tools: Final = optional_params.pop("tools", None)
tool_choice: Final = optional_params.pop("tool_choice", None)
## AUTHORIZATION ##
token, url = self._get_token_and_url_context_caching(
gemini_api_key=api_key,
@ -454,33 +475,31 @@ class ContextCachingEndpoints(VertexBase):
if extra_headers is not None:
headers.update(extra_headers)
if client is None or not isinstance(client, HTTPHandler):
_params: Final = {}
if timeout is not None:
if isinstance(timeout, float) or isinstance(timeout, int):
timeout = httpx.Timeout(timeout)
_params["timeout"] = timeout
client = HTTPHandler(**_params)
else:
client = client
http_client: Final = client if isinstance(client, HTTPHandler) else _sync_cache_client(timeout)
## CHECK IF CACHED ALREADY
generated_cache_key: Final = local_cache_obj.get_cache_key(
messages=cached_messages, tools=tools, tool_choice=tool_choice, model=target.model
)
google_cache_name: Final = self.check_cache(
cache_key=generated_cache_key,
client=client,
headers=headers,
api_key=api_key,
api_base=api_base,
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
vertex_project=target.project,
vertex_location=target.location,
vertex_auth_header=vertex_auth_header,
model=target.model,
)
try:
google_cache_name: Final = self.check_cache(
cache_key=generated_cache_key,
client=http_client,
headers=headers,
api_key=api_key,
api_base=api_base,
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
vertex_project=target.project,
vertex_location=target.location,
vertex_auth_header=vertex_auth_header,
model=target.model,
)
except _CachedContentsNotServedError:
verbose_logger.debug("Gemini context caching: %s answers 404. Skipping context caching.", url)
return messages, optional_params, None
for tool_param in _TOOL_PARAMS:
optional_params.pop(tool_param, None)
if google_cache_name:
return non_cached_messages, optional_params, google_cache_name
@ -510,7 +529,7 @@ class ContextCachingEndpoints(VertexBase):
)
try:
response: Final = client.post(
response: Final = http_client.post(
url=url,
headers=headers,
json=cached_content_request_body,
@ -574,16 +593,15 @@ class ContextCachingEndpoints(VertexBase):
return messages, optional_params, None
target: Final = _cache_target(custom_llm_provider, api_base, model, vertex_project, vertex_location)
tools: Final = optional_params.get("tools")
tool_choice: Final = optional_params.get("tool_choice")
skip_reason: Final = _context_caching_skip_reason(
cached_messages, optional_params.get("tools"), target.model, custom_llm_provider, api_base
cached_messages, tools, model, target.model, custom_llm_provider
)
if skip_reason is not None:
verbose_logger.debug("%s Skipping context caching.", skip_reason)
return messages, optional_params, None
tools: Final = optional_params.pop("tools", None)
tool_choice: Final = optional_params.pop("tool_choice", None)
## AUTHORIZATION ##
token, url = self._get_token_and_url_context_caching(
gemini_api_key=api_key,
@ -681,9 +699,15 @@ class ContextCachingEndpoints(VertexBase):
)
return raw_response_cached.get("name")
cache_name: Final = await self._known_caches.single_flight(
scope, generated_cache_key, lookup_or_create, _wait_seconds(timeout)
)
try:
cache_name: Final = await self._known_caches.single_flight(
scope, generated_cache_key, lookup_or_create, _wait_seconds(timeout)
)
except _CachedContentsNotServedError:
verbose_logger.debug("Gemini context caching: %s answers 404. Skipping context caching.", url)
return messages, optional_params, None
for tool_param in _TOOL_PARAMS:
optional_params.pop(tool_param, None)
return non_cached_messages, optional_params, cache_name
def get_cache(self):

View file

@ -376,7 +376,7 @@ class RequestBody(TypedDict, total=False):
class CachedContentRequestBody(TypedDict, total=False):
contents: Required[list[ContentType]]
system_instruction: SystemInstructions
tools: Tools
tools: Tools | None
toolConfig: ToolConfig
model: Required[str] # Format: models/{model}
ttl: str # ending in 's' - Example: "3.5s".

View file

@ -304,10 +304,13 @@ class TestContextCachingEndpoints:
]
assert cache_for("gemini-2.5-pro") == pro_cache
@pytest.mark.parametrize("api_base", ["https://gateway.example.com/v1", "https://gateway.example.com/v1/"])
@pytest.mark.parametrize("is_async", [False, True])
@pytest.mark.asyncio
async def test_check_and_create_cache_skips_a_gemini_v1_api_base(self, api_base: str, is_async: bool):
def _gemini_v1_gateway_call_args(self, list_response: MagicMock):
self.mock_client.get.return_value = list_response
self.mock_async_client.get = AsyncMock(return_value=list_response)
created: Final = MagicMock()
created.json.return_value = {"name": "cachedContents/gateway-cache"}
self.mock_client.post.return_value = created
self.mock_async_client.post = AsyncMock(return_value=created)
messages: Final = [
{
"role": "user",
@ -319,7 +322,7 @@ class TestContextCachingEndpoints:
messages=messages,
optional_params={"tools": self.sample_tools},
api_key="unused",
api_base=api_base,
api_base="https://gateway.example.com/gemini/v1",
model="gemini-2.5-flash",
timeout=30.0,
logging_obj=self.mock_logging,
@ -328,6 +331,19 @@ class TestContextCachingEndpoints:
vertex_location=None,
vertex_auth_header=None,
)
return call_args
@pytest.mark.parametrize("is_async", [False, True])
@pytest.mark.asyncio
async def test_check_and_create_cache_sends_uncached_when_the_gemini_api_base_serves_no_cached_contents(
self, is_async: bool
):
not_served: Final = MagicMock()
not_served.raise_for_status.side_effect = httpx.HTTPStatusError(
"Not Found", request=MagicMock(), response=MagicMock(status_code=404, text="")
)
call_args: Final = self._gemini_v1_gateway_call_args(not_served)
result: Final = (
await self.context_caching.async_check_and_create_cache(client=self.mock_async_client, **call_args)
@ -335,9 +351,31 @@ class TestContextCachingEndpoints:
else self.context_caching.check_and_create_cache(client=self.mock_client, **call_args)
)
assert result == (messages, {"tools": self.sample_tools}, None)
assert not self.mock_client.mock_calls
assert not self.mock_async_client.mock_calls
assert result == (call_args["messages"], {"tools": self.sample_tools}, None)
self.mock_client.post.assert_not_called()
self.mock_async_client.post.assert_not_called()
@pytest.mark.parametrize("is_async", [False, True])
@pytest.mark.asyncio
async def test_check_and_create_cache_caches_through_a_gemini_v1_gateway_that_serves_cached_contents(
self, is_async: bool
):
empty_list: Final = MagicMock()
empty_list.json.return_value = {}
call_args: Final = self._gemini_v1_gateway_call_args(empty_list)
_, params, cache_name = (
await self.context_caching.async_check_and_create_cache(client=self.mock_async_client, **call_args)
if is_async
else self.context_caching.check_and_create_cache(client=self.mock_client, **call_args)
)
client: Final = self.mock_async_client if is_async else self.mock_client
assert params == {}
assert cache_name == "cachedContents/gateway-cache"
assert client.get.call_args.kwargs["url"] == "https://gateway.example.com/gemini/v1/cachedContents"
assert client.post.call_args.kwargs["url"].startswith("https://gateway.example.com/gemini/v1/cachedContents")
@pytest.mark.parametrize("custom_llm_provider", ["gemini", "vertex_ai", "vertex_ai_beta"])
@patch("litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.separate_cached_messages")
@ -1430,10 +1468,14 @@ class TestContextCachingEndpoints:
self._token_check_patcher.start()
@pytest.mark.parametrize(
("api_base_model", "deployment_model"),
[("gemini-2.5-pro", "gemini-2.5-flash"), ("gemini-not-in-the-cost-map", "gemini-2.5-pro")],
)
@pytest.mark.parametrize("is_async", [False, True])
@pytest.mark.asyncio
async def test_check_and_create_cache_applies_the_api_base_model_minimum(
self, local_model_cost_map, is_async: bool
async def test_check_and_create_cache_applies_the_stricter_model_minimum(
self, local_model_cost_map, is_async: bool, api_base_model: str, deployment_model: str
):
self._token_check_patcher.stop()
messages: Final = [
@ -1444,8 +1486,8 @@ class TestContextCachingEndpoints:
messages=messages,
optional_params={},
api_key="unused",
api_base="https://gateway.example.com/v1/projects/base-project/locations/global/publishers/google/models/gemini-2.5-pro",
model="gemini-2.5-flash",
api_base=f"https://gateway.example.com/v1/projects/base-project/locations/global/publishers/google/models/{api_base_model}",
model=deployment_model,
timeout=30.0,
logging_obj=self.mock_logging,
custom_llm_provider="vertex_ai",