mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(vertex_ai): use the stricter cache minimum and send a Gemini base's 404 uncached
This commit is contained in:
parent
d2396358e6
commit
b80fd9bc93
3 changed files with 124 additions and 58 deletions
|
|
@ -9,7 +9,6 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict
|
|||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.caching.caching import Cache, LiteLLMCacheType
|
||||
from litellm.constants import MINIMUM_PROMPT_CACHE_TOKEN_COUNT
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
|
|
@ -18,7 +17,7 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
)
|
||||
from litellm.llms.openai.openai import AllMessageValues
|
||||
from litellm.types.llms.openai import ChatCompletionToolParam
|
||||
from litellm.utils import is_prompt_caching_valid_prompt
|
||||
from litellm.utils import get_prompt_cache_min_tokens, is_prompt_caching_valid_prompt
|
||||
|
||||
from ..common_utils import VertexAIError, get_vertex_base_url
|
||||
from ..vertex_llm_base import VertexBase, publisher_model_location
|
||||
|
|
@ -67,6 +66,19 @@ class _CreatedCachedContentRoot(RootModel[_CreatedCachedContent]):
|
|||
model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
|
||||
|
||||
|
||||
_TOOL_PARAMS: Final = ("tools", "tool_choice")
|
||||
|
||||
|
||||
def _sync_cache_client(timeout: float | httpx.Timeout | None) -> HTTPHandler:
|
||||
if timeout is None:
|
||||
return HTTPHandler()
|
||||
return HTTPHandler(timeout=httpx.Timeout(timeout) if isinstance(timeout, (float, int)) else timeout)
|
||||
|
||||
|
||||
class _CachedContentsNotServedError(VertexAIError):
|
||||
pass
|
||||
|
||||
|
||||
def _record_cache_creation_tokens(logging_obj: Logging, created: _CreatedCachedContent) -> None:
|
||||
tokens: Final = (created.get("usageMetadata") or {}).get("totalTokenCount") or 0
|
||||
logging_obj.vertex_ai_cache_creation_input_tokens = max(tokens, 0) # rebind-ok: set provider count
|
||||
|
|
@ -102,30 +114,36 @@ def _cache_target(
|
|||
return _CacheTarget(model=model_location.model, project=model_location.project, location=model_location.location)
|
||||
|
||||
|
||||
def _prompt_cache_min_tokens(models: Sequence[str], custom_llm_provider: str) -> int:
|
||||
return max(
|
||||
get_prompt_cache_min_tokens(
|
||||
model if model.startswith(custom_llm_provider) else f"{custom_llm_provider}/{model}"
|
||||
)
|
||||
for model in models
|
||||
)
|
||||
|
||||
|
||||
def _context_caching_skip_reason(
|
||||
cached_messages: Sequence[AllMessageValues],
|
||||
tools: Sequence[ChatCompletionToolParam] | None,
|
||||
model: str,
|
||||
deployment_model: str,
|
||||
cache_model: str,
|
||||
custom_llm_provider: Literal["vertex_ai", "vertex_ai_beta", "gemini"],
|
||||
api_base: str | None,
|
||||
) -> str | None:
|
||||
if not cached_messages_end_on_supported_turn(cached_messages):
|
||||
return (
|
||||
"Vertex AI context caching: cached message block ends on a model turn once "
|
||||
"system messages are extracted, which the cachedContents API rejects."
|
||||
)
|
||||
min_tokens: Final = _prompt_cache_min_tokens((deployment_model, cache_model), custom_llm_provider)
|
||||
if not is_prompt_caching_valid_prompt(
|
||||
model=model,
|
||||
model=cache_model,
|
||||
messages=list(cached_messages),
|
||||
tools=None if tools is None else list(tools),
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
min_token_count=min_tokens,
|
||||
):
|
||||
return (
|
||||
"Vertex AI context caching: cached content is below minimum token "
|
||||
f"count ({MINIMUM_PROMPT_CACHE_TOKEN_COUNT})."
|
||||
)
|
||||
if custom_llm_provider == "gemini" and api_base is not None and api_base.rstrip("/").endswith("/v1"):
|
||||
return "Gemini context caching: the Gemini API serves cachedContents only under v1beta, and this api_base names v1."
|
||||
return f"Vertex AI context caching: cached content is below minimum token count ({min_tokens})."
|
||||
return None
|
||||
|
||||
|
||||
|
|
@ -258,6 +276,8 @@ class ContextCachingEndpoints(VertexBase):
|
|||
except httpx.HTTPStatusError as e:
|
||||
if e.response.status_code == 403:
|
||||
return None
|
||||
if e.response.status_code == 404 and custom_llm_provider == "gemini":
|
||||
raise _CachedContentsNotServedError(status_code=404, message=e.response.text)
|
||||
raise VertexAIError(status_code=e.response.status_code, message=e.response.text)
|
||||
except Exception as e:
|
||||
raise VertexAIError(status_code=500, message=str(e))
|
||||
|
|
@ -357,6 +377,8 @@ class ContextCachingEndpoints(VertexBase):
|
|||
except httpx.HTTPStatusError as e:
|
||||
if e.response.status_code == 403:
|
||||
return None
|
||||
if e.response.status_code == 404 and custom_llm_provider == "gemini":
|
||||
raise _CachedContentsNotServedError(status_code=404, message=e.response.text)
|
||||
raise VertexAIError(status_code=e.response.status_code, message=e.response.text)
|
||||
except Exception as e:
|
||||
raise VertexAIError(status_code=500, message=str(e))
|
||||
|
|
@ -423,16 +445,15 @@ class ContextCachingEndpoints(VertexBase):
|
|||
return messages, optional_params, None
|
||||
|
||||
target: Final = _cache_target(custom_llm_provider, api_base, model, vertex_project, vertex_location)
|
||||
tools: Final = optional_params.get("tools")
|
||||
tool_choice: Final = optional_params.get("tool_choice")
|
||||
skip_reason: Final = _context_caching_skip_reason(
|
||||
cached_messages, optional_params.get("tools"), target.model, custom_llm_provider, api_base
|
||||
cached_messages, tools, model, target.model, custom_llm_provider
|
||||
)
|
||||
if skip_reason is not None:
|
||||
verbose_logger.debug("%s Skipping context caching.", skip_reason)
|
||||
return messages, optional_params, None
|
||||
|
||||
tools: Final = optional_params.pop("tools", None)
|
||||
tool_choice: Final = optional_params.pop("tool_choice", None)
|
||||
|
||||
## AUTHORIZATION ##
|
||||
token, url = self._get_token_and_url_context_caching(
|
||||
gemini_api_key=api_key,
|
||||
|
|
@ -454,33 +475,31 @@ class ContextCachingEndpoints(VertexBase):
|
|||
if extra_headers is not None:
|
||||
headers.update(extra_headers)
|
||||
|
||||
if client is None or not isinstance(client, HTTPHandler):
|
||||
_params: Final = {}
|
||||
if timeout is not None:
|
||||
if isinstance(timeout, float) or isinstance(timeout, int):
|
||||
timeout = httpx.Timeout(timeout)
|
||||
_params["timeout"] = timeout
|
||||
client = HTTPHandler(**_params)
|
||||
else:
|
||||
client = client
|
||||
http_client: Final = client if isinstance(client, HTTPHandler) else _sync_cache_client(timeout)
|
||||
|
||||
## CHECK IF CACHED ALREADY
|
||||
generated_cache_key: Final = local_cache_obj.get_cache_key(
|
||||
messages=cached_messages, tools=tools, tool_choice=tool_choice, model=target.model
|
||||
)
|
||||
google_cache_name: Final = self.check_cache(
|
||||
cache_key=generated_cache_key,
|
||||
client=client,
|
||||
headers=headers,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
logging_obj=logging_obj,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
vertex_project=target.project,
|
||||
vertex_location=target.location,
|
||||
vertex_auth_header=vertex_auth_header,
|
||||
model=target.model,
|
||||
)
|
||||
try:
|
||||
google_cache_name: Final = self.check_cache(
|
||||
cache_key=generated_cache_key,
|
||||
client=http_client,
|
||||
headers=headers,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
logging_obj=logging_obj,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
vertex_project=target.project,
|
||||
vertex_location=target.location,
|
||||
vertex_auth_header=vertex_auth_header,
|
||||
model=target.model,
|
||||
)
|
||||
except _CachedContentsNotServedError:
|
||||
verbose_logger.debug("Gemini context caching: %s answers 404. Skipping context caching.", url)
|
||||
return messages, optional_params, None
|
||||
for tool_param in _TOOL_PARAMS:
|
||||
optional_params.pop(tool_param, None)
|
||||
if google_cache_name:
|
||||
return non_cached_messages, optional_params, google_cache_name
|
||||
|
||||
|
|
@ -510,7 +529,7 @@ class ContextCachingEndpoints(VertexBase):
|
|||
)
|
||||
|
||||
try:
|
||||
response: Final = client.post(
|
||||
response: Final = http_client.post(
|
||||
url=url,
|
||||
headers=headers,
|
||||
json=cached_content_request_body,
|
||||
|
|
@ -574,16 +593,15 @@ class ContextCachingEndpoints(VertexBase):
|
|||
return messages, optional_params, None
|
||||
|
||||
target: Final = _cache_target(custom_llm_provider, api_base, model, vertex_project, vertex_location)
|
||||
tools: Final = optional_params.get("tools")
|
||||
tool_choice: Final = optional_params.get("tool_choice")
|
||||
skip_reason: Final = _context_caching_skip_reason(
|
||||
cached_messages, optional_params.get("tools"), target.model, custom_llm_provider, api_base
|
||||
cached_messages, tools, model, target.model, custom_llm_provider
|
||||
)
|
||||
if skip_reason is not None:
|
||||
verbose_logger.debug("%s Skipping context caching.", skip_reason)
|
||||
return messages, optional_params, None
|
||||
|
||||
tools: Final = optional_params.pop("tools", None)
|
||||
tool_choice: Final = optional_params.pop("tool_choice", None)
|
||||
|
||||
## AUTHORIZATION ##
|
||||
token, url = self._get_token_and_url_context_caching(
|
||||
gemini_api_key=api_key,
|
||||
|
|
@ -681,9 +699,15 @@ class ContextCachingEndpoints(VertexBase):
|
|||
)
|
||||
return raw_response_cached.get("name")
|
||||
|
||||
cache_name: Final = await self._known_caches.single_flight(
|
||||
scope, generated_cache_key, lookup_or_create, _wait_seconds(timeout)
|
||||
)
|
||||
try:
|
||||
cache_name: Final = await self._known_caches.single_flight(
|
||||
scope, generated_cache_key, lookup_or_create, _wait_seconds(timeout)
|
||||
)
|
||||
except _CachedContentsNotServedError:
|
||||
verbose_logger.debug("Gemini context caching: %s answers 404. Skipping context caching.", url)
|
||||
return messages, optional_params, None
|
||||
for tool_param in _TOOL_PARAMS:
|
||||
optional_params.pop(tool_param, None)
|
||||
return non_cached_messages, optional_params, cache_name
|
||||
|
||||
def get_cache(self):
|
||||
|
|
|
|||
|
|
@ -376,7 +376,7 @@ class RequestBody(TypedDict, total=False):
|
|||
class CachedContentRequestBody(TypedDict, total=False):
|
||||
contents: Required[list[ContentType]]
|
||||
system_instruction: SystemInstructions
|
||||
tools: Tools
|
||||
tools: Tools | None
|
||||
toolConfig: ToolConfig
|
||||
model: Required[str] # Format: models/{model}
|
||||
ttl: str # ending in 's' - Example: "3.5s".
|
||||
|
|
|
|||
|
|
@ -304,10 +304,13 @@ class TestContextCachingEndpoints:
|
|||
]
|
||||
assert cache_for("gemini-2.5-pro") == pro_cache
|
||||
|
||||
@pytest.mark.parametrize("api_base", ["https://gateway.example.com/v1", "https://gateway.example.com/v1/"])
|
||||
@pytest.mark.parametrize("is_async", [False, True])
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_and_create_cache_skips_a_gemini_v1_api_base(self, api_base: str, is_async: bool):
|
||||
def _gemini_v1_gateway_call_args(self, list_response: MagicMock):
|
||||
self.mock_client.get.return_value = list_response
|
||||
self.mock_async_client.get = AsyncMock(return_value=list_response)
|
||||
created: Final = MagicMock()
|
||||
created.json.return_value = {"name": "cachedContents/gateway-cache"}
|
||||
self.mock_client.post.return_value = created
|
||||
self.mock_async_client.post = AsyncMock(return_value=created)
|
||||
messages: Final = [
|
||||
{
|
||||
"role": "user",
|
||||
|
|
@ -319,7 +322,7 @@ class TestContextCachingEndpoints:
|
|||
messages=messages,
|
||||
optional_params={"tools": self.sample_tools},
|
||||
api_key="unused",
|
||||
api_base=api_base,
|
||||
api_base="https://gateway.example.com/gemini/v1",
|
||||
model="gemini-2.5-flash",
|
||||
timeout=30.0,
|
||||
logging_obj=self.mock_logging,
|
||||
|
|
@ -328,6 +331,19 @@ class TestContextCachingEndpoints:
|
|||
vertex_location=None,
|
||||
vertex_auth_header=None,
|
||||
)
|
||||
return call_args
|
||||
|
||||
@pytest.mark.parametrize("is_async", [False, True])
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_and_create_cache_sends_uncached_when_the_gemini_api_base_serves_no_cached_contents(
|
||||
self, is_async: bool
|
||||
):
|
||||
not_served: Final = MagicMock()
|
||||
not_served.raise_for_status.side_effect = httpx.HTTPStatusError(
|
||||
"Not Found", request=MagicMock(), response=MagicMock(status_code=404, text="")
|
||||
)
|
||||
|
||||
call_args: Final = self._gemini_v1_gateway_call_args(not_served)
|
||||
|
||||
result: Final = (
|
||||
await self.context_caching.async_check_and_create_cache(client=self.mock_async_client, **call_args)
|
||||
|
|
@ -335,9 +351,31 @@ class TestContextCachingEndpoints:
|
|||
else self.context_caching.check_and_create_cache(client=self.mock_client, **call_args)
|
||||
)
|
||||
|
||||
assert result == (messages, {"tools": self.sample_tools}, None)
|
||||
assert not self.mock_client.mock_calls
|
||||
assert not self.mock_async_client.mock_calls
|
||||
assert result == (call_args["messages"], {"tools": self.sample_tools}, None)
|
||||
self.mock_client.post.assert_not_called()
|
||||
self.mock_async_client.post.assert_not_called()
|
||||
|
||||
@pytest.mark.parametrize("is_async", [False, True])
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_and_create_cache_caches_through_a_gemini_v1_gateway_that_serves_cached_contents(
|
||||
self, is_async: bool
|
||||
):
|
||||
empty_list: Final = MagicMock()
|
||||
empty_list.json.return_value = {}
|
||||
|
||||
call_args: Final = self._gemini_v1_gateway_call_args(empty_list)
|
||||
|
||||
_, params, cache_name = (
|
||||
await self.context_caching.async_check_and_create_cache(client=self.mock_async_client, **call_args)
|
||||
if is_async
|
||||
else self.context_caching.check_and_create_cache(client=self.mock_client, **call_args)
|
||||
)
|
||||
client: Final = self.mock_async_client if is_async else self.mock_client
|
||||
|
||||
assert params == {}
|
||||
assert cache_name == "cachedContents/gateway-cache"
|
||||
assert client.get.call_args.kwargs["url"] == "https://gateway.example.com/gemini/v1/cachedContents"
|
||||
assert client.post.call_args.kwargs["url"].startswith("https://gateway.example.com/gemini/v1/cachedContents")
|
||||
|
||||
@pytest.mark.parametrize("custom_llm_provider", ["gemini", "vertex_ai", "vertex_ai_beta"])
|
||||
@patch("litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.separate_cached_messages")
|
||||
|
|
@ -1430,10 +1468,14 @@ class TestContextCachingEndpoints:
|
|||
|
||||
self._token_check_patcher.start()
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("api_base_model", "deployment_model"),
|
||||
[("gemini-2.5-pro", "gemini-2.5-flash"), ("gemini-not-in-the-cost-map", "gemini-2.5-pro")],
|
||||
)
|
||||
@pytest.mark.parametrize("is_async", [False, True])
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_and_create_cache_applies_the_api_base_model_minimum(
|
||||
self, local_model_cost_map, is_async: bool
|
||||
async def test_check_and_create_cache_applies_the_stricter_model_minimum(
|
||||
self, local_model_cost_map, is_async: bool, api_base_model: str, deployment_model: str
|
||||
):
|
||||
self._token_check_patcher.stop()
|
||||
messages: Final = [
|
||||
|
|
@ -1444,8 +1486,8 @@ class TestContextCachingEndpoints:
|
|||
messages=messages,
|
||||
optional_params={},
|
||||
api_key="unused",
|
||||
api_base="https://gateway.example.com/v1/projects/base-project/locations/global/publishers/google/models/gemini-2.5-pro",
|
||||
model="gemini-2.5-flash",
|
||||
api_base=f"https://gateway.example.com/v1/projects/base-project/locations/global/publishers/google/models/{api_base_model}",
|
||||
model=deployment_model,
|
||||
timeout=30.0,
|
||||
logging_obj=self.mock_logging,
|
||||
custom_llm_provider="vertex_ai",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue