fix(vertex_ai): check the cache token minimum against the api_base's model
Some checks failed
LiteLLM Rust DB / db-queries (push) Has been cancelled
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled

The cache is created for the model a publisher-model api_base names, so the minimum prompt size now comes from that model too. A prompt below its minimum is sent uncached instead of failing at cache create.
This commit is contained in:
mateo-berri 2026-10-28 21:39:48 -07:00
parent f606e4f749
commit d2396358e6
2 changed files with 39 additions and 4 deletions

View file

@ -422,14 +422,14 @@ class ContextCachingEndpoints(VertexBase):
if len(cached_messages) == 0:
return messages, optional_params, None
target: Final = _cache_target(custom_llm_provider, api_base, model, vertex_project, vertex_location)
skip_reason: Final = _context_caching_skip_reason(
cached_messages, optional_params.get("tools"), model, custom_llm_provider, api_base
cached_messages, optional_params.get("tools"), target.model, custom_llm_provider, api_base
)
if skip_reason is not None:
verbose_logger.debug("%s Skipping context caching.", skip_reason)
return messages, optional_params, None
target: Final = _cache_target(custom_llm_provider, api_base, model, vertex_project, vertex_location)
tools: Final = optional_params.pop("tools", None)
tool_choice: Final = optional_params.pop("tool_choice", None)
@ -573,14 +573,14 @@ class ContextCachingEndpoints(VertexBase):
if len(cached_messages) == 0:
return messages, optional_params, None
target: Final = _cache_target(custom_llm_provider, api_base, model, vertex_project, vertex_location)
skip_reason: Final = _context_caching_skip_reason(
cached_messages, optional_params.get("tools"), model, custom_llm_provider, api_base
cached_messages, optional_params.get("tools"), target.model, custom_llm_provider, api_base
)
if skip_reason is not None:
verbose_logger.debug("%s Skipping context caching.", skip_reason)
return messages, optional_params, None
target: Final = _cache_target(custom_llm_provider, api_base, model, vertex_project, vertex_location)
tools: Final = optional_params.pop("tools", None)
tool_choice: Final = optional_params.pop("tool_choice", None)

View file

@ -1430,6 +1430,41 @@ class TestContextCachingEndpoints:
self._token_check_patcher.start()
@pytest.mark.parametrize("is_async", [False, True])
@pytest.mark.asyncio
async def test_check_and_create_cache_applies_the_api_base_model_minimum(
self, local_model_cost_map, is_async: bool
):
self._token_check_patcher.stop()
messages: Final = [
{"role": "system", "content": " ".join(["word"] * 1500), "cache_control": {"type": "ephemeral"}},
{"role": "user", "content": "Hello"},
]
call_args: Final = dict(
messages=messages,
optional_params={},
api_key="unused",
api_base="https://gateway.example.com/v1/projects/base-project/locations/global/publishers/google/models/gemini-2.5-pro",
model="gemini-2.5-flash",
timeout=30.0,
logging_obj=self.mock_logging,
custom_llm_provider="vertex_ai",
vertex_project="config-project",
vertex_location="us-central1",
vertex_auth_header="vertex-token",
)
result: Final = (
await self.context_caching.async_check_and_create_cache(client=self.mock_async_client, **call_args)
if is_async
else self.context_caching.check_and_create_cache(client=self.mock_client, **call_args)
)
self._token_check_patcher.start()
assert result == (messages, {}, None)
assert not self.mock_client.mock_calls
assert not self.mock_async_client.mock_calls
@pytest.mark.parametrize("custom_llm_provider", ["gemini", "vertex_ai", "vertex_ai_beta"])
@patch("litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.separate_cached_messages")
@pytest.mark.asyncio