mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(vertex_ai): check the cache token minimum against the api_base's model
Some checks failed
LiteLLM Rust DB / db-queries (push) Has been cancelled
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
Some checks failed
LiteLLM Rust DB / db-queries (push) Has been cancelled
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
The cache is created for the model a publisher-model api_base names, so the minimum prompt size now comes from that model too. A prompt below its minimum is sent uncached instead of failing at cache create.
This commit is contained in:
parent
f606e4f749
commit
d2396358e6
2 changed files with 39 additions and 4 deletions
|
|
@ -422,14 +422,14 @@ class ContextCachingEndpoints(VertexBase):
|
|||
if len(cached_messages) == 0:
|
||||
return messages, optional_params, None
|
||||
|
||||
target: Final = _cache_target(custom_llm_provider, api_base, model, vertex_project, vertex_location)
|
||||
skip_reason: Final = _context_caching_skip_reason(
|
||||
cached_messages, optional_params.get("tools"), model, custom_llm_provider, api_base
|
||||
cached_messages, optional_params.get("tools"), target.model, custom_llm_provider, api_base
|
||||
)
|
||||
if skip_reason is not None:
|
||||
verbose_logger.debug("%s Skipping context caching.", skip_reason)
|
||||
return messages, optional_params, None
|
||||
|
||||
target: Final = _cache_target(custom_llm_provider, api_base, model, vertex_project, vertex_location)
|
||||
tools: Final = optional_params.pop("tools", None)
|
||||
tool_choice: Final = optional_params.pop("tool_choice", None)
|
||||
|
||||
|
|
@ -573,14 +573,14 @@ class ContextCachingEndpoints(VertexBase):
|
|||
if len(cached_messages) == 0:
|
||||
return messages, optional_params, None
|
||||
|
||||
target: Final = _cache_target(custom_llm_provider, api_base, model, vertex_project, vertex_location)
|
||||
skip_reason: Final = _context_caching_skip_reason(
|
||||
cached_messages, optional_params.get("tools"), model, custom_llm_provider, api_base
|
||||
cached_messages, optional_params.get("tools"), target.model, custom_llm_provider, api_base
|
||||
)
|
||||
if skip_reason is not None:
|
||||
verbose_logger.debug("%s Skipping context caching.", skip_reason)
|
||||
return messages, optional_params, None
|
||||
|
||||
target: Final = _cache_target(custom_llm_provider, api_base, model, vertex_project, vertex_location)
|
||||
tools: Final = optional_params.pop("tools", None)
|
||||
tool_choice: Final = optional_params.pop("tool_choice", None)
|
||||
|
||||
|
|
|
|||
|
|
@ -1430,6 +1430,41 @@ class TestContextCachingEndpoints:
|
|||
|
||||
self._token_check_patcher.start()
|
||||
|
||||
@pytest.mark.parametrize("is_async", [False, True])
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_and_create_cache_applies_the_api_base_model_minimum(
|
||||
self, local_model_cost_map, is_async: bool
|
||||
):
|
||||
self._token_check_patcher.stop()
|
||||
messages: Final = [
|
||||
{"role": "system", "content": " ".join(["word"] * 1500), "cache_control": {"type": "ephemeral"}},
|
||||
{"role": "user", "content": "Hello"},
|
||||
]
|
||||
call_args: Final = dict(
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
api_key="unused",
|
||||
api_base="https://gateway.example.com/v1/projects/base-project/locations/global/publishers/google/models/gemini-2.5-pro",
|
||||
model="gemini-2.5-flash",
|
||||
timeout=30.0,
|
||||
logging_obj=self.mock_logging,
|
||||
custom_llm_provider="vertex_ai",
|
||||
vertex_project="config-project",
|
||||
vertex_location="us-central1",
|
||||
vertex_auth_header="vertex-token",
|
||||
)
|
||||
|
||||
result: Final = (
|
||||
await self.context_caching.async_check_and_create_cache(client=self.mock_async_client, **call_args)
|
||||
if is_async
|
||||
else self.context_caching.check_and_create_cache(client=self.mock_client, **call_args)
|
||||
)
|
||||
|
||||
self._token_check_patcher.start()
|
||||
assert result == (messages, {}, None)
|
||||
assert not self.mock_client.mock_calls
|
||||
assert not self.mock_async_client.mock_calls
|
||||
|
||||
@pytest.mark.parametrize("custom_llm_provider", ["gemini", "vertex_ai", "vertex_ai_beta"])
|
||||
@patch("litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.separate_cached_messages")
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue