mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-24 00:52:24 +00:00
fix(caching): updating gemini-1.5 minimum token requirements
This commit is contained in:
parent
0695f0702d
commit
a244ad63af
5 changed files with 29 additions and 6 deletions
|
|
@ -326,7 +326,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
target["cache_control"] = cache_control # type: ignore[typeddict-item]
|
||||
else:
|
||||
# Fallback for non-dict objects (shouldn't happen in practice)
|
||||
cast(Dict[str, Any], target)["cache_control"] = cache_control
|
||||
setattr(target, "cache_control", cache_control)
|
||||
|
||||
def translatable_anthropic_params(self) -> List:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -174,14 +174,18 @@ def get_gemini_context_caching_min_tokens(model: str) -> int:
|
|||
|
||||
Gemini rejects a cachedContents create below a per-model floor with a 400, so
|
||||
the caller skips caching below this value. Figures from
|
||||
https://ai.google.dev/gemini-api/docs/caching (Gemini 2.5 -> 2048, Gemini 3.x
|
||||
-> 4096). Unknown Gemini models default to the highest known floor so a create
|
||||
is never attempted below the real minimum.
|
||||
https://ai.google.dev/gemini-api/docs/caching (Gemini 1.5 -> 32768, Gemini 2.5
|
||||
-> 2048, Gemini 3.x -> 4096). Unknown Gemini models default to the highest
|
||||
known floor so a create is never attempted below the real minimum.
|
||||
"""
|
||||
model_lower = model.lower()
|
||||
if "gemini-1.5" in model_lower or "gemini-1-5" in model_lower:
|
||||
return 32768
|
||||
if "gemini-2.5" in model_lower or "gemini-2-5" in model_lower:
|
||||
return 2048
|
||||
return 4096
|
||||
if "gemini-3" in model_lower:
|
||||
return 4096
|
||||
return 32768
|
||||
|
||||
|
||||
def separate_cached_messages(
|
||||
|
|
|
|||
|
|
@ -1431,6 +1431,21 @@ def test_should_add_cache_control_for_gemini_model():
|
|||
assert target.get("cache_control") == cache_control
|
||||
|
||||
|
||||
def test_cache_control_fallback_setattr():
|
||||
"""Verify cache_control is safely assigned to non-dict target objects using setattr."""
|
||||
adapter = LiteLLMAnthropicMessagesAdapter()
|
||||
cache_control = {"type": "ephemeral"}
|
||||
|
||||
class MockTarget:
|
||||
pass
|
||||
|
||||
target = MockTarget()
|
||||
adapter._add_cache_control_if_applicable(
|
||||
{"cache_control": cache_control}, target, "claude-3-opus-20240229"
|
||||
)
|
||||
assert getattr(target, "cache_control", None) == cache_control
|
||||
|
||||
|
||||
def test_cache_control_preserved_in_text_content_for_gemini():
|
||||
"""cache_control must survive message translation for a Gemini target."""
|
||||
anthropic_messages = [
|
||||
|
|
|
|||
|
|
@ -14,6 +14,9 @@ class TestGeminiContextCachingMinTokens:
|
|||
@pytest.mark.parametrize(
|
||||
"model, expected",
|
||||
[
|
||||
("gemini-1.5-pro", 32768),
|
||||
("gemini-1.5-flash", 32768),
|
||||
("vertex_ai/gemini-1.5-pro-001", 32768),
|
||||
("gemini-2.5-flash", 2048),
|
||||
("gemini-2.5-pro", 2048),
|
||||
("gemini/gemini-2.5-pro", 2048),
|
||||
|
|
@ -21,7 +24,7 @@ class TestGeminiContextCachingMinTokens:
|
|||
("gemini-3.5-flash", 4096),
|
||||
("gemini-3.1-pro-preview", 4096),
|
||||
("gemini/gemini-3.5-flash", 4096),
|
||||
("gemini-1.5-pro", 4096),
|
||||
("gemini-unknown-future-model", 32768),
|
||||
],
|
||||
)
|
||||
def test_min_tokens_by_model(self, model, expected):
|
||||
|
|
|
|||
|
|
@ -1407,6 +1407,7 @@ class TestContextCachingEndpoints:
|
|||
("gemini-3.5-flash", 4096),
|
||||
("gemini/gemini-3.5-flash", 4096),
|
||||
("gemini-3.1-pro-preview", 4096),
|
||||
("gemini-1.5-pro", 32768),
|
||||
("gemini-2.5-flash", 2048),
|
||||
("gemini-2.5-pro", 2048),
|
||||
],
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue