fix(caching): updating gemini-1.5 minimum token requirements

This commit is contained in:
Elliott de Launay 2026-07-09 23:25:01 -04:00
parent 0695f0702d
commit a244ad63af
5 changed files with 29 additions and 6 deletions

View file

@ -326,7 +326,7 @@ class LiteLLMAnthropicMessagesAdapter:
target["cache_control"] = cache_control # type: ignore[typeddict-item]
else:
# Fallback for non-dict objects (shouldn't happen in practice)
cast(Dict[str, Any], target)["cache_control"] = cache_control
setattr(target, "cache_control", cache_control)
def translatable_anthropic_params(self) -> List:
"""

View file

@ -174,14 +174,18 @@ def get_gemini_context_caching_min_tokens(model: str) -> int:
Gemini rejects a cachedContents create below a per-model floor with a 400, so
the caller skips caching below this value. Figures from
https://ai.google.dev/gemini-api/docs/caching (Gemini 2.5 -> 2048, Gemini 3.x
-> 4096). Unknown Gemini models default to the highest known floor so a create
is never attempted below the real minimum.
https://ai.google.dev/gemini-api/docs/caching (Gemini 1.5 -> 32768, Gemini 2.5
-> 2048, Gemini 3.x -> 4096). Unknown Gemini models default to the highest
known floor so a create is never attempted below the real minimum.
"""
model_lower = model.lower()
if "gemini-1.5" in model_lower or "gemini-1-5" in model_lower:
return 32768
if "gemini-2.5" in model_lower or "gemini-2-5" in model_lower:
return 2048
return 4096
if "gemini-3" in model_lower:
return 4096
return 32768
def separate_cached_messages(

View file

@ -1431,6 +1431,21 @@ def test_should_add_cache_control_for_gemini_model():
assert target.get("cache_control") == cache_control
def test_cache_control_fallback_setattr():
"""Verify cache_control is safely assigned to non-dict target objects using setattr."""
adapter = LiteLLMAnthropicMessagesAdapter()
cache_control = {"type": "ephemeral"}
class MockTarget:
pass
target = MockTarget()
adapter._add_cache_control_if_applicable(
{"cache_control": cache_control}, target, "claude-3-opus-20240229"
)
assert getattr(target, "cache_control", None) == cache_control
def test_cache_control_preserved_in_text_content_for_gemini():
"""cache_control must survive message translation for a Gemini target."""
anthropic_messages = [

View file

@ -14,6 +14,9 @@ class TestGeminiContextCachingMinTokens:
@pytest.mark.parametrize(
"model, expected",
[
("gemini-1.5-pro", 32768),
("gemini-1.5-flash", 32768),
("vertex_ai/gemini-1.5-pro-001", 32768),
("gemini-2.5-flash", 2048),
("gemini-2.5-pro", 2048),
("gemini/gemini-2.5-pro", 2048),
@ -21,7 +24,7 @@ class TestGeminiContextCachingMinTokens:
("gemini-3.5-flash", 4096),
("gemini-3.1-pro-preview", 4096),
("gemini/gemini-3.5-flash", 4096),
("gemini-1.5-pro", 4096),
("gemini-unknown-future-model", 32768),
],
)
def test_min_tokens_by_model(self, model, expected):

View file

@ -1407,6 +1407,7 @@ class TestContextCachingEndpoints:
("gemini-3.5-flash", 4096),
("gemini/gemini-3.5-flash", 4096),
("gemini-3.1-pro-preview", 4096),
("gemini-1.5-pro", 32768),
("gemini-2.5-flash", 2048),
("gemini-2.5-pro", 2048),
],