scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 35

fix(model_prices): set prompt_cache_min_tokens=4096 for Gemini 3.5/3.6/3.7 Flash and 3.1 Pro Preview

Browse source
This commit is contained in:
mateo-berri 2026-08-19 14:13:00 -07:00
parent e126975468
commit 77716eeaed
3 changed files with 70 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
litellm/model_prices_and_context_window_backup.json
View file

@ -19403,6 +19403,7 @@
"web_search_billing_unit": "per_query"
},
"gemini-3.1-pro-preview": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_creation_input_token_cost_above_200k_tokens": 2.5e-07,
@ -19460,6 +19461,7 @@
"web_search_billing_unit": "per_query"
},
"gemini-3.1-pro-preview-customtools": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_creation_input_token_cost_above_200k_tokens": 2.5e-07,
@ -19614,6 +19616,7 @@
"web_search_billing_unit": "per_query"
},
"vertex_ai/gemini-3.5-flash": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 1.5e-07,
"input_cost_per_token": 1.5e-06,
"input_cost_per_audio_token": 1e-06,
@ -19665,6 +19668,7 @@
"web_search_billing_unit": "per_query"
},
"vertex_ai/gemini-3.6-flash": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_flex": 3.75e-08,
"input_cost_per_token": 7.5e-07,
@ -19719,6 +19723,7 @@
"web_search_billing_unit": "per_query"
},
"vertex_ai/gemini-3.7-flash": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_flex": 3.75e-08,
"input_cost_per_token": 7.5e-07,
@ -19773,6 +19778,7 @@
"web_search_billing_unit": "per_query"
},
"vertex_ai/gemini-3.1-pro-preview": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_creation_input_token_cost_above_200k_tokens": 2.5e-07,
@ -19830,6 +19836,7 @@
"web_search_billing_unit": "per_query"
},
"vertex_ai/gemini-3.1-pro-preview-customtools": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_creation_input_token_cost_above_200k_tokens": 2.5e-07,
@ -21337,6 +21344,7 @@
"web_search_billing_unit": "per_query"
},
"gemini/gemini-3.5-flash": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 1.5e-07,
"input_cost_per_audio_token": 1e-06,
"input_cost_per_token": 1.5e-06,
@ -21391,6 +21399,7 @@
"web_search_billing_unit": "per_query"
},
"gemini/gemini-3.6-flash": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_flex": 3.75e-08,
"input_cost_per_token": 7.5e-07,
@ -21448,6 +21457,7 @@
"web_search_billing_unit": "per_query"
},
"gemini/gemini-3.7-flash": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_flex": 3.75e-08,
"input_cost_per_token": 7.5e-07,
@ -21538,6 +21548,7 @@
"tpm": 800000
},
"gemini/gemini-3.1-pro-preview": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"input_cost_per_token": 2e-06,
@ -21595,6 +21606,7 @@
"web_search_billing_unit": "per_query"
},
"gemini/gemini-3.1-pro-preview-customtools": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"input_cost_per_token": 2e-06,
@ -21733,6 +21745,7 @@
"supports_vision": true
},
"gemini-3.5-flash": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 1.5e-07,
"input_cost_per_audio_token": 1e-06,
"input_cost_per_token": 1.5e-06,
@ -21785,6 +21798,7 @@
"web_search_billing_unit": "per_query"
},
"gemini-3.6-flash": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_flex": 3.75e-08,
"input_cost_per_token": 7.5e-07,
@ -21840,6 +21854,7 @@
"web_search_billing_unit": "per_query"
},
"gemini-3.7-flash": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_flex": 3.75e-08,
"input_cost_per_token": 7.5e-07,

15
model_prices_and_context_window.json
View file

@ -19403,6 +19403,7 @@
"web_search_billing_unit": "per_query"
},
"gemini-3.1-pro-preview": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_creation_input_token_cost_above_200k_tokens": 2.5e-07,
@ -19460,6 +19461,7 @@
"web_search_billing_unit": "per_query"
},
"gemini-3.1-pro-preview-customtools": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_creation_input_token_cost_above_200k_tokens": 2.5e-07,
@ -19614,6 +19616,7 @@
"web_search_billing_unit": "per_query"
},
"vertex_ai/gemini-3.5-flash": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 1.5e-07,
"input_cost_per_token": 1.5e-06,
"input_cost_per_audio_token": 1e-06,
@ -19665,6 +19668,7 @@
"web_search_billing_unit": "per_query"
},
"vertex_ai/gemini-3.6-flash": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_flex": 3.75e-08,
"input_cost_per_token": 7.5e-07,
@ -19719,6 +19723,7 @@
"web_search_billing_unit": "per_query"
},
"vertex_ai/gemini-3.7-flash": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_flex": 3.75e-08,
"input_cost_per_token": 7.5e-07,
@ -19773,6 +19778,7 @@
"web_search_billing_unit": "per_query"
},
"vertex_ai/gemini-3.1-pro-preview": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_creation_input_token_cost_above_200k_tokens": 2.5e-07,
@ -19830,6 +19836,7 @@
"web_search_billing_unit": "per_query"
},
"vertex_ai/gemini-3.1-pro-preview-customtools": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_creation_input_token_cost_above_200k_tokens": 2.5e-07,
@ -21337,6 +21344,7 @@
"web_search_billing_unit": "per_query"
},
"gemini/gemini-3.5-flash": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 1.5e-07,
"input_cost_per_audio_token": 1e-06,
"input_cost_per_token": 1.5e-06,
@ -21391,6 +21399,7 @@
"web_search_billing_unit": "per_query"
},
"gemini/gemini-3.6-flash": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_flex": 3.75e-08,
"input_cost_per_token": 7.5e-07,
@ -21448,6 +21457,7 @@
"web_search_billing_unit": "per_query"
},
"gemini/gemini-3.7-flash": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_flex": 3.75e-08,
"input_cost_per_token": 7.5e-07,
@ -21538,6 +21548,7 @@
"tpm": 800000
},
"gemini/gemini-3.1-pro-preview": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"input_cost_per_token": 2e-06,
@ -21595,6 +21606,7 @@
"web_search_billing_unit": "per_query"
},
"gemini/gemini-3.1-pro-preview-customtools": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"input_cost_per_token": 2e-06,
@ -21733,6 +21745,7 @@
"supports_vision": true
},
"gemini-3.5-flash": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 1.5e-07,
"input_cost_per_audio_token": 1e-06,
"input_cost_per_token": 1.5e-06,
@ -21785,6 +21798,7 @@
"web_search_billing_unit": "per_query"
},
"gemini-3.6-flash": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_flex": 3.75e-08,
"input_cost_per_token": 7.5e-07,
@ -21840,6 +21854,7 @@
"web_search_billing_unit": "per_query"
},
"gemini-3.7-flash": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_flex": 3.75e-08,
"input_cost_per_token": 7.5e-07,

40
tests/test_litellm/test_utils.py
View file

@ -2,6 +2,7 @@ import json
import logging
import os
import sys
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@ -4384,6 +4385,45 @@ def test_get_prompt_cache_min_tokens_differs_per_platform_for_same_model(local_m
)
GEMINI_4096_CACHE_MIN_MODELS: Final = tuple(
prefix + base
for base in (
"gemini-3.5-flash",
"gemini-3.6-flash",
"gemini-3.7-flash",
"gemini-3.1-pro-preview",
"gemini-3.1-pro-preview-customtools",
)
for prefix in ("", "gemini/", "vertex_ai/")
)
def test_gemini_3_flash_and_31_pro_preview_resolve_4096_cache_minimum(local_model_cost_map: None) -> None:
"""Regression for the cost map missing prompt_cache_min_tokens on these models: Google rejects
explicit caching below 4,096 tokens for them (https://ai.google.dev/gemini-api/docs/caching), so
the 1024 default sent cachedContents creates Vertex answered with a hard 400."""
wrong: Final = {
model: get_prompt_cache_min_tokens(model=model)
for model in GEMINI_4096_CACHE_MIN_MODELS
if get_prompt_cache_min_tokens(model=model) != 4096
}
assert not wrong, f"prompt_cache_min_tokens must be 4096: {wrong}"
def test_gemini_4096_cache_minimum_present_in_root_cost_map() -> None:
"""The root map ships to the CDN independently of the bundled backup, so both must carry the
minimum or proxies reading one of them regress to the 1024 default."""
root_map_path: Final = os.path.join(os.path.dirname(__file__), "..", "..", "model_prices_and_context_window.json")
with open(root_map_path) as f:
root_map: Final = json.load(f)
wrong: Final = {
model: root_map[model].get("prompt_cache_min_tokens")
for model in GEMINI_4096_CACHE_MIN_MODELS
if root_map[model].get("prompt_cache_min_tokens") != 4096
}
assert not wrong, f"prompt_cache_min_tokens must be 4096: {wrong}"
def test_get_prompt_cache_min_tokens_unmapped_model_falls_back_to_default(local_model_cost_map: None) -> None:
"""get_model_info raises for a model it has no entry for. The resolver must swallow that and
fall back to the default, otherwise the raise reaches callers that would read it as