scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-30 06:43:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 249

fix(model_cost): resync backup cost map with canonical and guard against drift

Browse source 
The two shipped cost maps had drifted. `model_prices_and_context_window.json` is
the canonical map; `litellm/model_prices_and_context_window_backup.json` is bundled
into the package and is what gets used when the remote fetch fails or when
LITELLM_LOCAL_MODEL_COST_MAP=True. Users were silently getting different model
metadata depending on which one loaded:

- gpt-5.4-mini / gpt-5.4-nano (and dated aliases): max_input_tokens 1050000 vs
  272000. The sibling gpt-5.4 entry uses 1050000, so the backup was carrying a
  stale gpt-5-generation value.
- gpt-5-pro / gpt-5-pro-2025-10-06: max_input_tokens 400000 vs 128000 and
  max_output_tokens 128000 vs 272000. The backup values are transposed - its
  input window was smaller than the base gpt-5 entry's 272000.
- github_copilot/claude-{haiku,opus,sonnet}-4.5: supported_endpoints was missing
  /v1/messages in the canonical map. Here the backup was the correct side;
  0b0fd6a4d1 (#31802) only updated the backup.
- snowflake/claude-sonnet-4-6: supports_adaptive_thinking missing from the backup;
  98ced0ae43 only updated the canonical map.
- Five entries absent from the backup entirely: gemini-3-pro-image,
  gemini-3.1-flash-image, openrouter/z-ai/glm-5.1, zai/glm-4.7-flash, zai/glm-5.1.

Drift ran in both directions, so this resolves each divergence against the commit
that introduced it rather than blindly copying one file over the other. The
/v1/messages endpoints are added to the canonical map first; everything else is
already correct there, so the backup is then regenerated from it via the existing
ci_cd/check_files_match.py.

ci_cd/check_files_match.py was meant to prevent exactly this, but it is referenced
nowhere: it used to run as a pre-commit hook and 51af6fedb3 removed
.pre-commit-config.yaml when hooks moved to `make pre-commit`, without carrying it
over. The existing per-model checks (e.g. test_gpt_5_6_backup_matches_main) only
assert the specific models each was written for, so nothing caught this.

Adds tests/test_litellm/test_model_cost_map_sync.py, which asserts the invariant
globally in CI and reports the actual divergences on failure.

Signed-off-by: onatozmenn <onatozmen44@gmail.com>
This commit is contained in:
onatozmenn 2026-07-24 21:27:41 +03:00
parent f6a1050cbf
commit 94897ad8a8
No known key found for this signature in database
3 changed files with 261 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

217
litellm/model_prices_and_context_window_backup.json
View file

@ -17670,6 +17670,46 @@
"tpm": 8000000,
"supports_image_size": false
},
"gemini-3-pro-image": {
"input_cost_per_image": 0.0011,
"input_cost_per_token": 2e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "vertex_ai-language-models",
"max_input_tokens": 65536,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "image_generation",
"output_cost_per_image": 0.134,
"output_cost_per_image_token": 0.00012,
"output_cost_per_token": 1.2e-05,
"output_cost_per_token_batches": 6e-06,
"source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3-pro-image",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text",
"image"
],
"supports_function_calling": false,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true,
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query"
},
"gemini-3-pro-image-preview": {
"input_cost_per_image": 0.0011,
"input_cost_per_token": 2e-06,
@ -17710,6 +17750,44 @@
},
"web_search_billing_unit": "per_query"
},
"gemini-3.1-flash-image": {
"input_cost_per_image": 0.00056,
"input_cost_per_token": 5e-07,
"litellm_provider": "vertex_ai-language-models",
"max_input_tokens": 65536,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "image_generation",
"output_cost_per_image": 0.0672,
"output_cost_per_image_token": 6e-05,
"output_cost_per_token": 3e-06,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text",
"image"
],
"supports_function_calling": false,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true,
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query"
},
"gemini-3.1-flash-image-preview": {
"input_cost_per_image": 0.00056,
"input_cost_per_token": 5e-07,
@ -19264,7 +19342,6 @@
],
"supports_function_calling": false,
"supports_prompt_caching": true,
"supports_reasoning": false,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_vision": true,
@ -19274,7 +19351,8 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query"
"web_search_billing_unit": "per_query",
"supports_reasoning": false
},
"gemini/gemini-3-pro-image-preview": {
"input_cost_per_image": 0.0011,
@ -24239,7 +24317,7 @@
"input_cost_per_token_batches": 3.75e-07,
"input_cost_per_token_priority": 1.5e-06,
"litellm_provider": "openai",
"max_input_tokens": 272000,
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
@ -24285,7 +24363,7 @@
"input_cost_per_token_batches": 3.75e-07,
"input_cost_per_token_priority": 1.5e-06,
"litellm_provider": "openai",
"max_input_tokens": 272000,
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
@ -24329,7 +24407,7 @@
"input_cost_per_token_flex": 1e-07,
"input_cost_per_token_batches": 1e-07,
"litellm_provider": "openai",
"max_input_tokens": 272000,
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
@ -24372,7 +24450,7 @@
"input_cost_per_token_flex": 1e-07,
"input_cost_per_token_batches": 1e-07,
"litellm_provider": "openai",
"max_input_tokens": 272000,
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
@ -24412,9 +24490,9 @@
"input_cost_per_token": 1.5e-05,
"input_cost_per_token_batches": 7.5e-06,
"litellm_provider": "openai",
"max_input_tokens": 128000,
"max_output_tokens": 272000,
"max_tokens": 272000,
"max_input_tokens": 400000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
"output_cost_per_token": 0.00012,
"output_cost_per_token_batches": 6e-05,
@ -24448,9 +24526,9 @@
"input_cost_per_token": 1.5e-05,
"input_cost_per_token_batches": 7.5e-06,
"litellm_provider": "openai",
"max_input_tokens": 128000,
"max_output_tokens": 272000,
"max_tokens": 272000,
"max_input_tokens": 400000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
"output_cost_per_token": 0.00012,
"output_cost_per_token_batches": 6e-05,
@ -31813,6 +31891,22 @@
"supports_reasoning": true,
"supports_tool_choice": true
},
"openrouter/z-ai/glm-5.1": {
"input_cost_per_token": 1.05e-06,
"output_cost_per_token": 3.5e-06,
"cache_read_input_token_cost": 5.25e-07,
"cache_creation_input_token_cost": 0.0,
"litellm_provider": "openrouter",
"max_input_tokens": 202752,
"max_output_tokens": 65535,
"max_tokens": 65535,
"mode": "chat",
"source": "https://openrouter.ai/z-ai/glm-5.1",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
},
"openrouter/minimax/minimax-m2.1": {
"input_cost_per_token": 2.7e-07,
"output_cost_per_token": 1.2e-06,
@ -40086,6 +40180,21 @@
"supports_tool_choice": true,
"source": "https://docs.z.ai/guides/overview/pricing"
},
"zai/glm-5.1": {
"cache_creation_input_token_cost": 0,
"cache_read_input_token_cost": 2.6e-07,
"input_cost_per_token": 1.4e-06,
"output_cost_per_token": 4.4e-06,
"litellm_provider": "zai",
"max_input_tokens": 200000,
"max_output_tokens": 128000,
"mode": "chat",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"source": "https://docs.z.ai/guides/overview/pricing"
},
"zai/glm-5-code": {
"cache_creation_input_token_cost": 0,
"cache_read_input_token_cost": 3e-07,
@ -40116,6 +40225,21 @@
"supports_tool_choice": true,
"source": "https://docs.z.ai/guides/overview/pricing"
},
"zai/glm-4.7-flash": {
"cache_creation_input_token_cost": 0,
"cache_read_input_token_cost": 0,
"input_cost_per_token": 0,
"output_cost_per_token": 0,
"litellm_provider": "zai",
"max_input_tokens": 200000,
"max_output_tokens": 128000,
"mode": "chat",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"source": "https://docs.z.ai/guides/overview/pricing"
},
"zai/glm-4.6": {
"cache_creation_input_token_cost": 0,
"cache_read_input_token_cost": 1.1e-07,
@ -45527,6 +45651,7 @@
"supports_response_schema": true
},
"snowflake/claude-sonnet-4-6": {
"supports_adaptive_thinking": true,
"max_tokens": 16384,
"max_input_tokens": 200000,
"max_output_tokens": 16384,
@ -45945,40 +46070,6 @@
"supports_tool_choice": true,
"supports_vision": false
},
"darkbloom/gemma-4-26b": {
"input_cost_per_token": 3e-08,
"litellm_provider": "darkbloom",
"max_input_tokens": 131072,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 1.65e-07,
"source": "https://www.darkbloom.dev/",
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"darkbloom/gpt-oss-20b": {
"input_cost_per_token": 1.45e-08,
"litellm_provider": "darkbloom",
"max_input_tokens": 131072,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 7e-08,
"source": "https://www.darkbloom.dev/",
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"deepseek/deepseek-v4-pro": {
"cache_creation_input_token_cost": 0.0,
"cache_read_input_token_cost": 3.625e-09,
@ -46134,6 +46225,40 @@
"supports_reasoning": false,
"source": "https://pinstripes.io/pricing"
},
"darkbloom/gemma-4-26b": {
"input_cost_per_token": 3e-08,
"litellm_provider": "darkbloom",
"max_input_tokens": 131072,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 1.65e-07,
"source": "https://www.darkbloom.dev/",
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"darkbloom/gpt-oss-20b": {
"input_cost_per_token": 1.45e-08,
"litellm_provider": "darkbloom",
"max_input_tokens": 131072,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 7e-08,
"source": "https://www.darkbloom.dev/",
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"fallback_generalizations": {
"rules": [
{

9
model_prices_and_context_window.json
View file

@ -20957,7 +20957,8 @@
"max_tokens": 16000,
"mode": "chat",
"supported_endpoints": [
"/v1/chat/completions"
"/v1/chat/completions",
"/v1/messages"
],
"supports_function_calling": true,
"supports_parallel_function_calling": true,
@ -20970,7 +20971,8 @@
"max_tokens": 16000,
"mode": "chat",
"supported_endpoints": [
"/v1/chat/completions"
"/v1/chat/completions",
"/v1/messages"
],
"supports_function_calling": true,
"supports_parallel_function_calling": true,
@ -21022,7 +21024,8 @@
"max_tokens": 16000,
"mode": "chat",
"supported_endpoints": [
"/v1/chat/completions"
"/v1/chat/completions",
"/v1/messages"
],
"supports_function_calling": true,
"supports_parallel_function_calling": true,

84
tests/test_litellm/test_model_cost_map_sync.py Normal file
View file

@ -0,0 +1,84 @@
"""The two shipped model cost maps must stay in lockstep.
``model_prices_and_context_window.json`` (repo root) is the canonical map that gets
published and fetched at runtime. ``litellm/model_prices_and_context_window_backup.json``
is bundled into the package and is what LiteLLM falls back to when the remote fetch
fails or when ``LITELLM_LOCAL_MODEL_COST_MAP=True``. If they drift, the same model
silently resolves to different metadata depending on whether the remote fetch
succeeded -- wrong context windows, missing models, missing endpoints.
``ci_cd/check_files_match.py`` was written to enforce exactly this, but it is wired
into nothing (it used to be a pre-commit hook, and the pre-commit config was removed),
so drift accumulated unnoticed. Existing per-model checks such as
``test_gpt_5_6_backup_matches_main`` only cover the handful of models each was written
for, so they cannot catch drift in any other entry.
These tests assert the invariant globally.
"""
import filecmp
import json
from pathlib import Path
REPO_ROOT = Path(__file__).parents[2]
MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json"
BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
_RESYNC_HINT = "Run `python ci_cd/check_files_match.py` after updating the canonical map."
def _load(path: Path) -> dict:
with open(path, encoding="utf-8") as fh:
return json.load(fh)
def _describe_drift(main_map: dict, backup_map: dict, limit: int = 20) -> str:
only_main = sorted(set(main_map) - set(backup_map))
only_backup = sorted(set(backup_map) - set(main_map))
field_diffs = []
for key in sorted(set(main_map) & set(backup_map)):
main_entry, backup_entry = main_map[key], backup_map[key]
if not isinstance(main_entry, dict) or not isinstance(backup_entry, dict):
if main_entry != backup_entry:
field_diffs.append(f"{key}: {main_entry!r} != {backup_entry!r}")
continue
for field in sorted(set(main_entry) | set(backup_entry)):
main_value = main_entry.get(field, "<absent>")
backup_value = backup_entry.get(field, "<absent>")
if main_value != backup_value:
field_diffs.append(f"{key}.{field}: main={main_value!r} backup={backup_value!r}")
lines = []
if only_main:
lines.append(f"{len(only_main)} entries missing from backup: {only_main[:limit]}")
if only_backup:
lines.append(f"{len(only_backup)} entries missing from main: {only_backup[:limit]}")
if field_diffs:
lines.append(f"{len(field_diffs)} field differences:")
lines.extend(f" {diff}" for diff in field_diffs[:limit])
if len(field_diffs) > limit:
lines.append(f" ... and {len(field_diffs) - limit} more")
return "\n".join(lines)
def test_backup_cost_map_has_same_contents_as_canonical():
"""Every entry and field must resolve identically from either map."""
main_map = _load(MAIN_PATH)
backup_map = _load(BACKUP_PATH)
if main_map != backup_map:
raise AssertionError(
"model cost maps have drifted.\n" + _describe_drift(main_map, backup_map) + "\n" + _RESYNC_HINT
)
def test_backup_cost_map_is_byte_identical_to_canonical():
"""The two files are kept byte-identical, which is the contract ci_cd/check_files_match.py enforces.
Separate from the parsed-content check so a pure formatting divergence reports as
a formatting problem rather than as missing or wrong model metadata.
"""
assert filecmp.cmp(MAIN_PATH, BACKUP_PATH, shallow=False), (
f"{MAIN_PATH.name} and {BACKUP_PATH.name} differ byte-for-byte. " + _RESYNC_HINT
)