mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
fix(mistral): correct zai-glm-5-2 limits, add cached-input price and glm-5-2 alias
Mistral's live /v1/models reports max_context_length 1048576 and capabilities.reasoning true for zai-glm-5-2, and its docs price cached input at $0.14/M. Without cache_read_input_token_cost LiteLLM billed every cached prompt token at $0, so a repeat request against a 21k-token cached prefix logged $0.0000135 instead of its real cost. Mistral also serves the model under the short glm-5-2 name, which had no cost map entry at all and therefore no pricing, so add it alongside.
This commit is contained in:
parent
ffa37d05b7
commit
4fac88790d
3 changed files with 149 additions and 6 deletions
|
|
@ -29046,16 +29046,36 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"mistral/zai-glm-5-2": {
|
||||
"cache_read_input_token_cost": 1.4e-07,
|
||||
"input_cost_per_token": 1.4e-06,
|
||||
"litellm_provider": "mistral",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 131072,
|
||||
"max_tokens": 131072,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4.4e-06,
|
||||
"source": "https://docs.mistral.ai/models/zai-glm-5-2",
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"mistral/glm-5-2": {
|
||||
"cache_read_input_token_cost": 1.4e-07,
|
||||
"input_cost_per_token": 1.4e-06,
|
||||
"litellm_provider": "mistral",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 131072,
|
||||
"max_tokens": 131072,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4.4e-06,
|
||||
"source": "https://docs.mistral.ai/models/zai-glm-5-2",
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
|
|
|
|||
|
|
@ -29046,16 +29046,36 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"mistral/zai-glm-5-2": {
|
||||
"cache_read_input_token_cost": 1.4e-07,
|
||||
"input_cost_per_token": 1.4e-06,
|
||||
"litellm_provider": "mistral",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 131072,
|
||||
"max_tokens": 131072,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4.4e-06,
|
||||
"source": "https://docs.mistral.ai/models/zai-glm-5-2",
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"mistral/glm-5-2": {
|
||||
"cache_read_input_token_cost": 1.4e-07,
|
||||
"input_cost_per_token": 1.4e-06,
|
||||
"litellm_provider": "mistral",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 131072,
|
||||
"max_tokens": 131072,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4.4e-06,
|
||||
"source": "https://docs.mistral.ai/models/zai-glm-5-2",
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
|
|
|
|||
103
tests/test_litellm/test_mistral_zai_glm_5_2_model_metadata.py
Normal file
103
tests/test_litellm/test_mistral_zai_glm_5_2_model_metadata.py
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.types.utils import PromptTokensDetailsWrapper, Usage
|
||||
from litellm.utils import supports_prompt_caching, supports_reasoning
|
||||
|
||||
REPO_ROOT = Path(__file__).parents[2]
|
||||
MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json"
|
||||
BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
|
||||
|
||||
GLM_5_2_MODELS = ("mistral/zai-glm-5-2", "mistral/glm-5-2")
|
||||
|
||||
INPUT_COST = 1.4e-06
|
||||
CACHED_INPUT_COST = 1.4e-07
|
||||
OUTPUT_COST = 4.4e-06
|
||||
|
||||
|
||||
def _load(path):
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def local_model_cost_map(monkeypatch):
|
||||
"""Force get_model_info to resolve against the in-repo cost map instead of the
|
||||
remote one fetched at import time, which still carries the pre-merge pricing."""
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
|
||||
litellm.get_model_info.cache_clear()
|
||||
yield
|
||||
litellm.get_model_info.cache_clear()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", GLM_5_2_MODELS)
|
||||
def test_zai_glm_5_2_specs(model):
|
||||
info = _load(MAIN_PATH).get(model)
|
||||
assert info is not None, f"{model} missing from model_prices_and_context_window.json"
|
||||
|
||||
assert info["litellm_provider"] == "mistral"
|
||||
assert info["mode"] == "chat"
|
||||
|
||||
assert info["input_cost_per_token"] == INPUT_COST
|
||||
assert info["output_cost_per_token"] == OUTPUT_COST
|
||||
assert info["cache_read_input_token_cost"] == CACHED_INPUT_COST
|
||||
|
||||
assert info["max_input_tokens"] == 1048576
|
||||
assert info["max_output_tokens"] == 131072
|
||||
assert info["max_tokens"] == 131072
|
||||
|
||||
assert info["supports_assistant_prefill"] is True
|
||||
assert info["supports_function_calling"] is True
|
||||
assert info["supports_prompt_caching"] is True
|
||||
assert info["supports_reasoning"] is True
|
||||
assert info["supports_response_schema"] is True
|
||||
assert info["supports_tool_choice"] is True
|
||||
|
||||
routed_model, provider, _, _ = get_llm_provider(model=model)
|
||||
assert routed_model == model.split("/", 1)[1]
|
||||
assert provider == "mistral"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", GLM_5_2_MODELS)
|
||||
def test_zai_glm_5_2_capabilities_are_visible_to_callers(local_model_cost_map, model):
|
||||
"""Mistral advertises reasoning and prompt caching on this model, so the helpers
|
||||
every caller checks before sending a request must say so too."""
|
||||
assert supports_reasoning(model=model) is True
|
||||
assert supports_prompt_caching(model=model) is True
|
||||
|
||||
info = litellm.get_model_info(model=model)
|
||||
assert info["max_input_tokens"] == 1048576
|
||||
assert info["max_output_tokens"] == 131072
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", GLM_5_2_MODELS)
|
||||
def test_cached_prompt_tokens_bill_at_the_cached_rate(local_model_cost_map, model):
|
||||
"""A cache hit reports its reused tokens under prompt_tokens_details, and those
|
||||
tokens cost a tenth of the input rate, not the full rate and not nothing."""
|
||||
usage = Usage(
|
||||
prompt_tokens=21010,
|
||||
completion_tokens=100,
|
||||
total_tokens=21110,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=20992),
|
||||
)
|
||||
|
||||
prompt_cost, completion_cost = litellm.cost_per_token(
|
||||
model=model, usage_object=usage, custom_llm_provider="mistral"
|
||||
)
|
||||
|
||||
assert prompt_cost == pytest.approx(18 * INPUT_COST + 20992 * CACHED_INPUT_COST)
|
||||
assert completion_cost == pytest.approx(100 * OUTPUT_COST)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", GLM_5_2_MODELS)
|
||||
def test_backup_matches_main(model):
|
||||
"""Ensure the bundled (backup) cost map stays in sync with the canonical file."""
|
||||
main_cost = _load(MAIN_PATH)
|
||||
backup_cost = _load(BACKUP_PATH)
|
||||
|
||||
assert backup_cost.get(model) == main_cost.get(model), f"{model} differs between main and backup model cost maps"
|
||||
Loading…
Add table
Reference in a new issue