scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-08 03:08:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 115

fix(cost-map): retarget mistral-medium-latest to Medium 3.5 and add date-pinned aliases (#31373)

Browse source 
* fix(cost-map): retarget mistral-medium-latest to Medium 3.5 and add date-pinned aliases

Mistral repointed the rolling mistral-medium-latest alias from Medium 3.1
to Medium 3.5, but the static cost map still carried Medium 3.1 specs,
showing wrong pricing/context in the model hub and undercharging spend by
about 3.75x (LIT-3883).

Update mistral/mistral-medium-latest to Medium 3.5 ($1.50/$7.50 per 1M,
256K context, reasoning + vision), add the bare date-pinned aliases
mistral/mistral-medium-2604 (Medium 3.5) and mistral/mistral-medium-2508
(Medium 3.1) that match Mistral's real API model ids, and add
supports_reasoning to mistral/mistral-medium-3-5.

Apply every change to both model_prices_and_context_window.json and the
bundled litellm/model_prices_and_context_window_backup.json so the two
stay in sync, and extend the regression tests to lock the resolved
get_model_info values and the main/backup parity for all touched models.

* test(cost-map): force local cost map in mistral-medium-latest resolution test

get_model_info reads litellm.model_cost, which is fetched from the remote
main branch at import time when LITELLM_LOCAL_MODEL_COST_MAP is unset. Until
this PR lands on main, that remote map still carries the pre-merge Medium 3.1
pricing, so the assertion was only passing when the remote fetch happened to
fail and fell back to the bundled backup. Force the local cost map (the same
fixture pattern the other get_model_info tests use) so the alias resolution is
verified deterministically against the in-repo file.
This commit is contained in:
Mateo Wang 2026-06-25 18:27:18 -07:00 • committed by GitHub
parent 97008bad29
commit 6cc9ea2538
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 134 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
litellm/model_prices_and_context_window_backup.json
View file

@ -25793,7 +25793,7 @@
"supports_response_schema": true,
"supports_tool_choice": true
},
"mistral/mistral-medium-latest": {
"mistral/mistral-medium-2508": {
"input_cost_per_token": 4e-07,
"litellm_provider": "mistral",
"max_input_tokens": 131072,
@ -25801,12 +25801,45 @@
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 2e-06,
"source": "https://mistral.ai/news/mistral-medium-3",
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"mistral/mistral-medium-2604": {
"input_cost_per_token": 1.5e-06,
"litellm_provider": "mistral",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 7.5e-06,
"source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04",
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"mistral/mistral-medium-latest": {
"input_cost_per_token": 1.5e-06,
"litellm_provider": "mistral",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 7.5e-06,
"source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04",
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"mistral/mistral-medium-3-1-2508": {
"input_cost_per_token": 4e-07,
"litellm_provider": "mistral",
@ -25833,6 +25866,7 @@
"source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04",
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true

36
model_prices_and_context_window.json
View file

@ -25965,7 +25965,7 @@
"supports_response_schema": true,
"supports_tool_choice": true
},
"mistral/mistral-medium-latest": {
"mistral/mistral-medium-2508": {
"input_cost_per_token": 4e-07,
"litellm_provider": "mistral",
"max_input_tokens": 131072,
@ -25973,12 +25973,45 @@
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 2e-06,
"source": "https://mistral.ai/news/mistral-medium-3",
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"mistral/mistral-medium-2604": {
"input_cost_per_token": 1.5e-06,
"litellm_provider": "mistral",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 7.5e-06,
"source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04",
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"mistral/mistral-medium-latest": {
"input_cost_per_token": 1.5e-06,
"litellm_provider": "mistral",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 7.5e-06,
"source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04",
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"mistral/mistral-medium-3-1-2508": {
"input_cost_per_token": 4e-07,
"litellm_provider": "mistral",
@ -26005,6 +26038,7 @@
"source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04",
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true

87
tests/test_litellm/test_mistral_medium_3_5_model_metadata.py
View file

@ -3,19 +3,45 @@ from pathlib import Path
import pytest
import litellm
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
REPO_ROOT = Path(__file__).parents[2]
MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json"
BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
@pytest.mark.parametrize("model", ["mistral/mistral-medium-3-5"])
def test_mistral_medium_3_5_model_info(model):
json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json"
with open(json_path) as f:
model_cost = json.load(f)
MEDIUM_3_5_MODELS = (
"mistral/mistral-medium-3-5",
"mistral/mistral-medium-2604",
"mistral/mistral-medium-latest",
)
info = model_cost.get(model)
assert (
info is not None
), f"{model} not found in model_prices_and_context_window.json"
SYNCED_MODELS = MEDIUM_3_5_MODELS + (
"mistral/mistral-medium-2508",
"mistral/mistral-medium-3-1-2508",
)
def _load(path):
with open(path) as f:
return json.load(f)
@pytest.fixture
def local_model_cost_map(monkeypatch):
"""Force get_model_info to resolve against the in-repo cost map instead of the
remote one fetched at import time, which still carries the pre-merge pricing."""
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
litellm.get_model_info.cache_clear()
yield
litellm.get_model_info.cache_clear()
@pytest.mark.parametrize("model", MEDIUM_3_5_MODELS)
def test_medium_3_5_specs(model):
info = _load(MAIN_PATH).get(model)
assert info is not None, f"{model} missing from model_prices_and_context_window.json"
assert info["litellm_provider"] == "mistral"
assert info["mode"] == "chat"
@ -27,10 +53,11 @@ def test_mistral_medium_3_5_model_info(model):
assert info["max_output_tokens"] == 262144
assert info["max_tokens"] == 262144
assert info["supports_reasoning"] is True
assert info["supports_vision"] is True
assert info["supports_function_calling"] is True
assert info["supports_response_schema"] is True
assert info["supports_tool_choice"] is True
assert info["supports_vision"] is True
assert info["supports_assistant_prefill"] is True
routed_model, provider, _, _ = get_llm_provider(model=model)
@ -38,18 +65,32 @@ def test_mistral_medium_3_5_model_info(model):
assert provider == "mistral"
def test_mistral_medium_3_5_backup_matches_main():
"""Ensure the bundled model cost map stays in sync with the canonical file."""
repo_root = Path(__file__).parents[2]
main_path = repo_root / "model_prices_and_context_window.json"
backup_path = repo_root / "litellm" / "model_prices_and_context_window_backup.json"
def test_mistral_medium_latest_resolves_to_medium_3_5(local_model_cost_map):
"""LIT-3883: the -latest alias was retargeted to Medium 3.5; get_model_info must
return the 3.5 pricing/context/reasoning, not the stale Medium 3.1 values."""
info = litellm.get_model_info(model="mistral/mistral-medium-latest")
with open(main_path) as f:
main_cost = json.load(f)
with open(backup_path) as f:
backup_cost = json.load(f)
assert info["input_cost_per_token"] == 1.5e-06
assert info["output_cost_per_token"] == 7.5e-06
assert info["max_input_tokens"] == 262144
assert info["supports_reasoning"] is True
for model in ("mistral/mistral-medium-3-5",):
assert backup_cost.get(model) == main_cost.get(
model
), f"{model} differs between main and backup model cost maps"
def test_mistral_medium_2508_keeps_medium_3_1_specs():
"""The date-pinned 2508 alias is Medium 3.1 and must not inherit 3.5 pricing."""
info = _load(MAIN_PATH).get("mistral/mistral-medium-2508")
assert info is not None, "mistral/mistral-medium-2508 missing from cost map"
assert info["input_cost_per_token"] == 4e-07
assert info["output_cost_per_token"] == 2e-06
assert info["max_input_tokens"] == 131072
assert info.get("supports_reasoning") is not True
@pytest.mark.parametrize("model", SYNCED_MODELS)
def test_backup_matches_main(model):
"""Ensure the bundled (backup) cost map stays in sync with the canonical file."""
main_cost = _load(MAIN_PATH)
backup_cost = _load(BACKUP_PATH)
assert backup_cost.get(model) == main_cost.get(model), f"{model} differs between main and backup model cost maps"