scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-22 00:31:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 96

fix(mistral): send reasoning_effort as a level the model accepts

Browse source 
Declare the live-verified reasoning_effort_levels on the Mistral cost-map
entries and round an undeclared request to the nearest declared level
(up to the weakest level at least as strong, down to the strongest when
the request exceeds the ceiling). Codex's default medium no longer 400s
on mistral-medium-latest, mistral-small-latest, or the vibe-cli family;
an entry that declares nothing keeps forwarding the value verbatim
This commit is contained in:
mateo-berri 2026-09-18 11:58:04 -07:00
parent ba93c4943c
commit c1e39810ec
6 changed files with 246 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
litellm/llms/mistral/chat/transformation.py
View file

@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, cast, get_type_hints, ove
import httpx
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.prompt_templates.common_utils import (
handle_messages_with_content_list_to_str_conversion,
@ -20,6 +21,10 @@ from litellm.llms.openai.chat.gpt_transformation import (
OpenAIChatCompletionStreamingHandler,
OpenAIGPTConfig,
)
from litellm.router_utils.reasoning_effort_capability import (
declared_reasoning_efforts_for_model,
nearest_declared_reasoning_effort,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.mistral import MistralThinkingBlock, MistralToolCallMessage
from litellm.types.llms.openai import AllMessageValues
@ -30,6 +35,18 @@ if TYPE_CHECKING:
import tiktoken
def _accepted_reasoning_effort(model: str, requested: str) -> str:
declared: Final = declared_reasoning_efforts_for_model(model, "mistral")
if declared is None:
return requested
accepted: Final = nearest_declared_reasoning_effort(requested, declared)
if accepted != requested:
verbose_logger.debug(
"mistral: %s takes reasoning_effort %s, sending %s in place of %s", model, declared, accepted, requested
)
return accepted
class MistralConfig(OpenAIGPTConfig):
"""
Reference: https://docs.mistral.ai/api/
@ -170,7 +187,7 @@ class MistralConfig(OpenAIGPTConfig):
if param == "response_format":
optional_params["response_format"] = value
if param == "reasoning_effort" and "magistral" not in model.lower():
optional_params["reasoning_effort"] = value
optional_params["reasoning_effort"] = _accepted_reasoning_effort(model, value)
if param in ("reasoning_effort", "thinking") and "magistral" in model.lower():
# Flag that we need to add reasoning system prompt
optional_params["_add_reasoning_prompt"] = True

77
litellm/model_prices_and_context_window_backup.json
View file

@ -36993,6 +36993,10 @@
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 7.5e-06,
"reasoning_effort_levels": [
"none",
"high"
],
"source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04",
"supports_assistant_prefill": true,
"supports_function_calling": true,
@ -37072,6 +37076,15 @@
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 4.4e-06,
"reasoning_effort_levels": [
"none",
"minimal",
"low",
"medium",
"high",
"xhigh",
"max"
],
"source": "https://docs.mistral.ai/models/zai-glm-5-2",
"supports_assistant_prefill": true,
"supports_function_calling": true,
@ -37089,6 +37102,11 @@
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 4.4e-06,
"reasoning_effort_levels": [
"low",
"high",
"max"
],
"source": "https://docs.mistral.ai/models/zai-glm-5-3",
"supports_assistant_prefill": true,
"supports_function_calling": true,
@ -37106,6 +37124,11 @@
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 4.4e-06,
"reasoning_effort_levels": [
"low",
"high",
"max"
],
"source": "https://docs.mistral.ai/models/zai-glm-5-3",
"supports_assistant_prefill": true,
"supports_function_calling": true,
@ -37123,6 +37146,11 @@
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 4.4e-06,
"reasoning_effort_levels": [
"low",
"high",
"max"
],
"source": "https://docs.mistral.ai/models/zai-glm-5-3",
"supports_assistant_prefill": true,
"supports_function_calling": true,
@ -37140,6 +37168,15 @@
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 4.4e-06,
"reasoning_effort_levels": [
"none",
"minimal",
"low",
"medium",
"high",
"xhigh",
"max"
],
"source": "https://docs.mistral.ai/models/zai-glm-5-2",
"supports_assistant_prefill": true,
"supports_function_calling": true,
@ -37435,6 +37472,10 @@
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 7.5e-06,
"reasoning_effort_levels": [
"none",
"high"
],
"source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04",
"supports_assistant_prefill": true,
"supports_function_calling": true,
@ -37495,6 +37536,10 @@
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 7.5e-06,
"reasoning_effort_levels": [
"none",
"high"
],
"source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04",
"supports_assistant_prefill": true,
"supports_function_calling": true,
@ -37512,6 +37557,10 @@
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 7.5e-06,
"reasoning_effort_levels": [
"none",
"high"
],
"source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04",
"supports_assistant_prefill": true,
"supports_function_calling": true,
@ -37545,6 +37594,10 @@
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 7.5e-06,
"reasoning_effort_levels": [
"none",
"high"
],
"source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04",
"supports_assistant_prefill": true,
"supports_function_calling": true,
@ -37575,6 +37628,10 @@
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 6e-07,
"reasoning_effort_levels": [
"none",
"high"
],
"source": "https://docs.mistral.ai/models/model-cards/mistral-small-4-0-26-03",
"supports_assistant_prefill": true,
"supports_function_calling": true,
@ -59375,6 +59432,10 @@
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 6e-07,
"reasoning_effort_levels": [
"none",
"high"
],
"source": "https://docs.mistral.ai/models/model-cards/mistral-small-4-0-26-03",
"supports_assistant_prefill": true,
"supports_function_calling": true,
@ -62700,6 +62761,10 @@
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 7.5e-06,
"reasoning_effort_levels": [
"none",
"high"
],
"source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04",
"supports_assistant_prefill": true,
"supports_function_calling": true,
@ -62717,6 +62782,10 @@
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 7.5e-06,
"reasoning_effort_levels": [
"none",
"high"
],
"source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04",
"supports_assistant_prefill": true,
"supports_function_calling": true,
@ -62734,6 +62803,10 @@
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 7.5e-06,
"reasoning_effort_levels": [
"none",
"high"
],
"source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04",
"supports_assistant_prefill": true,
"supports_function_calling": true,
@ -62751,6 +62824,10 @@
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 6e-07,
"reasoning_effort_levels": [
"none",
"high"
],
"source": "https://docs.mistral.ai/models/model-cards/mistral-small-4-0-26-03",
"supports_assistant_prefill": true,
"supports_function_calling": true,

17
litellm/router_utils/reasoning_effort_capability.py
View file

@ -103,6 +103,23 @@ def declared_reasoning_efforts_for_model(model: str, custom_llm_provider: str) -
return declared_reasoning_efforts(entry)
REASONING_EFFORT_STRENGTH_ORDER: Final = ("none", "minimal", "low", "medium", "high", "xhigh", "max")
_STRENGTH_RANK: Final = MappingProxyType({effort: rank for rank, effort in enumerate(REASONING_EFFORT_STRENGTH_ORDER)})
def nearest_declared_reasoning_effort(requested: str, declared: Sequence[str]) -> str:
"""Rounds a request up to the weakest declared level at least as strong as it, and down to the
strongest declared level when it asks for more than the model has, so the caller gets no less
reasoning than it asked for instead of a rejected call. A level outside the strength order is
returned as is for upstream to judge."""
ranked: Final = sorted(
(effort for effort in declared if effort in _STRENGTH_RANK), key=lambda effort: _STRENGTH_RANK[effort]
)
if requested in ranked or requested not in _STRENGTH_RANK or not ranked:
return requested
return next((effort for effort in ranked if _STRENGTH_RANK[effort] >= _STRENGTH_RANK[requested]), ranked[-1])
def _supports_none_reasoning_effort(model_info: Mapping[str, object], flag: object) -> bool:
"""Opt-in only where a request path refuses the level. AzureOpenAIGPT5Config raises
UnsupportedParamsError on reasoning_effort='none' without an explicit true, and it is selected

77
model_prices_and_context_window.json
View file

@ -36993,6 +36993,10 @@
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 7.5e-06,
"reasoning_effort_levels": [
"none",
"high"
],
"source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04",
"supports_assistant_prefill": true,
"supports_function_calling": true,
@ -37072,6 +37076,15 @@
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 4.4e-06,
"reasoning_effort_levels": [
"none",
"minimal",
"low",
"medium",
"high",
"xhigh",
"max"
],
"source": "https://docs.mistral.ai/models/zai-glm-5-2",
"supports_assistant_prefill": true,
"supports_function_calling": true,
@ -37089,6 +37102,11 @@
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 4.4e-06,
"reasoning_effort_levels": [
"low",
"high",
"max"
],
"source": "https://docs.mistral.ai/models/zai-glm-5-3",
"supports_assistant_prefill": true,
"supports_function_calling": true,
@ -37106,6 +37124,11 @@
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 4.4e-06,
"reasoning_effort_levels": [
"low",
"high",
"max"
],
"source": "https://docs.mistral.ai/models/zai-glm-5-3",
"supports_assistant_prefill": true,
"supports_function_calling": true,
@ -37123,6 +37146,11 @@
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 4.4e-06,
"reasoning_effort_levels": [
"low",
"high",
"max"
],
"source": "https://docs.mistral.ai/models/zai-glm-5-3",
"supports_assistant_prefill": true,
"supports_function_calling": true,
@ -37140,6 +37168,15 @@
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 4.4e-06,
"reasoning_effort_levels": [
"none",
"minimal",
"low",
"medium",
"high",
"xhigh",
"max"
],
"source": "https://docs.mistral.ai/models/zai-glm-5-2",
"supports_assistant_prefill": true,
"supports_function_calling": true,
@ -37435,6 +37472,10 @@
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 7.5e-06,
"reasoning_effort_levels": [
"none",
"high"
],
"source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04",
"supports_assistant_prefill": true,
"supports_function_calling": true,
@ -37495,6 +37536,10 @@
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 7.5e-06,
"reasoning_effort_levels": [
"none",
"high"
],
"source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04",
"supports_assistant_prefill": true,
"supports_function_calling": true,
@ -37512,6 +37557,10 @@
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 7.5e-06,
"reasoning_effort_levels": [
"none",
"high"
],
"source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04",
"supports_assistant_prefill": true,
"supports_function_calling": true,
@ -37545,6 +37594,10 @@
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 7.5e-06,
"reasoning_effort_levels": [
"none",
"high"
],
"source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04",
"supports_assistant_prefill": true,
"supports_function_calling": true,
@ -37575,6 +37628,10 @@
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 6e-07,
"reasoning_effort_levels": [
"none",
"high"
],
"source": "https://docs.mistral.ai/models/model-cards/mistral-small-4-0-26-03",
"supports_assistant_prefill": true,
"supports_function_calling": true,
@ -59375,6 +59432,10 @@
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 6e-07,
"reasoning_effort_levels": [
"none",
"high"
],
"source": "https://docs.mistral.ai/models/model-cards/mistral-small-4-0-26-03",
"supports_assistant_prefill": true,
"supports_function_calling": true,
@ -62700,6 +62761,10 @@
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 7.5e-06,
"reasoning_effort_levels": [
"none",
"high"
],
"source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04",
"supports_assistant_prefill": true,
"supports_function_calling": true,
@ -62717,6 +62782,10 @@
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 7.5e-06,
"reasoning_effort_levels": [
"none",
"high"
],
"source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04",
"supports_assistant_prefill": true,
"supports_function_calling": true,
@ -62734,6 +62803,10 @@
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 7.5e-06,
"reasoning_effort_levels": [
"none",
"high"
],
"source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04",
"supports_assistant_prefill": true,
"supports_function_calling": true,
@ -62751,6 +62824,10 @@
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 6e-07,
"reasoning_effort_levels": [
"none",
"high"
],
"source": "https://docs.mistral.ai/models/model-cards/mistral-small-4-0-26-03",
"supports_assistant_prefill": true,
"supports_function_calling": true,

44
tests/test_litellm/llms/mistral/test_mistral_chat_transformation.py
View file

@ -63,7 +63,7 @@ class TestMistralReasoningSupport:
assert "reasoning_effort" not in supported_params_normal
assert "thinking" not in supported_params_normal
def test_map_openai_params_reasoning_effort(self):
def test_map_openai_params_reasoning_effort(self, local_model_cost_map):
"""Test that reasoning_effort parameter is properly mapped for magistral models."""
mistral_config = MistralConfig()
@ -87,21 +87,51 @@ class TestMistralReasoningSupport:
)
assert "_add_reasoning_prompt" not in result_normal
assert result_normal["reasoning_effort"] == "low"
assert result_normal["reasoning_effort"] == "high"
@pytest.mark.parametrize(
("model", "reasoning_effort"),
[("mistral-medium-latest", "high"), ("zai-glm-5-2", "xhigh")],
("model", "requested", "sent"),
[
("mistral-medium-latest", "high", "high"),
("mistral-medium-latest", "none", "none"),
("mistral-medium-latest", "low", "high"),
("mistral-medium-latest", "medium", "high"),
("mistral-medium-latest", "xhigh", "high"),
("mistral-small-latest", "medium", "high"),
("mistral-vibe-cli-latest", "medium", "high"),
("zai-glm-5", "none", "low"),
("zai-glm-5", "medium", "high"),
("zai-glm-5", "xhigh", "max"),
("zai-glm-5-2", "medium", "medium"),
("zai-glm-5-2", "xhigh", "xhigh"),
],
)
def test_reasoning_effort_forwarded_verbatim_for_reasoning_models(self, model, reasoning_effort):
def test_reasoning_effort_is_sent_as_a_level_the_model_accepts(self, local_model_cost_map, model, requested, sent):
import litellm
optional_params = litellm.get_optional_params(
model=model,
custom_llm_provider="mistral",
reasoning_effort=reasoning_effort,
reasoning_effort=requested,
)
assert optional_params["reasoning_effort"] == reasoning_effort
assert optional_params["reasoning_effort"] == sent
def test_reasoning_effort_is_forwarded_verbatim_when_the_map_declares_no_levels(
self, local_model_cost_map, monkeypatch
):
import litellm
monkeypatch.setitem(
litellm.model_cost,
"mistral/undeclared-reasoner",
{"litellm_provider": "mistral", "mode": "chat", "supports_reasoning": True},
)
optional_params = litellm.get_optional_params(
model="undeclared-reasoner",
custom_llm_provider="mistral",
reasoning_effort="medium",
)
assert optional_params["reasoning_effort"] == "medium"
def test_reasoning_effort_stays_unsupported_for_non_reasoning_models(self):
import litellm

20
tests/test_litellm/router_utils/test_reasoning_effort_capability.py
View file

@ -4,6 +4,7 @@ import litellm
from litellm.router_utils.reasoning_effort_capability import (
deployment_is_catalog_mapped,
intersect_supported_reasoning_efforts,
nearest_declared_reasoning_effort,
resolve_supported_reasoning_efforts,
)
@ -415,3 +416,22 @@ class TestGpt6AstraAdvertisesItsDocumentedLevels:
"high",
"xhigh",
)
class TestNearestDeclaredReasoningEffort:
def test_a_declared_level_is_kept(self):
assert nearest_declared_reasoning_effort("high", ("none", "high")) == "high"
assert nearest_declared_reasoning_effort("none", ("none", "high")) == "none"
def test_an_undeclared_level_rounds_up_to_the_next_declared_one(self):
assert nearest_declared_reasoning_effort("medium", ("none", "high")) == "high"
assert nearest_declared_reasoning_effort("none", ("low", "high", "max")) == "low"
assert nearest_declared_reasoning_effort("xhigh", ("low", "high", "max")) == "max"
def test_a_level_above_the_ceiling_takes_the_strongest_declared_one(self):
assert nearest_declared_reasoning_effort("max", ("none", "high")) == "high"
assert nearest_declared_reasoning_effort("xhigh", ("none", "low", "medium", "high")) == "high"
def test_a_level_outside_the_strength_order_is_left_for_upstream(self):
assert nearest_declared_reasoning_effort("turbo", ("none", "high")) == "turbo"
assert nearest_declared_reasoning_effort("medium", ()) == "medium"