scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-05 07:37:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 233

fix(mistral): a named tool_choice is dropped, the model picks any tool (#39737)

Browse source 
* fix(mistral): a named tool_choice is dropped, the model picks any tool

Mistral's chat API takes tool_choice as an enum or as an object naming
one function, {"type": "function", "function": {"name": ...}}. The
mapper only handled strings, so the object was skipped and the request
reached Mistral with no tool_choice at all.

The pin is now rebuilt from the function name alone, since Mistral
rejects fields it does not define. Enum mapping is unchanged, and an
object without a usable name is still left out.

Fixes #39736

* fix(mistral): reject a tool_choice Mistral cannot take unless drop_params is set

* fix(mistral): warn when drop_params drops an unusable tool_choice

Return the unsupported tool_choice error as a value and raise it in
map_openai_params, so the helper models the failure and the caller owns
the exception. A tool_choice dropped under drop_params now logs a
warning naming the value and why Mistral cannot take it.

* fix(mistral): leave tool_choice to the no-tools removal when no tools are sent

An unusable tool_choice on a request with no tools answered 400, while the
merge base and every other OpenAI-compatible route strip tool_choice from
such a request and answer 200. Map tool_choice only when tools are present.

---------

Co-authored-by: Pasqual Troncone <pasqual.troncone@stratebi.com>
Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
Pasqual Troncone 2026-11-05 07:38:14 +01:00 • committed by GitHub
parent 3577aa2d94
commit 728c687418
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 189 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

84
litellm/llms/mistral/chat/transformation.py
View file

@ -6,6 +6,7 @@ Why separate file? Make it easy to see how transformation works
Docs - https://docs.mistral.ai/api/
"""
import reprlib
from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence
from itertools import accumulate, count
from typing import TYPE_CHECKING, Final, Literal, cast, get_type_hints, overload
@ -45,7 +46,13 @@ from litellm.types.llms.mistral import (
MistralThinkingBlock,
MistralToolCallMessage,
)
from litellm.types.llms.openai import AllMessageValues, ChatCompletionAssistantMessage, ChatCompletionTextObject
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionAssistantMessage,
ChatCompletionTextObject,
ChatCompletionToolChoiceFunctionParam,
ChatCompletionToolChoiceObjectParam,
)
from litellm.types.utils import ModelResponse, ModelResponseStream
from litellm.utils import convert_to_model_response_object, supports_reasoning
@ -79,6 +86,10 @@ _REASONING_PROMPT_REASON: Final = "litellm sends Magistral a reasoning system pr
_NON_LEVEL_EFFORT_REASON: Final = (
"the Mistral API takes it only as a level such as 'high', so Mistral applies its default effort."
)
_UNUSABLE_TOOL_CHOICE_REASON: Final = (
"the Mistral API takes tool_choice only as 'auto', 'none', 'any', 'required', "
"or {'type': 'function', 'function': {'name': <tool name>}}."
)
_DOCUMENTED_MAGISTRAL_EFFORTS: Final = frozenset({"low", "medium", "high"})
_REASONING_ON_THINKING_TYPES: Final = frozenset({"enabled", "adaptive"})
@ -118,7 +129,7 @@ class MistralConfig(OpenAIGPTConfig):
- `tools` (list or null): A list of available tools for the model. Use this to specify functions for which the model can generate JSON inputs.
- `tool_choice` (string - 'auto'/'any'/'none' or null): Specifies if/how functions are called. If set to none the model won't call a function and will generate a message instead. If set to auto the model can choose to either generate a message or call a function. If set to any the model is forced to call a function. Default - 'auto'.
- `tool_choice` (string - 'auto'/'any'/'none', an object naming one function such as {"type": "function", "function": {"name": "my_function"}}, or null): Specifies if/how functions are called. If set to none the model won't call a function and will generate a message instead. If set to auto the model can choose to either generate a message or call a function. If set to any the model is forced to call a function. Default - 'auto'.
- `stop` (string or array of strings): Stop generation if this token is detected. Or if one of these tokens is detected when providing an array
@ -135,7 +146,7 @@ class MistralConfig(OpenAIGPTConfig):
top_p: int | None = None
max_tokens: int | None = None
tools: list | None = None
tool_choice: Literal["auto", "any", "none"] | None = None
tool_choice: Literal["auto", "any", "none"] | ChatCompletionToolChoiceObjectParam | None = None
random_seed: int | None = None
safe_prompt: bool | None = None
response_format: dict | None = None
@ -147,7 +158,7 @@ class MistralConfig(OpenAIGPTConfig):
top_p: int | None = None,
max_tokens: int | None = None,
tools: list | None = None,
tool_choice: Literal["auto", "any", "none"] | None = None,
tool_choice: Literal["auto", "any", "none"] | ChatCompletionToolChoiceObjectParam | None = None,
random_seed: int | None = None,
safe_prompt: bool | None = None,
response_format: dict | None = None,
@ -192,13 +203,40 @@ class MistralConfig(OpenAIGPTConfig):
*(("prompt_cache_key",) if accepts_prompt_cache_key else ()),
]
def _map_tool_choice(self, tool_choice: str) -> str:
if tool_choice == "auto" or tool_choice == "none":
return tool_choice
elif tool_choice == "required":
return "any"
else: # openai 'tool_choice' object param not supported by Mistral API
return "any"
@staticmethod
def _map_tool_choice(tool_choice: object) -> str | ChatCompletionToolChoiceObjectParam | None:
match tool_choice:
case str(choice) if choice in ("auto", "none"):
return choice
case "required" | "any":
return "any"
case {"type": "function", "function": {"name": str(name)}} if name:
return ChatCompletionToolChoiceObjectParam(
type="function", function=ChatCompletionToolChoiceFunctionParam(name=name)
)
case _:
return None
def _tool_choice_to_send(
self, model: str, tool_choice: object, drop_params: bool
) -> str | ChatCompletionToolChoiceObjectParam | UnsupportedParamsError | None:
mapped: Final = self._map_tool_choice(tool_choice)
if mapped is not None:
return mapped
if drop_params or litellm.drop_params is True:
warn_dropped_reasoning(
"tool_choice", tool_choice, model, _UNUSABLE_TOOL_CHOICE_REASON, drop_params_gated=True
)
return None
return UnsupportedParamsError(
message=(
f"Unsupported tool_choice={reprlib.repr(tool_choice)} for model={model}, since "
f"{_UNUSABLE_TOOL_CHOICE_REASON} "
"To drop it, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n"
),
llm_provider=self.custom_llm_provider,
model=model,
)
@staticmethod
def _get_mistral_reasoning_system_prompt() -> str:
@ -244,18 +282,20 @@ class MistralConfig(OpenAIGPTConfig):
optional_params["tools"] = self._clean_tool_schema_for_mistral(value)
if param == "stream" and value is True:
optional_params["stream"] = value
if param == "temperature":
optional_params["temperature"] = value
if param == "top_p":
optional_params["top_p"] = value
if param == "stop":
optional_params["stop"] = value
if param == "tool_choice" and isinstance(value, str):
optional_params["tool_choice"] = self._map_tool_choice(tool_choice=value)
if param == "tool_choice" and "tools" in non_default_params and non_default_params["tools"]:
match self._tool_choice_to_send(
model,
cast(object, value), # cast-ok: untyped request params, the match narrows it
drop_params,
):
case UnsupportedParamsError() as unusable_tool_choice:
raise unusable_tool_choice
case None:
pass
case tool_choice:
optional_params["tool_choice"] = tool_choice
if param == "seed":
optional_params["extra_body"] = {"random_seed": value}
if param == "response_format":
optional_params["response_format"] = value
if (
param == "reasoning_effort"
and "magistral" not in model.lower()
@ -271,7 +311,7 @@ class MistralConfig(OpenAIGPTConfig):
# Flag that we need to add reasoning system prompt
optional_params["_add_reasoning_prompt"] = True
_warn_unless_reasoning_prompt_honors(param, OPAQUE_VALUE.validate_python(value), model)
if param in ("parallel_tool_calls", "prompt_cache_key"):
if param in ("temperature", "top_p", "stop", "response_format", "parallel_tool_calls", "prompt_cache_key"):
optional_params[param] = value
return optional_params

130
tests/unit/llms/mistral/test_mistral_chat_transformation.py
View file

@ -1,7 +1,7 @@
import copy
import json
import logging
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from typing import Final, List, cast
from unittest.mock import MagicMock, patch
from urllib.parse import urlparse
@ -18,7 +18,7 @@ from litellm.llms.mistral.chat.transformation import (
MistralConfig,
)
from litellm.llms.vertex_ai.vertex_ai_partner_models.mistral.transformation import VertexAIMistralConfig
from litellm.types.llms.openai import AllMessageValues
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
from litellm.types.utils import ModelResponse
from litellm.utils import supports_reasoning
@ -1690,7 +1690,9 @@ def test_mistral_request_drops_a_user_name_without_touching_the_callers_message(
def _warnings_naming(caplog: pytest.LogCaptureFixture, param: str) -> tuple[str, ...]:
return tuple(
r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage()
r.getMessage()
for r in caplog.records
if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage()
)
@ -2060,3 +2062,125 @@ def test_prompt_cache_key_stays_unsupported_for_vertex_hosted_mistral(local_mode
model="mistral-medium-3", custom_llm_provider="vertex_ai", prompt_cache_key="s", drop_params=True
)
assert "prompt_cache_key" not in dropped
class TestMistralToolChoice:
"""Mistral takes tool_choice as an enum or as {"type": "function", "function": {"name": ...}}, per
https://docs.mistral.ai/api/endpoint/chat (checked 2026-11-03)"""
TOOLS: Final[Sequence[ChatCompletionToolParam]] = [
{
"type": "function",
"function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}},
}
]
UNUSABLE_TOOL_CHOICES: Final[Sequence[str | Mapping[str, object]]] = (
{"type": "function", "function": {}},
{"type": "function", "function": {"name": ""}},
{"type": "function", "function": {"name": 123}},
{"type": "function"},
"foo",
)
def _request_body(
self,
tool_choice: str | Mapping[str, object],
drop_params: bool = False,
tools: Sequence[ChatCompletionToolParam] | None = TOOLS,
) -> Mapping[str, object]:
import litellm
optional_params: Final = litellm.get_optional_params(
model="mistral-large-latest",
custom_llm_provider="mistral",
tools=None if tools is None else list(tools),
tool_choice=tool_choice,
drop_params=drop_params,
)
return MistralConfig().transform_request(
model="mistral-large-latest",
messages=[{"role": "user", "content": "what is the weather in Madrid"}],
optional_params=optional_params,
litellm_params={},
headers={},
)
def test_named_function_is_forwarded_to_mistral(self, local_model_cost_map: None) -> None:
body: Final = self._request_body({"type": "function", "function": {"name": "get_weather"}})
assert body["tool_choice"] == {"type": "function", "function": {"name": "get_weather"}}
@pytest.mark.parametrize(
"custom_llm_provider, model",
[("mistral", "mistral-large-latest"), ("codestral", "codestral-latest"), ("vertex_ai", "mistral-small-2503")],
)
def test_named_function_is_kept_on_every_mistral_route(
self, local_model_cost_map: None, custom_llm_provider: str, model: str
) -> None:
import litellm
optional_params: Final = litellm.get_optional_params(
model=model,
custom_llm_provider=custom_llm_provider,
tools=list(self.TOOLS),
tool_choice={"type": "function", "function": {"name": "get_weather"}},
)
assert optional_params["tool_choice"] == {"type": "function", "function": {"name": "get_weather"}}
def test_keys_mistral_does_not_define_are_not_replayed(self, local_model_cost_map: None) -> None:
body: Final = self._request_body(
{
"type": "function",
"function": {"name": "get_weather", "strict": True},
"cache_control": {"type": "ephemeral"},
}
)
assert body["tool_choice"] == {"type": "function", "function": {"name": "get_weather"}}
@pytest.mark.parametrize(
"tool_choice, expected",
[("auto", "auto"), ("none", "none"), ("required", "any"), ("any", "any")],
)
def test_enum_tool_choice_is_mapped(self, local_model_cost_map: None, tool_choice: str, expected: str) -> None:
assert self._request_body(tool_choice)["tool_choice"] == expected
@pytest.mark.parametrize("tool_choice", UNUSABLE_TOOL_CHOICES)
def test_unusable_tool_choice_is_rejected_without_drop_params(
self, local_model_cost_map: None, tool_choice: str | Mapping[str, object]
) -> None:
import litellm
with pytest.raises(litellm.UnsupportedParamsError, match="tool_choice"):
self._request_body(tool_choice)
@pytest.mark.parametrize("tools", [None, ()])
@pytest.mark.parametrize("tool_choice", UNUSABLE_TOOL_CHOICES)
def test_unusable_tool_choice_without_tools_is_not_sent(
self,
local_model_cost_map: None,
tool_choice: str | Mapping[str, object],
tools: Sequence[ChatCompletionToolParam] | None,
) -> None:
assert "tool_choice" not in self._request_body(tool_choice, tools=tools)
@pytest.mark.parametrize("tool_choice", UNUSABLE_TOOL_CHOICES)
def test_unusable_tool_choice_is_dropped_with_a_warning_under_drop_params(
self, local_model_cost_map: None, tool_choice: str | Mapping[str, object], caplog: pytest.LogCaptureFixture
) -> None:
with caplog.at_level(logging.WARNING, logger=verbose_logger.name):
body: Final = self._request_body(tool_choice, drop_params=True)
assert "tool_choice" not in body
assert len(_warnings_naming(caplog, "tool_choice")) == 1
def test_unusable_tool_choice_is_dropped_under_proxy_wide_drop_params(
self, local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch
) -> None:
import litellm
monkeypatch.setattr(litellm, "drop_params", True)
assert "tool_choice" not in self._request_body({"type": "function", "function": {}})