From 728c6874182d988897db08526a64fbc4e0fc46ae Mon Sep 17 00:00:00 2001 From: Pasqual Troncone Date: Thu, 5 Nov 2026 07:38:14 +0100 Subject: [PATCH] fix(mistral): a named tool_choice is dropped, the model picks any tool (#39737) * fix(mistral): a named tool_choice is dropped, the model picks any tool Mistral's chat API takes tool_choice as an enum or as an object naming one function, {"type": "function", "function": {"name": ...}}. The mapper only handled strings, so the object was skipped and the request reached Mistral with no tool_choice at all. The pin is now rebuilt from the function name alone, since Mistral rejects fields it does not define. Enum mapping is unchanged, and an object without a usable name is still left out. Fixes #39736 * fix(mistral): reject a tool_choice Mistral cannot take unless drop_params is set * fix(mistral): warn when drop_params drops an unusable tool_choice Return the unsupported tool_choice error as a value and raise it in map_openai_params, so the helper models the failure and the caller owns the exception. A tool_choice dropped under drop_params now logs a warning naming the value and why Mistral cannot take it. * fix(mistral): leave tool_choice to the no-tools removal when no tools are sent An unusable tool_choice on a request with no tools answered 400, while the merge base and every other OpenAI-compatible route strip tool_choice from such a request and answer 200. Map tool_choice only when tools are present. --------- Co-authored-by: Pasqual Troncone Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/llms/mistral/chat/transformation.py | 84 ++++++++--- .../test_mistral_chat_transformation.py | 130 +++++++++++++++++- 2 files changed, 189 insertions(+), 25 deletions(-) diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index 1ecef46ffb1..61d237de2d8 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -6,6 +6,7 @@ Why separate file? Make it easy to see how transformation works Docs - https://docs.mistral.ai/api/ """ +import reprlib from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence from itertools import accumulate, count from typing import TYPE_CHECKING, Final, Literal, cast, get_type_hints, overload @@ -45,7 +46,13 @@ from litellm.types.llms.mistral import ( MistralThinkingBlock, MistralToolCallMessage, ) -from litellm.types.llms.openai import AllMessageValues, ChatCompletionAssistantMessage, ChatCompletionTextObject +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionAssistantMessage, + ChatCompletionTextObject, + ChatCompletionToolChoiceFunctionParam, + ChatCompletionToolChoiceObjectParam, +) from litellm.types.utils import ModelResponse, ModelResponseStream from litellm.utils import convert_to_model_response_object, supports_reasoning @@ -79,6 +86,10 @@ _REASONING_PROMPT_REASON: Final = "litellm sends Magistral a reasoning system pr _NON_LEVEL_EFFORT_REASON: Final = ( "the Mistral API takes it only as a level such as 'high', so Mistral applies its default effort." ) +_UNUSABLE_TOOL_CHOICE_REASON: Final = ( + "the Mistral API takes tool_choice only as 'auto', 'none', 'any', 'required', " + "or {'type': 'function', 'function': {'name': }}." +) _DOCUMENTED_MAGISTRAL_EFFORTS: Final = frozenset({"low", "medium", "high"}) _REASONING_ON_THINKING_TYPES: Final = frozenset({"enabled", "adaptive"}) @@ -118,7 +129,7 @@ class MistralConfig(OpenAIGPTConfig): - `tools` (list or null): A list of available tools for the model. Use this to specify functions for which the model can generate JSON inputs. - - `tool_choice` (string - 'auto'/'any'/'none' or null): Specifies if/how functions are called. If set to none the model won't call a function and will generate a message instead. If set to auto the model can choose to either generate a message or call a function. If set to any the model is forced to call a function. Default - 'auto'. + - `tool_choice` (string - 'auto'/'any'/'none', an object naming one function such as {"type": "function", "function": {"name": "my_function"}}, or null): Specifies if/how functions are called. If set to none the model won't call a function and will generate a message instead. If set to auto the model can choose to either generate a message or call a function. If set to any the model is forced to call a function. Default - 'auto'. - `stop` (string or array of strings): Stop generation if this token is detected. Or if one of these tokens is detected when providing an array @@ -135,7 +146,7 @@ class MistralConfig(OpenAIGPTConfig): top_p: int | None = None max_tokens: int | None = None tools: list | None = None - tool_choice: Literal["auto", "any", "none"] | None = None + tool_choice: Literal["auto", "any", "none"] | ChatCompletionToolChoiceObjectParam | None = None random_seed: int | None = None safe_prompt: bool | None = None response_format: dict | None = None @@ -147,7 +158,7 @@ class MistralConfig(OpenAIGPTConfig): top_p: int | None = None, max_tokens: int | None = None, tools: list | None = None, - tool_choice: Literal["auto", "any", "none"] | None = None, + tool_choice: Literal["auto", "any", "none"] | ChatCompletionToolChoiceObjectParam | None = None, random_seed: int | None = None, safe_prompt: bool | None = None, response_format: dict | None = None, @@ -192,13 +203,40 @@ class MistralConfig(OpenAIGPTConfig): *(("prompt_cache_key",) if accepts_prompt_cache_key else ()), ] - def _map_tool_choice(self, tool_choice: str) -> str: - if tool_choice == "auto" or tool_choice == "none": - return tool_choice - elif tool_choice == "required": - return "any" - else: # openai 'tool_choice' object param not supported by Mistral API - return "any" + @staticmethod + def _map_tool_choice(tool_choice: object) -> str | ChatCompletionToolChoiceObjectParam | None: + match tool_choice: + case str(choice) if choice in ("auto", "none"): + return choice + case "required" | "any": + return "any" + case {"type": "function", "function": {"name": str(name)}} if name: + return ChatCompletionToolChoiceObjectParam( + type="function", function=ChatCompletionToolChoiceFunctionParam(name=name) + ) + case _: + return None + + def _tool_choice_to_send( + self, model: str, tool_choice: object, drop_params: bool + ) -> str | ChatCompletionToolChoiceObjectParam | UnsupportedParamsError | None: + mapped: Final = self._map_tool_choice(tool_choice) + if mapped is not None: + return mapped + if drop_params or litellm.drop_params is True: + warn_dropped_reasoning( + "tool_choice", tool_choice, model, _UNUSABLE_TOOL_CHOICE_REASON, drop_params_gated=True + ) + return None + return UnsupportedParamsError( + message=( + f"Unsupported tool_choice={reprlib.repr(tool_choice)} for model={model}, since " + f"{_UNUSABLE_TOOL_CHOICE_REASON} " + "To drop it, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n" + ), + llm_provider=self.custom_llm_provider, + model=model, + ) @staticmethod def _get_mistral_reasoning_system_prompt() -> str: @@ -244,18 +282,20 @@ class MistralConfig(OpenAIGPTConfig): optional_params["tools"] = self._clean_tool_schema_for_mistral(value) if param == "stream" and value is True: optional_params["stream"] = value - if param == "temperature": - optional_params["temperature"] = value - if param == "top_p": - optional_params["top_p"] = value - if param == "stop": - optional_params["stop"] = value - if param == "tool_choice" and isinstance(value, str): - optional_params["tool_choice"] = self._map_tool_choice(tool_choice=value) + if param == "tool_choice" and "tools" in non_default_params and non_default_params["tools"]: + match self._tool_choice_to_send( + model, + cast(object, value), # cast-ok: untyped request params, the match narrows it + drop_params, + ): + case UnsupportedParamsError() as unusable_tool_choice: + raise unusable_tool_choice + case None: + pass + case tool_choice: + optional_params["tool_choice"] = tool_choice if param == "seed": optional_params["extra_body"] = {"random_seed": value} - if param == "response_format": - optional_params["response_format"] = value if ( param == "reasoning_effort" and "magistral" not in model.lower() @@ -271,7 +311,7 @@ class MistralConfig(OpenAIGPTConfig): # Flag that we need to add reasoning system prompt optional_params["_add_reasoning_prompt"] = True _warn_unless_reasoning_prompt_honors(param, OPAQUE_VALUE.validate_python(value), model) - if param in ("parallel_tool_calls", "prompt_cache_key"): + if param in ("temperature", "top_p", "stop", "response_format", "parallel_tool_calls", "prompt_cache_key"): optional_params[param] = value return optional_params diff --git a/tests/unit/llms/mistral/test_mistral_chat_transformation.py b/tests/unit/llms/mistral/test_mistral_chat_transformation.py index af18a9caf3d..42673e17acf 100644 --- a/tests/unit/llms/mistral/test_mistral_chat_transformation.py +++ b/tests/unit/llms/mistral/test_mistral_chat_transformation.py @@ -1,7 +1,7 @@ import copy import json import logging -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from typing import Final, List, cast from unittest.mock import MagicMock, patch from urllib.parse import urlparse @@ -18,7 +18,7 @@ from litellm.llms.mistral.chat.transformation import ( MistralConfig, ) from litellm.llms.vertex_ai.vertex_ai_partner_models.mistral.transformation import VertexAIMistralConfig -from litellm.types.llms.openai import AllMessageValues +from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam from litellm.types.utils import ModelResponse from litellm.utils import supports_reasoning @@ -1690,7 +1690,9 @@ def test_mistral_request_drops_a_user_name_without_touching_the_callers_message( def _warnings_naming(caplog: pytest.LogCaptureFixture, param: str) -> tuple[str, ...]: return tuple( - r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage() + r.getMessage() + for r in caplog.records + if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage() ) @@ -2060,3 +2062,125 @@ def test_prompt_cache_key_stays_unsupported_for_vertex_hosted_mistral(local_mode model="mistral-medium-3", custom_llm_provider="vertex_ai", prompt_cache_key="s", drop_params=True ) assert "prompt_cache_key" not in dropped + + +class TestMistralToolChoice: + """Mistral takes tool_choice as an enum or as {"type": "function", "function": {"name": ...}}, per + https://docs.mistral.ai/api/endpoint/chat (checked 2026-11-03)""" + + TOOLS: Final[Sequence[ChatCompletionToolParam]] = [ + { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}}, + } + ] + + UNUSABLE_TOOL_CHOICES: Final[Sequence[str | Mapping[str, object]]] = ( + {"type": "function", "function": {}}, + {"type": "function", "function": {"name": ""}}, + {"type": "function", "function": {"name": 123}}, + {"type": "function"}, + "foo", + ) + + def _request_body( + self, + tool_choice: str | Mapping[str, object], + drop_params: bool = False, + tools: Sequence[ChatCompletionToolParam] | None = TOOLS, + ) -> Mapping[str, object]: + import litellm + + optional_params: Final = litellm.get_optional_params( + model="mistral-large-latest", + custom_llm_provider="mistral", + tools=None if tools is None else list(tools), + tool_choice=tool_choice, + drop_params=drop_params, + ) + return MistralConfig().transform_request( + model="mistral-large-latest", + messages=[{"role": "user", "content": "what is the weather in Madrid"}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + def test_named_function_is_forwarded_to_mistral(self, local_model_cost_map: None) -> None: + body: Final = self._request_body({"type": "function", "function": {"name": "get_weather"}}) + + assert body["tool_choice"] == {"type": "function", "function": {"name": "get_weather"}} + + @pytest.mark.parametrize( + "custom_llm_provider, model", + [("mistral", "mistral-large-latest"), ("codestral", "codestral-latest"), ("vertex_ai", "mistral-small-2503")], + ) + def test_named_function_is_kept_on_every_mistral_route( + self, local_model_cost_map: None, custom_llm_provider: str, model: str + ) -> None: + import litellm + + optional_params: Final = litellm.get_optional_params( + model=model, + custom_llm_provider=custom_llm_provider, + tools=list(self.TOOLS), + tool_choice={"type": "function", "function": {"name": "get_weather"}}, + ) + + assert optional_params["tool_choice"] == {"type": "function", "function": {"name": "get_weather"}} + + def test_keys_mistral_does_not_define_are_not_replayed(self, local_model_cost_map: None) -> None: + body: Final = self._request_body( + { + "type": "function", + "function": {"name": "get_weather", "strict": True}, + "cache_control": {"type": "ephemeral"}, + } + ) + + assert body["tool_choice"] == {"type": "function", "function": {"name": "get_weather"}} + + @pytest.mark.parametrize( + "tool_choice, expected", + [("auto", "auto"), ("none", "none"), ("required", "any"), ("any", "any")], + ) + def test_enum_tool_choice_is_mapped(self, local_model_cost_map: None, tool_choice: str, expected: str) -> None: + assert self._request_body(tool_choice)["tool_choice"] == expected + + @pytest.mark.parametrize("tool_choice", UNUSABLE_TOOL_CHOICES) + def test_unusable_tool_choice_is_rejected_without_drop_params( + self, local_model_cost_map: None, tool_choice: str | Mapping[str, object] + ) -> None: + import litellm + + with pytest.raises(litellm.UnsupportedParamsError, match="tool_choice"): + self._request_body(tool_choice) + + @pytest.mark.parametrize("tools", [None, ()]) + @pytest.mark.parametrize("tool_choice", UNUSABLE_TOOL_CHOICES) + def test_unusable_tool_choice_without_tools_is_not_sent( + self, + local_model_cost_map: None, + tool_choice: str | Mapping[str, object], + tools: Sequence[ChatCompletionToolParam] | None, + ) -> None: + assert "tool_choice" not in self._request_body(tool_choice, tools=tools) + + @pytest.mark.parametrize("tool_choice", UNUSABLE_TOOL_CHOICES) + def test_unusable_tool_choice_is_dropped_with_a_warning_under_drop_params( + self, local_model_cost_map: None, tool_choice: str | Mapping[str, object], caplog: pytest.LogCaptureFixture + ) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + body: Final = self._request_body(tool_choice, drop_params=True) + + assert "tool_choice" not in body + assert len(_warnings_naming(caplog, "tool_choice")) == 1 + + def test_unusable_tool_choice_is_dropped_under_proxy_wide_drop_params( + self, local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch + ) -> None: + import litellm + + monkeypatch.setattr(litellm, "drop_params", True) + + assert "tool_choice" not in self._request_body({"type": "function", "function": {}})