From 2ca3b36782a78d2cbed094157cbba7ac19a0f5b1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 19 Oct 2026 16:07:36 -0700 Subject: [PATCH 1/8] fix(mistral): forward prompt_cache_key to the Mistral API --- litellm/llms/mistral/chat/transformation.py | 6 ++- .../test_mistral_chat_transformation.py | 38 +++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index b8a6cb1beb9..dd4682e6d69 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -122,6 +122,7 @@ class MistralConfig(OpenAIGPTConfig): accepts_reasoning_effort: Final = is_magistral or supports_reasoning( model=model, custom_llm_provider=self.custom_llm_provider ) + accepts_prompt_cache_key: Final = self.custom_llm_provider == "mistral" return [ "stream", "temperature", @@ -136,6 +137,7 @@ class MistralConfig(OpenAIGPTConfig): "parallel_tool_calls", *(("thinking",) if is_magistral else ()), *(("reasoning_effort",) if accepts_reasoning_effort else ()), + *(("prompt_cache_key",) if accepts_prompt_cache_key else ()), ] def _map_tool_choice(self, tool_choice: str) -> str: @@ -207,8 +209,8 @@ class MistralConfig(OpenAIGPTConfig): if param in ("reasoning_effort", "thinking") and "magistral" in model.lower(): # Flag that we need to add reasoning system prompt optional_params["_add_reasoning_prompt"] = True - if param == "parallel_tool_calls": - optional_params["parallel_tool_calls"] = value + if param in ("parallel_tool_calls", "prompt_cache_key"): + optional_params[param] = value return optional_params def _get_openai_compatible_provider_info(self, api_base: str | None, api_key: str | None) -> tuple[str, str | None]: diff --git a/tests/unit/llms/mistral/test_mistral_chat_transformation.py b/tests/unit/llms/mistral/test_mistral_chat_transformation.py index 10f57be33fe..0c0efb7ece4 100644 --- a/tests/unit/llms/mistral/test_mistral_chat_transformation.py +++ b/tests/unit/llms/mistral/test_mistral_chat_transformation.py @@ -1162,3 +1162,41 @@ def test_mistral_transform_request_drops_an_empty_tool_message_whose_call_id_is_ ) assert [m.get("role") for m in request["messages"]] == ["user", "assistant"] + + +@pytest.mark.parametrize( + ("model", "custom_llm_provider"), + [ + ("mistral-large-latest", "mistral"), + ("mistral-large-4", "mistral"), + ("a-mistral-model-missing-from-the-cost-map", "mistral"), + ("codestral-latest", "codestral"), + ], +) +def test_prompt_cache_key_reaches_the_mistral_request_body(local_model_cost_map, model, custom_llm_provider): + import litellm + + optional_params = litellm.get_optional_params( + model=model, custom_llm_provider=custom_llm_provider, prompt_cache_key="session-1" + ) + request = MistralConfig().transform_request( + model=model, + messages=[{"role": "user", "content": "hi"}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert request["prompt_cache_key"] == "session-1" + + +def test_prompt_cache_key_stays_unsupported_for_vertex_hosted_mistral(local_model_cost_map): + import litellm + + with pytest.raises(litellm.UnsupportedParamsError): + litellm.get_optional_params(model="mistral-medium-3", custom_llm_provider="vertex_ai", prompt_cache_key="s") + + dropped = litellm.get_optional_params( + model="mistral-medium-3", custom_llm_provider="vertex_ai", prompt_cache_key="s", drop_params=True + ) + assert "prompt_cache_key" not in dropped From 9f7b4157c80dd7c3be75d46212836699caa48797 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:43:30 -0700 Subject: [PATCH 2/8] fix(mistral): send prompt_cache_key only to Mistral's API and accept reasoning_effort on devstral and code-agent models --- .../pass_through/adapters/handler.py | 2 + .../pass_through/adapters/transformation.py | 17 ++- litellm/llms/mistral/chat/transformation.py | 37 ++++- ...odel_prices_and_context_window_backup.json | 25 ++++ litellm/types/llms/mistral.py | 7 +- model_prices_and_context_window.json | 25 ++++ .../adapters/test_handler_prompt_cache_key.py | 25 ++++ .../test_mistral_chat_transformation.py | 128 ++++++++++-------- 8 files changed, 204 insertions(+), 62 deletions(-) diff --git a/litellm/llms/anthropic/pass_through/adapters/handler.py b/litellm/llms/anthropic/pass_through/adapters/handler.py index 666823d6ec3..eefc2301e3d 100644 --- a/litellm/llms/anthropic/pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/adapters/handler.py @@ -633,12 +633,14 @@ class LiteLLMMessagesToCompletionTransformationHandler: request_data["output_config"] = extra_kwargs["output_config"] custom_llm_provider: Final = extra_kwargs.get("custom_llm_provider") + api_base: Final = extra_kwargs.get("api_base") ( openai_request, tool_name_mapping, ) = ANTHROPIC_ADAPTER.translate_completion_input_params_with_tool_mapping( request_data, custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None, + api_base=api_base if isinstance(api_base, str) else None, ) if openai_request is None: diff --git a/litellm/llms/anthropic/pass_through/adapters/transformation.py b/litellm/llms/anthropic/pass_through/adapters/transformation.py index de929acbfb2..bbdde20bb08 100644 --- a/litellm/llms/anthropic/pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/adapters/transformation.py @@ -23,6 +23,7 @@ OPENAI_MAX_TOOL_NAME_LENGTH: Final = 64 TOOL_NAME_HASH_LENGTH: Final = 8 TOOL_NAME_PREFIX_LENGTH: Final = OPENAI_MAX_TOOL_NAME_LENGTH - TOOL_NAME_HASH_LENGTH - 1 # 55 PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND: Final = frozenset({"litellm_proxy"}) +PROVIDERS_SERVED_BY_THE_MISTRAL_CHAT_CONFIG: Final = frozenset({"mistral", "codestral"}) _COMPACTION_BLOCK: Final = TypeAdapter(dict[str, JsonValue]) @@ -248,7 +249,7 @@ class AnthropicAdapter: return result def translate_completion_input_params_with_tool_mapping( - self, kwargs, *, custom_llm_provider: str | None = None + self, kwargs, *, custom_llm_provider: str | None = None, api_base: str | None = None ) -> tuple[ChatCompletionRequest | None, dict[str, str]]: """ Translate Anthropic request params to OpenAI format, returning tool name mapping. @@ -282,6 +283,7 @@ class AnthropicAdapter: ) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( anthropic_message_request=request_body, custom_llm_provider=custom_llm_provider, + api_base=api_base, ) return translated_body, tool_name_mapping @@ -988,7 +990,7 @@ class LiteLLMAnthropicMessagesAdapter: ) @staticmethod - def _supports_prompt_cache_key(model: str | None, custom_llm_provider: str | None) -> bool: + def _supports_prompt_cache_key(model: str | None, custom_llm_provider: str | None, api_base: str | None) -> bool: if not model or not custom_llm_provider: return False if custom_llm_provider in PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND: @@ -996,7 +998,11 @@ class LiteLLMAnthropicMessagesAdapter: supported_params: Final = litellm.get_supported_openai_params( model=model, custom_llm_provider=custom_llm_provider ) - return "prompt_cache_key" in (supported_params or ()) + if "prompt_cache_key" not in (supported_params or ()): + return False + return custom_llm_provider not in PROVIDERS_SERVED_BY_THE_MISTRAL_CHAT_CONFIG or ( + litellm.MistralConfig().accepts_prompt_cache_key_at(api_base) + ) @staticmethod def _target_declares_reasoning_effort(model: str, custom_llm_provider: str | None) -> bool: @@ -1029,6 +1035,7 @@ class LiteLLMAnthropicMessagesAdapter: new_kwargs: ChatCompletionRequest, *, custom_llm_provider: str | None = None, + api_base: str | None = None, ) -> None: """Translate metadata fields from Anthropic request to OpenAI request.""" if "metadata" in anthropic_message_request: @@ -1037,7 +1044,7 @@ class LiteLLMAnthropicMessagesAdapter: new_kwargs["user"] = metadata["user_id"] prompt_cache_key: Final = prompt_cache_key_from_user_id(metadata["user_id"]) if prompt_cache_key is not None and self._supports_prompt_cache_key( - anthropic_message_request.get("model"), custom_llm_provider + anthropic_message_request.get("model"), custom_llm_provider, api_base ): new_kwargs["prompt_cache_key"] = prompt_cache_key @@ -1227,6 +1234,7 @@ class LiteLLMAnthropicMessagesAdapter: anthropic_message_request: AnthropicMessagesRequest, *, custom_llm_provider: str | None = None, + api_base: str | None = None, preserve_midturn_system: bool = False, ) -> tuple[ChatCompletionRequest, dict[str, str]]: """ @@ -1264,6 +1272,7 @@ class LiteLLMAnthropicMessagesAdapter: anthropic_message_request=anthropic_message_request, new_kwargs=new_kwargs, custom_llm_provider=custom_llm_provider, + api_base=api_base, ) ## CONVERT TOOL CHOICE self._translate_tool_choice_to_openai( diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index dd4682e6d69..b84376a074a 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -9,10 +9,14 @@ Docs - https://docs.mistral.ai/api/ from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping from itertools import accumulate, count from typing import TYPE_CHECKING, Final, Literal, cast, get_type_hints, overload +from urllib.parse import urlparse import httpx +from pydantic import TypeAdapter +import litellm from litellm._logging import verbose_logger +from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.prompt_templates.common_utils import ( handle_messages_with_content_list_to_str_conversion, @@ -28,7 +32,7 @@ from litellm.router_utils.reasoning_effort_capability import ( nearest_declared_reasoning_effort, ) from litellm.secret_managers.main import get_secret_str -from litellm.types.llms.mistral import MistralThinkingBlock, MistralToolCallMessage +from litellm.types.llms.mistral import MistralRequestRouting, MistralThinkingBlock, MistralToolCallMessage from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse, ModelResponseStream from litellm.utils import convert_to_model_response_object, supports_reasoning @@ -55,6 +59,7 @@ def _accepted_reasoning_effort(model: str, requested: str, custom_llm_provider: _NO_TOOL_CALL_TURN: Final[tuple[int, frozenset[str]]] = (-1, frozenset()) +_MISTRAL_REQUEST_ROUTING: Final = TypeAdapter(MistralRequestRouting) class MistralConfig(OpenAIGPTConfig): @@ -213,6 +218,28 @@ class MistralConfig(OpenAIGPTConfig): optional_params[param] = value return optional_params + def accepts_prompt_cache_key_at(self, api_base: str | None) -> bool: + resolved_api_base, _ = self._get_openai_compatible_provider_info(api_base, None) + hostname: Final = urlparse(resolved_api_base).hostname or "" + return hostname == "mistral.ai" or hostname.endswith(".mistral.ai") + + def _prompt_cache_key_withheld(self, model: str, routing: MistralRequestRouting) -> bool: + requested_api_base: Final = routing.get("api_base") + api_base: Final = requested_api_base if isinstance(requested_api_base, str) else None + if self.accepts_prompt_cache_key_at(api_base): + return False + if litellm.drop_params is True or routing.get("drop_params") is True: + return True + raise UnsupportedParamsError( + message=( + f"{self.custom_llm_provider} does not support parameters: ['prompt_cache_key'], for model={model} " + f"at api_base={api_base}, since only the Mistral API (*.mistral.ai) takes it. To drop it, set " + "`litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n" + ), + llm_provider=self.custom_llm_provider, + model=model, + ) + def _get_openai_compatible_provider_info(self, api_base: str | None, api_key: str | None) -> tuple[str, str | None]: # mistral is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.mistral.ai api_base = ( @@ -635,7 +662,13 @@ class MistralConfig(OpenAIGPTConfig): if "magistral" in model.lower() and optional_params.get("_add_reasoning_prompt", False): messages = self._add_reasoning_system_prompt_if_needed(messages, optional_params) - upstream_params: Final = {key: value for key, value in optional_params.items() if key != "client_metadata"} + withheld_params: Final = ( + ("client_metadata", "prompt_cache_key") + if "prompt_cache_key" in optional_params + and self._prompt_cache_key_withheld(model, _MISTRAL_REQUEST_ROUTING.validate_python(litellm_params)) + else ("client_metadata",) + ) + upstream_params: Final = {key: value for key, value in optional_params.items() if key not in withheld_params} # Call parent transform_request which handles _transform_messages return super().transform_request( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1cadc93969b..6d99a24b41d 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -41596,9 +41596,14 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 3e-07, + "reasoning_effort_levels": [ + "none", + "high" + ], "source": "https://docs.mistral.ai/models/devstral-small-2-25-12", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, @@ -41611,9 +41616,14 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, + "reasoning_effort_levels": [ + "none", + "high" + ], "source": "https://mistral.ai/news/devstral-2-vibe-cli", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, @@ -41626,9 +41636,14 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, + "reasoning_effort_levels": [ + "none", + "high" + ], "source": "https://mistral.ai/news/devstral-2-vibe-cli", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, @@ -42199,8 +42214,13 @@ "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 3e-07, + "reasoning_effort_levels": [ + "none", + "high" + ], "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, @@ -66379,9 +66399,14 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, + "reasoning_effort_levels": [ + "none", + "high" + ], "source": "https://mistral.ai/news/devstral-2-vibe-cli", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, diff --git a/litellm/types/llms/mistral.py b/litellm/types/llms/mistral.py index e64aa9c13ab..f6068d6d0a5 100644 --- a/litellm/types/llms/mistral.py +++ b/litellm/types/llms/mistral.py @@ -1,6 +1,6 @@ from typing import Literal -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict class FunctionCall(TypedDict): @@ -22,3 +22,8 @@ class MistralTextBlock(TypedDict): class MistralThinkingBlock(TypedDict): type: Literal["thinking"] thinking: list[MistralTextBlock] + + +class MistralRequestRouting(TypedDict, total=False): + api_base: ReadOnly[object] + drop_params: ReadOnly[object] diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1cadc93969b..6d99a24b41d 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -41596,9 +41596,14 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 3e-07, + "reasoning_effort_levels": [ + "none", + "high" + ], "source": "https://docs.mistral.ai/models/devstral-small-2-25-12", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, @@ -41611,9 +41616,14 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, + "reasoning_effort_levels": [ + "none", + "high" + ], "source": "https://mistral.ai/news/devstral-2-vibe-cli", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, @@ -41626,9 +41636,14 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, + "reasoning_effort_levels": [ + "none", + "high" + ], "source": "https://mistral.ai/news/devstral-2-vibe-cli", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, @@ -42199,8 +42214,13 @@ "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 3e-07, + "reasoning_effort_levels": [ + "none", + "high" + ], "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, @@ -66379,9 +66399,14 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, + "reasoning_effort_levels": [ + "none", + "high" + ], "source": "https://mistral.ai/news/devstral-2-vibe-cli", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_prompt_cache_key.py b/tests/unit/llms/anthropic/pass_through/adapters/test_handler_prompt_cache_key.py index c31d85be0a8..ec8c30c9d37 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_prompt_cache_key.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_handler_prompt_cache_key.py @@ -70,3 +70,28 @@ def test_prepare_completion_kwargs_keeps_prompt_cache_key_through_responses_rero ) assert completion_kwargs["model"] == "openai/responses/gpt-5.6-luna" assert completion_kwargs["prompt_cache_key"] == "session-abc" + + +@pytest.mark.parametrize( + ("api_base", "azure_api_base_env", "derives_key"), + [ + (None, None, True), + ("https://api.mistral.ai/v1", None, True), + ("https://my-resource.openai.azure.com/openai/v1", None, False), + (None, "https://my-resource.services.ai.azure.com/openai", False), + ], +) +def test_prepare_completion_kwargs_derives_prompt_cache_key_only_for_a_mistral_hosted_endpoint( + monkeypatch: pytest.MonkeyPatch, api_base: str | None, azure_api_base_env: str | None, derives_key: bool +): + if azure_api_base_env is None: + monkeypatch.delenv("MISTRAL_AZURE_API_BASE", raising=False) + else: + monkeypatch.setenv("MISTRAL_AZURE_API_BASE", azure_api_base_env) + + completion_kwargs = _prepare( + "mistral/mistral-small-latest", {"custom_llm_provider": "mistral", "api_base": api_base} + ) + + assert completion_kwargs["user"] == CLAUDE_CODE_USER_ID + assert ("prompt_cache_key" in completion_kwargs) is derives_key diff --git a/tests/unit/llms/mistral/test_mistral_chat_transformation.py b/tests/unit/llms/mistral/test_mistral_chat_transformation.py index 0c0efb7ece4..a79999320ed 100644 --- a/tests/unit/llms/mistral/test_mistral_chat_transformation.py +++ b/tests/unit/llms/mistral/test_mistral_chat_transformation.py @@ -1,4 +1,5 @@ -from typing import List, cast +from collections.abc import Mapping +from typing import Final, List, cast from unittest.mock import MagicMock, patch import pytest @@ -43,21 +44,15 @@ class TestMistralReasoningSupport: mistral_config = MistralConfig() # Test magistral model supports reasoning parameters - supported_params = mistral_config.get_supported_openai_params( - "mistral/magistral-medium-2506" - ) + supported_params = mistral_config.get_supported_openai_params("mistral/magistral-medium-2506") assert "reasoning_effort" in supported_params assert "thinking" in supported_params - supported_params_reasoning = mistral_config.get_supported_openai_params( - "mistral/mistral-medium-latest" - ) + supported_params_reasoning = mistral_config.get_supported_openai_params("mistral/mistral-medium-latest") assert "reasoning_effort" in supported_params_reasoning assert "thinking" not in supported_params_reasoning - supported_params_normal = mistral_config.get_supported_openai_params( - "mistral/mistral-large-latest" - ) + supported_params_normal = mistral_config.get_supported_openai_params("mistral/mistral-large-latest") assert "reasoning_effort" not in supported_params_normal assert "thinking" not in supported_params_normal @@ -192,9 +187,7 @@ class TestMistralReasoningSupport: messages = [{"role": "user", "content": "What is 2+2?"}] optional_params = {"_add_reasoning_prompt": True} - result = mistral_config._add_reasoning_system_prompt_if_needed( - messages, optional_params - ) + result = mistral_config._add_reasoning_system_prompt_if_needed(messages, optional_params) # Should add a new system message at the beginning assert len(result) == 2 @@ -216,9 +209,7 @@ class TestMistralReasoningSupport: ] optional_params = {"_add_reasoning_prompt": True} - result = mistral_config._add_reasoning_system_prompt_if_needed( - messages, optional_params - ) + result = mistral_config._add_reasoning_system_prompt_if_needed(messages, optional_params) # Should modify existing system message assert len(result) == 2 @@ -249,9 +240,7 @@ class TestMistralReasoningSupport: ] optional_params = {"_add_reasoning_prompt": True} - result = mistral_config._add_reasoning_system_prompt_if_needed( - messages, optional_params - ) + result = mistral_config._add_reasoning_system_prompt_if_needed(messages, optional_params) # Should modify existing system message preserving list format assert len(result) == 2 @@ -264,10 +253,7 @@ class TestMistralReasoningSupport: # Original content should be preserved assert "You are a helpful assistant." in result[0]["content"][1]["text"] - assert ( - "You always provide detailed explanations." - in result[0]["content"][2]["text"] - ) + assert "You always provide detailed explanations." in result[0]["content"][2]["text"] assert result[1]["role"] == "user" @@ -285,9 +271,7 @@ class TestMistralReasoningSupport: ] string_params = {"_add_reasoning_prompt": True} - string_result = mistral_config._add_reasoning_system_prompt_if_needed( - string_messages, string_params - ) + string_result = mistral_config._add_reasoning_system_prompt_if_needed(string_messages, string_params) assert isinstance(string_result[0]["content"], str) assert "" in string_result[0]["content"] assert "You are helpful." in string_result[0]["content"] @@ -302,9 +286,7 @@ class TestMistralReasoningSupport: ] list_params = {"_add_reasoning_prompt": True} - list_result = mistral_config._add_reasoning_system_prompt_if_needed( - list_messages, list_params - ) + list_result = mistral_config._add_reasoning_system_prompt_if_needed(list_messages, list_params) assert isinstance(list_result[0]["content"], list) assert list_result[0]["content"][0]["type"] == "text" assert "" in list_result[0]["content"][0]["text"] @@ -317,9 +299,7 @@ class TestMistralReasoningSupport: messages = [{"role": "user", "content": "What is 2+2?"}] optional_params = {} - result = mistral_config._add_reasoning_system_prompt_if_needed( - messages, optional_params - ) + result = mistral_config._add_reasoning_system_prompt_if_needed(messages, optional_params) # Should return messages unchanged assert result == messages @@ -442,9 +422,7 @@ class TestMistralReasoningSupport: def test_mistral_streaming_chunk_preserves_thinking_blocks(): """Ensure streaming chunks keep magistral reasoning content.""" - iterator = MistralChatResponseIterator( - streaming_response=iter([]), sync_stream=True, json_mode=False - ) + iterator = MistralChatResponseIterator(streaming_response=iter([]), sync_stream=True, json_mode=False) streamed_chunk = { "id": "chunk-1", @@ -521,9 +499,7 @@ class TestMistralParallelToolCalls: def test_get_supported_openai_params_includes_parallel_tool_calls(self): """Test that parallel_tool_calls is in supported parameters.""" mistral_config = MistralConfig() - supported_params = mistral_config.get_supported_openai_params( - "mistral/mistral-large-latest" - ) + supported_params = mistral_config.get_supported_openai_params("mistral/mistral-large-latest") assert "parallel_tool_calls" in supported_params def test_transform_request_preserves_parallel_tool_calls(self): @@ -689,9 +665,7 @@ class TestMistralEmptyContentHandling: result = MistralConfig._handle_empty_content_response(response_data) - assert ( - result["choices"][0]["message"]["content"] == "Hello, how can I help you?" - ) + assert result["choices"][0]["message"]["content"] == "Hello, how can I help you?" def test_handle_empty_content_response_handles_multiple_choices(self): """Test that only the first choice is processed for empty content.""" @@ -928,9 +902,7 @@ def test_mistral_transform_request_keeps_an_empty_tool_message_after_its_tool_ca { "role": "assistant", "content": "", - "tool_calls": [ - {"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}} - ], + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}}], }, {"role": "tool", "tool_call_id": "call_1", "content": ""}, ], @@ -954,9 +926,7 @@ def test_mistral_transform_request_sends_a_null_tool_reply_as_empty_text(reply): { "role": "assistant", "content": "", - "tool_calls": [ - {"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}} - ], + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}}], }, {"role": "tool", "tool_call_id": "call_1", **reply}, ], @@ -1011,9 +981,7 @@ def test_mistral_transform_request_hoists_tool_message_image(): { "role": "assistant", "content": "", - "tool_calls": [ - {"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}} - ], + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}}], }, { "role": "tool", @@ -1173,13 +1141,15 @@ def test_mistral_transform_request_drops_an_empty_tool_message_whose_call_id_is_ ("codestral-latest", "codestral"), ], ) -def test_prompt_cache_key_reaches_the_mistral_request_body(local_model_cost_map, model, custom_llm_provider): +def test_prompt_cache_key_reaches_the_mistral_request_body( + local_model_cost_map: None, model: str, custom_llm_provider: str +) -> None: import litellm - optional_params = litellm.get_optional_params( + optional_params: Final = litellm.get_optional_params( model=model, custom_llm_provider=custom_llm_provider, prompt_cache_key="session-1" ) - request = MistralConfig().transform_request( + request: Final = MistralConfig().transform_request( model=model, messages=[{"role": "user", "content": "hi"}], optional_params=optional_params, @@ -1190,13 +1160,61 @@ def test_prompt_cache_key_reaches_the_mistral_request_body(local_model_cost_map, assert request["prompt_cache_key"] == "session-1" -def test_prompt_cache_key_stays_unsupported_for_vertex_hosted_mistral(local_model_cost_map): +AZURE_HOSTED_MISTRAL_API_BASE: Final = "https://my-resource.services.ai.azure.com/openai/v1" + + +def _transform_with_prompt_cache_key(litellm_params: dict[str, object]) -> Mapping[str, object]: + return cast( + Mapping[str, object], + MistralConfig().transform_request( + model="mistral-small-latest", + messages=[{"role": "user", "content": "hi"}], + optional_params={"prompt_cache_key": "session-1", "temperature": 0.2}, + litellm_params=litellm_params, + headers={}, + ), + ) + + +@pytest.mark.parametrize("api_base", ["https://api.mistral.ai/v1", "https://codestral.mistral.ai/v1"]) +def test_prompt_cache_key_reaches_a_mistral_hosted_endpoint(api_base: str) -> None: + request: Final = _transform_with_prompt_cache_key({"api_base": api_base}) + + assert request["prompt_cache_key"] == "session-1" + + +@pytest.mark.parametrize("api_base", [AZURE_HOSTED_MISTRAL_API_BASE, "https://mistral.ai.example.com/v1"]) +def test_prompt_cache_key_for_another_host_is_rejected_without_drop_params(api_base: str) -> None: + import litellm + + with pytest.raises(litellm.UnsupportedParamsError, match="prompt_cache_key"): + _transform_with_prompt_cache_key({"api_base": api_base}) + + +def test_prompt_cache_key_for_another_host_is_dropped_under_drop_params() -> None: + request: Final = _transform_with_prompt_cache_key({"api_base": AZURE_HOSTED_MISTRAL_API_BASE, "drop_params": True}) + + assert "prompt_cache_key" not in request + assert request["temperature"] == 0.2 + + +def test_prompt_cache_key_for_the_azure_endpoint_set_in_the_environment_is_dropped_under_drop_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MISTRAL_AZURE_API_BASE", "https://my-resource.services.ai.azure.com/openai") + + request: Final = _transform_with_prompt_cache_key({"drop_params": True}) + + assert "prompt_cache_key" not in request + + +def test_prompt_cache_key_stays_unsupported_for_vertex_hosted_mistral(local_model_cost_map: None) -> None: import litellm with pytest.raises(litellm.UnsupportedParamsError): litellm.get_optional_params(model="mistral-medium-3", custom_llm_provider="vertex_ai", prompt_cache_key="s") - dropped = litellm.get_optional_params( + dropped: Final = litellm.get_optional_params( model="mistral-medium-3", custom_llm_provider="vertex_ai", prompt_cache_key="s", drop_params=True ) assert "prompt_cache_key" not in dropped From cacb634355fcf2a35f398d3a2c8f371ac3815708 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:54:16 -0700 Subject: [PATCH 3/8] fix(mistral): keep the upstream api_base out of the prompt_cache_key rejection message --- litellm/llms/mistral/chat/transformation.py | 4 ++-- tests/unit/llms/mistral/test_mistral_chat_transformation.py | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index b84376a074a..fdf89a250f1 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -232,8 +232,8 @@ class MistralConfig(OpenAIGPTConfig): return True raise UnsupportedParamsError( message=( - f"{self.custom_llm_provider} does not support parameters: ['prompt_cache_key'], for model={model} " - f"at api_base={api_base}, since only the Mistral API (*.mistral.ai) takes it. To drop it, set " + f"{self.custom_llm_provider} does not support parameters: ['prompt_cache_key'], for model={model}, " + "since only the Mistral API (*.mistral.ai) takes it. To drop it, set " "`litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n" ), llm_provider=self.custom_llm_provider, diff --git a/tests/unit/llms/mistral/test_mistral_chat_transformation.py b/tests/unit/llms/mistral/test_mistral_chat_transformation.py index a79999320ed..1b9568eec84 100644 --- a/tests/unit/llms/mistral/test_mistral_chat_transformation.py +++ b/tests/unit/llms/mistral/test_mistral_chat_transformation.py @@ -1,6 +1,7 @@ from collections.abc import Mapping from typing import Final, List, cast from unittest.mock import MagicMock, patch +from urllib.parse import urlparse import pytest @@ -1187,9 +1188,12 @@ def test_prompt_cache_key_reaches_a_mistral_hosted_endpoint(api_base: str) -> No def test_prompt_cache_key_for_another_host_is_rejected_without_drop_params(api_base: str) -> None: import litellm - with pytest.raises(litellm.UnsupportedParamsError, match="prompt_cache_key"): + with pytest.raises(litellm.UnsupportedParamsError, match="prompt_cache_key") as rejection: _transform_with_prompt_cache_key({"api_base": api_base}) + upstream_host: Final = urlparse(api_base).hostname or api_base + assert upstream_host not in str(rejection.value) + def test_prompt_cache_key_for_another_host_is_dropped_under_drop_params() -> None: request: Final = _transform_with_prompt_cache_key({"api_base": AZURE_HOSTED_MISTRAL_API_BASE, "drop_params": True}) From 2b35904c83d45e528fa7ec532ff42834be1db28f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:06:36 -0700 Subject: [PATCH 4/8] fix(mistral): honor allowed_openai_params for prompt_cache_key and gate codestral async and env hosts --- .../litellm_core_utils/get_litellm_params.py | 1 + .../pass_through/adapters/transformation.py | 7 +- litellm/llms/mistral/chat/transformation.py | 8 ++- litellm/main.py | 1 + litellm/types/llms/mistral.py | 1 + .../adapters/test_handler_prompt_cache_key.py | 18 +++++ .../test_mistral_chat_transformation.py | 71 +++++++++++++++++++ 7 files changed, 104 insertions(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index 6ca02cbbe64..879a5aeafe0 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -79,6 +79,7 @@ OPTIONAL_KWARGS_KEYS: Final = ( "github_copilot_user_session", "fireworks_forward_user_id", "responses_api_version", + "allowed_openai_params", PROVIDER_AFFINITY_HEADER_KWARG_KEY, } ) diff --git a/litellm/llms/anthropic/pass_through/adapters/transformation.py b/litellm/llms/anthropic/pass_through/adapters/transformation.py index bbdde20bb08..5c81514b5c8 100644 --- a/litellm/llms/anthropic/pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/adapters/transformation.py @@ -1000,9 +1000,12 @@ class LiteLLMAnthropicMessagesAdapter: ) if "prompt_cache_key" not in (supported_params or ()): return False - return custom_llm_provider not in PROVIDERS_SERVED_BY_THE_MISTRAL_CHAT_CONFIG or ( - litellm.MistralConfig().accepts_prompt_cache_key_at(api_base) + if custom_llm_provider not in PROVIDERS_SERVED_BY_THE_MISTRAL_CHAT_CONFIG: + return True + _, _, _, resolved_api_base = litellm.get_llm_provider( + model=model, custom_llm_provider=custom_llm_provider, api_base=api_base ) + return litellm.MistralConfig().accepts_prompt_cache_key_at(resolved_api_base) @staticmethod def _target_declares_reasoning_effort(model: str, custom_llm_provider: str | None) -> bool: diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index fdf89a250f1..c0fcd6b37e0 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -87,6 +87,8 @@ class MistralConfig(OpenAIGPTConfig): - `response_format` (object or null): An object specifying the format that the model must output. Setting to { "type": "json_object" } enables JSON mode, which guarantees the message the model generates is in JSON. When using JSON mode you MUST also instruct the model to produce JSON yourself with a system or a user message. """ + _is_base_class = False + temperature: int | None = None top_p: int | None = None max_tokens: int | None = None @@ -226,6 +228,9 @@ class MistralConfig(OpenAIGPTConfig): def _prompt_cache_key_withheld(self, model: str, routing: MistralRequestRouting) -> bool: requested_api_base: Final = routing.get("api_base") api_base: Final = requested_api_base if isinstance(requested_api_base, str) else None + allowed_openai_params: Final = routing.get("allowed_openai_params") + if isinstance(allowed_openai_params, (list, tuple)) and "prompt_cache_key" in allowed_openai_params: + return False if self.accepts_prompt_cache_key_at(api_base): return False if litellm.drop_params is True or routing.get("drop_params") is True: @@ -233,7 +238,8 @@ class MistralConfig(OpenAIGPTConfig): raise UnsupportedParamsError( message=( f"{self.custom_llm_provider} does not support parameters: ['prompt_cache_key'], for model={model}, " - "since only the Mistral API (*.mistral.ai) takes it. To drop it, set " + "since only the Mistral API (*.mistral.ai) takes it. To send it anyway, pass " + "`allowed_openai_params=['prompt_cache_key']`. To drop it, set " "`litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n" ), llm_provider=self.custom_llm_provider, diff --git a/litellm/main.py b/litellm/main.py index 38591441e01..e658c9dfe17 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5951,6 +5951,7 @@ def completion( PROVIDER_AFFINITY_HEADER_KWARG_KEY, "fireworks_forward_user_id", "responses_api_version", + "allowed_openai_params", ) if key in kwargs }, diff --git a/litellm/types/llms/mistral.py b/litellm/types/llms/mistral.py index f6068d6d0a5..51fad11edd2 100644 --- a/litellm/types/llms/mistral.py +++ b/litellm/types/llms/mistral.py @@ -27,3 +27,4 @@ class MistralThinkingBlock(TypedDict): class MistralRequestRouting(TypedDict, total=False): api_base: ReadOnly[object] drop_params: ReadOnly[object] + allowed_openai_params: ReadOnly[object] diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_prompt_cache_key.py b/tests/unit/llms/anthropic/pass_through/adapters/test_handler_prompt_cache_key.py index ec8c30c9d37..b000863db4e 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_prompt_cache_key.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_handler_prompt_cache_key.py @@ -95,3 +95,21 @@ def test_prepare_completion_kwargs_derives_prompt_cache_key_only_for_a_mistral_h assert completion_kwargs["user"] == CLAUDE_CODE_USER_ID assert ("prompt_cache_key" in completion_kwargs) is derives_key + + +@pytest.mark.parametrize( + ("codestral_api_base_env", "derives_key"), + [(None, True), ("https://my-resource.services.ai.azure.com/openai/v1", False)], +) +def test_prepare_completion_kwargs_derives_prompt_cache_key_for_codestral_only_on_its_mistral_host( + monkeypatch: pytest.MonkeyPatch, codestral_api_base_env: str | None, derives_key: bool +): + monkeypatch.delenv("MISTRAL_AZURE_API_BASE", raising=False) + if codestral_api_base_env is None: + monkeypatch.delenv("CODESTRAL_API_BASE", raising=False) + else: + monkeypatch.setenv("CODESTRAL_API_BASE", codestral_api_base_env) + + completion_kwargs = _prepare("codestral/codestral-latest", {"custom_llm_provider": "codestral"}) + + assert ("prompt_cache_key" in completion_kwargs) is derives_key diff --git a/tests/unit/llms/mistral/test_mistral_chat_transformation.py b/tests/unit/llms/mistral/test_mistral_chat_transformation.py index 1b9568eec84..be847590b0a 100644 --- a/tests/unit/llms/mistral/test_mistral_chat_transformation.py +++ b/tests/unit/llms/mistral/test_mistral_chat_transformation.py @@ -1,9 +1,13 @@ +import json from collections.abc import Mapping from typing import Final, List, cast from unittest.mock import MagicMock, patch from urllib.parse import urlparse +import httpx import pytest +import respx +from openai import AsyncOpenAI from litellm.litellm_core_utils.prompt_templates.common_utils import TOOL_RESULT_IMAGE_BOUNDARY from litellm.llms.mistral.chat.transformation import ( @@ -1195,6 +1199,73 @@ def test_prompt_cache_key_for_another_host_is_rejected_without_drop_params(api_b assert upstream_host not in str(rejection.value) +def test_prompt_cache_key_allowlisted_for_another_purpose_is_still_rejected_for_another_host() -> None: + import litellm + + with pytest.raises(litellm.UnsupportedParamsError, match="allowed_openai_params"): + _transform_with_prompt_cache_key( + {"api_base": AZURE_HOSTED_MISTRAL_API_BASE, "allowed_openai_params": ["reasoning_effort"]} + ) + + +def _chat_completion_reply() -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 0, + "model": "mistral-small-latest", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + + +def test_allowlisted_prompt_cache_key_reaches_a_gateway_in_front_of_mistral(respx_mock: respx.MockRouter) -> None: + import litellm + + gateway_route: Final = respx_mock.post("https://gateway.example.com/mistral/v1/chat/completions").mock( + return_value=_chat_completion_reply() + ) + + litellm.completion( + model="mistral/mistral-small-latest", + messages=[{"role": "user", "content": "hi"}], + api_base="https://gateway.example.com/mistral/v1", + api_key="gateway-key", + prompt_cache_key="session-1", + allowed_openai_params=["prompt_cache_key"], + max_retries=0, + ) + + assert json.loads(gateway_route.calls.last.request.content)["prompt_cache_key"] == "session-1" + + +@pytest.mark.asyncio +async def test_prompt_cache_key_is_dropped_from_an_async_codestral_call_to_another_host( + respx_mock: respx.MockRouter, +) -> None: + import litellm + + azure_route: Final = respx_mock.post(f"{AZURE_HOSTED_MISTRAL_API_BASE}/chat/completions").mock( + return_value=_chat_completion_reply() + ) + + await litellm.acompletion( + model="codestral/codestral-latest", + messages=[{"role": "user", "content": "hi"}], + api_base=AZURE_HOSTED_MISTRAL_API_BASE, + api_key="azure-key", + prompt_cache_key="session-1", + drop_params=True, + max_retries=0, + client=AsyncOpenAI(api_key="azure-key", base_url=AZURE_HOSTED_MISTRAL_API_BASE, http_client=httpx.AsyncClient()), + ) + + assert "prompt_cache_key" not in json.loads(azure_route.calls.last.request.content) + + def test_prompt_cache_key_for_another_host_is_dropped_under_drop_params() -> None: request: Final = _transform_with_prompt_cache_key({"api_base": AZURE_HOSTED_MISTRAL_API_BASE, "drop_params": True}) From 1e6f423012b56740fc03955cc7eaf1ce9cf172e9 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:35:32 -0700 Subject: [PATCH 5/8] test(mistral): annotate the new adapter prompt_cache_key tests --- .../adapters/test_handler_prompt_cache_key.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_prompt_cache_key.py b/tests/unit/llms/anthropic/pass_through/adapters/test_handler_prompt_cache_key.py index b000863db4e..7baee037138 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_prompt_cache_key.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_handler_prompt_cache_key.py @@ -1,6 +1,7 @@ import json import os import sys +from typing import Final import pytest @@ -83,13 +84,13 @@ def test_prepare_completion_kwargs_keeps_prompt_cache_key_through_responses_rero ) def test_prepare_completion_kwargs_derives_prompt_cache_key_only_for_a_mistral_hosted_endpoint( monkeypatch: pytest.MonkeyPatch, api_base: str | None, azure_api_base_env: str | None, derives_key: bool -): +) -> None: if azure_api_base_env is None: monkeypatch.delenv("MISTRAL_AZURE_API_BASE", raising=False) else: monkeypatch.setenv("MISTRAL_AZURE_API_BASE", azure_api_base_env) - completion_kwargs = _prepare( + completion_kwargs: Final = _prepare( "mistral/mistral-small-latest", {"custom_llm_provider": "mistral", "api_base": api_base} ) @@ -103,13 +104,13 @@ def test_prepare_completion_kwargs_derives_prompt_cache_key_only_for_a_mistral_h ) def test_prepare_completion_kwargs_derives_prompt_cache_key_for_codestral_only_on_its_mistral_host( monkeypatch: pytest.MonkeyPatch, codestral_api_base_env: str | None, derives_key: bool -): +) -> None: monkeypatch.delenv("MISTRAL_AZURE_API_BASE", raising=False) if codestral_api_base_env is None: monkeypatch.delenv("CODESTRAL_API_BASE", raising=False) else: monkeypatch.setenv("CODESTRAL_API_BASE", codestral_api_base_env) - completion_kwargs = _prepare("codestral/codestral-latest", {"custom_llm_provider": "codestral"}) + completion_kwargs: Final = _prepare("codestral/codestral-latest", {"custom_llm_provider": "codestral"}) assert ("prompt_cache_key" in completion_kwargs) is derives_key From 981f0e37d17688c03b240ae32475802aa295e890 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:21:20 -0700 Subject: [PATCH 6/8] fix(mistral): drop or reject a prompt_cache_key the Mistral API cannot take A non-ASCII prompt_cache_key gets a 500 from api.mistral.ai, and a non-string one a 422, so both now drop under drop_params and answer 400 without it, like any value the provider cannot take. The /v1/messages adapter derives no key from a non-ASCII Claude Code session for Mistral, and the 400 names the caller's provider, so a codestral deployment's error says codestral. --- .../pass_through/adapters/transformation.py | 11 +++- litellm/llms/mistral/chat/transformation.py | 40 +++++++++--- litellm/types/llms/mistral.py | 5 ++ .../adapters/test_handler_prompt_cache_key.py | 24 ++++++++ .../test_mistral_chat_transformation.py | 61 ++++++++++++++++++- 5 files changed, 127 insertions(+), 14 deletions(-) diff --git a/litellm/llms/anthropic/pass_through/adapters/transformation.py b/litellm/llms/anthropic/pass_through/adapters/transformation.py index 5c81514b5c8..22c2ac03995 100644 --- a/litellm/llms/anthropic/pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/adapters/transformation.py @@ -990,7 +990,9 @@ class LiteLLMAnthropicMessagesAdapter: ) @staticmethod - def _supports_prompt_cache_key(model: str | None, custom_llm_provider: str | None, api_base: str | None) -> bool: + def _supports_prompt_cache_key( + prompt_cache_key: str, model: str | None, custom_llm_provider: str | None, api_base: str | None + ) -> bool: if not model or not custom_llm_provider: return False if custom_llm_provider in PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND: @@ -1005,7 +1007,10 @@ class LiteLLMAnthropicMessagesAdapter: _, _, _, resolved_api_base = litellm.get_llm_provider( model=model, custom_llm_provider=custom_llm_provider, api_base=api_base ) - return litellm.MistralConfig().accepts_prompt_cache_key_at(resolved_api_base) + mistral_config: Final = litellm.MistralConfig() + return mistral_config.takes_prompt_cache_key_value( + prompt_cache_key + ) and mistral_config.accepts_prompt_cache_key_at(resolved_api_base) @staticmethod def _target_declares_reasoning_effort(model: str, custom_llm_provider: str | None) -> bool: @@ -1047,7 +1052,7 @@ class LiteLLMAnthropicMessagesAdapter: new_kwargs["user"] = metadata["user_id"] prompt_cache_key: Final = prompt_cache_key_from_user_id(metadata["user_id"]) if prompt_cache_key is not None and self._supports_prompt_cache_key( - anthropic_message_request.get("model"), custom_llm_provider, api_base + prompt_cache_key, anthropic_message_request.get("model"), custom_llm_provider, api_base ): new_kwargs["prompt_cache_key"] = prompt_cache_key diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index c0fcd6b37e0..18a33376f86 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -32,7 +32,12 @@ from litellm.router_utils.reasoning_effort_capability import ( nearest_declared_reasoning_effort, ) from litellm.secret_managers.main import get_secret_str -from litellm.types.llms.mistral import MistralRequestRouting, MistralThinkingBlock, MistralToolCallMessage +from litellm.types.llms.mistral import ( + MistralCacheKeyParams, + MistralRequestRouting, + MistralThinkingBlock, + MistralToolCallMessage, +) from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse, ModelResponseStream from litellm.utils import convert_to_model_response_object, supports_reasoning @@ -60,6 +65,7 @@ def _accepted_reasoning_effort(model: str, requested: str, custom_llm_provider: _NO_TOOL_CALL_TURN: Final[tuple[int, frozenset[str]]] = (-1, frozenset()) _MISTRAL_REQUEST_ROUTING: Final = TypeAdapter(MistralRequestRouting) +_MISTRAL_CACHE_KEY_PARAMS: Final = TypeAdapter(MistralCacheKeyParams) class MistralConfig(OpenAIGPTConfig): @@ -225,24 +231,37 @@ class MistralConfig(OpenAIGPTConfig): hostname: Final = urlparse(resolved_api_base).hostname or "" return hostname == "mistral.ai" or hostname.endswith(".mistral.ai") - def _prompt_cache_key_withheld(self, model: str, routing: MistralRequestRouting) -> bool: + @staticmethod + def takes_prompt_cache_key_value(prompt_cache_key: object) -> bool: + return isinstance(prompt_cache_key, str) and prompt_cache_key.isascii() + + def _prompt_cache_key_rejection(self, prompt_cache_key: object, api_base: str | None) -> str | None: + if not self.accepts_prompt_cache_key_at(api_base): + return "since only the Mistral API (*.mistral.ai) takes it" + if not self.takes_prompt_cache_key_value(prompt_cache_key): + return "since the Mistral API takes it only as a string of ASCII characters" + return None + + def _prompt_cache_key_withheld(self, model: str, prompt_cache_key: object, routing: MistralRequestRouting) -> bool: requested_api_base: Final = routing.get("api_base") api_base: Final = requested_api_base if isinstance(requested_api_base, str) else None allowed_openai_params: Final = routing.get("allowed_openai_params") if isinstance(allowed_openai_params, (list, tuple)) and "prompt_cache_key" in allowed_openai_params: return False - if self.accepts_prompt_cache_key_at(api_base): + rejection: Final = self._prompt_cache_key_rejection(prompt_cache_key, api_base) + if rejection is None: return False if litellm.drop_params is True or routing.get("drop_params") is True: return True + requested_provider: Final = routing.get("custom_llm_provider") + provider: Final = requested_provider if isinstance(requested_provider, str) else self.custom_llm_provider raise UnsupportedParamsError( message=( - f"{self.custom_llm_provider} does not support parameters: ['prompt_cache_key'], for model={model}, " - "since only the Mistral API (*.mistral.ai) takes it. To send it anyway, pass " - "`allowed_openai_params=['prompt_cache_key']`. To drop it, set " + f"{provider} does not support parameters: ['prompt_cache_key'], for model={model}, {rejection}. " + "To send it anyway, pass `allowed_openai_params=['prompt_cache_key']`. To drop it, set " "`litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n" ), - llm_provider=self.custom_llm_provider, + llm_provider=provider, model=model, ) @@ -668,10 +687,13 @@ class MistralConfig(OpenAIGPTConfig): if "magistral" in model.lower() and optional_params.get("_add_reasoning_prompt", False): messages = self._add_reasoning_system_prompt_if_needed(messages, optional_params) + cache_key_params: Final = _MISTRAL_CACHE_KEY_PARAMS.validate_python(optional_params) withheld_params: Final = ( ("client_metadata", "prompt_cache_key") - if "prompt_cache_key" in optional_params - and self._prompt_cache_key_withheld(model, _MISTRAL_REQUEST_ROUTING.validate_python(litellm_params)) + if "prompt_cache_key" in cache_key_params + and self._prompt_cache_key_withheld( + model, cache_key_params["prompt_cache_key"], _MISTRAL_REQUEST_ROUTING.validate_python(litellm_params) + ) else ("client_metadata",) ) upstream_params: Final = {key: value for key, value in optional_params.items() if key not in withheld_params} diff --git a/litellm/types/llms/mistral.py b/litellm/types/llms/mistral.py index 51fad11edd2..364e54a9937 100644 --- a/litellm/types/llms/mistral.py +++ b/litellm/types/llms/mistral.py @@ -28,3 +28,8 @@ class MistralRequestRouting(TypedDict, total=False): api_base: ReadOnly[object] drop_params: ReadOnly[object] allowed_openai_params: ReadOnly[object] + custom_llm_provider: ReadOnly[object] + + +class MistralCacheKeyParams(TypedDict, total=False): + prompt_cache_key: ReadOnly[object] diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_prompt_cache_key.py b/tests/unit/llms/anthropic/pass_through/adapters/test_handler_prompt_cache_key.py index 7baee037138..e15ee361067 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_prompt_cache_key.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_handler_prompt_cache_key.py @@ -114,3 +114,27 @@ def test_prepare_completion_kwargs_derives_prompt_cache_key_for_codestral_only_o completion_kwargs: Final = _prepare("codestral/codestral-latest", {"custom_llm_provider": "codestral"}) assert ("prompt_cache_key" in completion_kwargs) is derives_key + + +def test_prepare_completion_kwargs_derives_no_prompt_cache_key_mistral_cannot_take_from_a_non_ascii_session() -> None: + non_ascii_user_id: Final = json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": "sitzung-ü"}) + + mistral_kwargs: Final = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( + max_tokens=1024, + messages=MESSAGES, + model="mistral/mistral-small-latest", + metadata={"user_id": non_ascii_user_id}, + thinking=None, + extra_kwargs={"custom_llm_provider": "mistral"}, + )[0] + openai_kwargs: Final = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( + max_tokens=1024, + messages=MESSAGES, + model="openai/gpt-5.6-luna", + metadata={"user_id": non_ascii_user_id}, + thinking=None, + extra_kwargs={"custom_llm_provider": "openai"}, + )[0] + + assert "prompt_cache_key" not in mistral_kwargs + assert openai_kwargs["prompt_cache_key"] == "sitzung-ü" diff --git a/tests/unit/llms/mistral/test_mistral_chat_transformation.py b/tests/unit/llms/mistral/test_mistral_chat_transformation.py index be847590b0a..9a8d0143f6a 100644 --- a/tests/unit/llms/mistral/test_mistral_chat_transformation.py +++ b/tests/unit/llms/mistral/test_mistral_chat_transformation.py @@ -1168,13 +1168,15 @@ def test_prompt_cache_key_reaches_the_mistral_request_body( AZURE_HOSTED_MISTRAL_API_BASE: Final = "https://my-resource.services.ai.azure.com/openai/v1" -def _transform_with_prompt_cache_key(litellm_params: dict[str, object]) -> Mapping[str, object]: +def _transform_with_prompt_cache_key( + litellm_params: dict[str, object], prompt_cache_key: object = "session-1" +) -> Mapping[str, object]: return cast( Mapping[str, object], MistralConfig().transform_request( model="mistral-small-latest", messages=[{"role": "user", "content": "hi"}], - optional_params={"prompt_cache_key": "session-1", "temperature": 0.2}, + optional_params={"prompt_cache_key": prompt_cache_key, "temperature": 0.2}, litellm_params=litellm_params, headers={}, ), @@ -1199,6 +1201,35 @@ def test_prompt_cache_key_for_another_host_is_rejected_without_drop_params(api_b assert upstream_host not in str(rejection.value) +UNSENDABLE_PROMPT_CACHE_KEYS: Final = ("Büro-Sitzung-42", "日本", 123, ["session-1"]) + + +@pytest.mark.parametrize("prompt_cache_key", UNSENDABLE_PROMPT_CACHE_KEYS) +def test_prompt_cache_key_the_mistral_api_cannot_take_is_rejected_without_drop_params( + prompt_cache_key: object, +) -> None: + import litellm + + with pytest.raises(litellm.UnsupportedParamsError, match="ASCII"): + _transform_with_prompt_cache_key({"api_base": "https://api.mistral.ai/v1"}, prompt_cache_key) + + +@pytest.mark.parametrize("prompt_cache_key", UNSENDABLE_PROMPT_CACHE_KEYS) +def test_prompt_cache_key_the_mistral_api_cannot_take_is_dropped_under_drop_params(prompt_cache_key: object) -> None: + request: Final = _transform_with_prompt_cache_key( + {"api_base": "https://api.mistral.ai/v1", "drop_params": True}, prompt_cache_key + ) + + assert "prompt_cache_key" not in request + assert request["temperature"] == 0.2 + + +def test_prompt_cache_key_with_ascii_punctuation_reaches_the_mistral_api() -> None: + request: Final = _transform_with_prompt_cache_key({}, "team a/session:42") + + assert request["prompt_cache_key"] == "team a/session:42" + + def test_prompt_cache_key_allowlisted_for_another_purpose_is_still_rejected_for_another_host() -> None: import litellm @@ -1266,6 +1297,32 @@ async def test_prompt_cache_key_is_dropped_from_an_async_codestral_call_to_anoth assert "prompt_cache_key" not in json.loads(azure_route.calls.last.request.content) +@pytest.mark.asyncio +async def test_prompt_cache_key_rejection_for_an_async_codestral_call_names_codestral( + respx_mock: respx.MockRouter, +) -> None: + import litellm + + azure_route: Final = respx_mock.post(f"{AZURE_HOSTED_MISTRAL_API_BASE}/chat/completions").mock( + return_value=_chat_completion_reply() + ) + + with pytest.raises(litellm.BadRequestError, match=r"codestral does not support parameters: \['prompt_cache_key'\]"): + await litellm.acompletion( + model="codestral/codestral-latest", + messages=[{"role": "user", "content": "hi"}], + api_base=AZURE_HOSTED_MISTRAL_API_BASE, + api_key="azure-key", + prompt_cache_key="session-1", + max_retries=0, + client=AsyncOpenAI( + api_key="azure-key", base_url=AZURE_HOSTED_MISTRAL_API_BASE, http_client=httpx.AsyncClient() + ), + ) + + assert not azure_route.called + + def test_prompt_cache_key_for_another_host_is_dropped_under_drop_params() -> None: request: Final = _transform_with_prompt_cache_key({"api_base": AZURE_HOSTED_MISTRAL_API_BASE, "drop_params": True}) From 68f1b98b3cc944ba8e84ae518f050346e6ff9f08 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 19 Oct 2026 22:40:28 -0700 Subject: [PATCH 7/8] fix(mistral): send a reasoning_effort object as its level and keep UnsupportedParamsError on sync calls --- litellm/llms/mistral/chat/transformation.py | 44 +++++++++++++- litellm/llms/openai/openai.py | 5 +- litellm/types/llms/mistral.py | 4 ++ .../test_mistral_chat_transformation.py | 58 ++++++++++++++++++- 4 files changed, 106 insertions(+), 5 deletions(-) diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index 18a33376f86..236308ab7b5 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -17,6 +17,7 @@ from pydantic import TypeAdapter import litellm from litellm._logging import verbose_logger from litellm.exceptions import UnsupportedParamsError +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.prompt_templates.common_utils import ( handle_messages_with_content_list_to_str_conversion, @@ -34,6 +35,7 @@ from litellm.router_utils.reasoning_effort_capability import ( from litellm.secret_managers.main import get_secret_str from litellm.types.llms.mistral import ( MistralCacheKeyParams, + MistralReasoningEffortParams, MistralRequestRouting, MistralThinkingBlock, MistralToolCallMessage, @@ -66,6 +68,14 @@ def _accepted_reasoning_effort(model: str, requested: str, custom_llm_provider: _NO_TOOL_CALL_TURN: Final[tuple[int, frozenset[str]]] = (-1, frozenset()) _MISTRAL_REQUEST_ROUTING: Final = TypeAdapter(MistralRequestRouting) _MISTRAL_CACHE_KEY_PARAMS: Final = TypeAdapter(MistralCacheKeyParams) +_MISTRAL_REASONING_EFFORT_PARAMS: Final = TypeAdapter(MistralReasoningEffortParams) + + +def _reasoning_level_and_summary(reasoning_effort: object) -> tuple[object, object]: + requested: Final = as_str_mapping(reasoning_effort) + if requested is None: + return reasoning_effort, None + return requested.get("effort"), requested.get("summary") class MistralConfig(OpenAIGPTConfig): @@ -217,8 +227,17 @@ class MistralConfig(OpenAIGPTConfig): optional_params["extra_body"] = {"random_seed": value} if param == "response_format": optional_params["response_format"] = value - if param == "reasoning_effort" and "magistral" not in model.lower(): - optional_params["reasoning_effort"] = _accepted_reasoning_effort(model, value, self.custom_llm_provider) + if ( + param == "reasoning_effort" + and "magistral" not in model.lower() + and ( + reasoning_level := self._reasoning_level_to_send( + model, _MISTRAL_REASONING_EFFORT_PARAMS.validate_python(non_default_params), drop_params + ) + ) + is not None + ): + optional_params["reasoning_effort"] = reasoning_level if param in ("reasoning_effort", "thinking") and "magistral" in model.lower(): # Flag that we need to add reasoning system prompt optional_params["_add_reasoning_prompt"] = True @@ -226,6 +245,27 @@ class MistralConfig(OpenAIGPTConfig): optional_params[param] = value return optional_params + def _reasoning_level_to_send( + self, model: str, requested: MistralReasoningEffortParams, drop_params: bool + ) -> str | None: + level, summary = _reasoning_level_and_summary(requested.get("reasoning_effort")) + sendable: Final = ( + _accepted_reasoning_effort(model, level, self.custom_llm_provider) if isinstance(level, str) else None + ) + if sendable is not None and summary is None: + return sendable + if drop_params or litellm.drop_params is True: + return sendable + raise UnsupportedParamsError( + message=( + f"{self.custom_llm_provider} does not support parameters: ['reasoning_effort'], for model={model}, " + "since the Mistral API takes it only as a level such as 'high', with no reasoning summary. " + "To drop it, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n" + ), + llm_provider=self.custom_llm_provider, + model=model, + ) + def accepts_prompt_cache_key_at(self, api_base: str | None) -> bool: resolved_api_base, _ = self._get_openai_compatible_provider_info(api_base, None) hostname: Final = urlparse(resolved_api_base).hostname or "" diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 73f40f5e0e4..4e8009eeace 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -27,6 +27,7 @@ import litellm from litellm import LlmProviders from litellm._logging import verbose_logger from litellm.constants import DEFAULT_MAX_RETRIES +from litellm.exceptions import UnsupportedParamsError from litellm.files.types import FileContentStreamingResult from litellm.litellm_core_utils.core_helpers import set_provider_response_headers_in_hidden_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -942,8 +943,8 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): litellm.remove_index_from_tool_calls(messages=messages) else: raise e - except OpenAIError as e: - raise e + except (OpenAIError, UnsupportedParamsError): + raise except Exception as e: status_code: Final = getattr(e, "status_code", 500) error_headers = getattr(e, "headers", None) diff --git a/litellm/types/llms/mistral.py b/litellm/types/llms/mistral.py index 364e54a9937..4c0fd9c0aee 100644 --- a/litellm/types/llms/mistral.py +++ b/litellm/types/llms/mistral.py @@ -33,3 +33,7 @@ class MistralRequestRouting(TypedDict, total=False): class MistralCacheKeyParams(TypedDict, total=False): prompt_cache_key: ReadOnly[object] + + +class MistralReasoningEffortParams(TypedDict, total=False): + reasoning_effort: ReadOnly[object] diff --git a/tests/unit/llms/mistral/test_mistral_chat_transformation.py b/tests/unit/llms/mistral/test_mistral_chat_transformation.py index 9a8d0143f6a..b7f892e9ce5 100644 --- a/tests/unit/llms/mistral/test_mistral_chat_transformation.py +++ b/tests/unit/llms/mistral/test_mistral_chat_transformation.py @@ -115,6 +115,36 @@ class TestMistralReasoningSupport: ) assert optional_params["reasoning_effort"] == sent + @pytest.mark.parametrize( + ("requested", "drop_params", "sent"), + [ + ({"effort": "medium"}, False, {"reasoning_effort": "high"}), + ({"effort": "high", "summary": "auto"}, True, {"reasoning_effort": "high"}), + ({"summary": "auto"}, True, {}), + ], + ) + def test_reasoning_effort_object_is_sent_as_its_level(self, local_model_cost_map, requested, drop_params, sent): + import litellm + + optional_params = litellm.get_optional_params( + model="mistral-small-latest", + custom_llm_provider="mistral", + reasoning_effort=requested, + drop_params=drop_params, + ) + assert {key: optional_params[key] for key in ("reasoning_effort",) if key in optional_params} == sent + + @pytest.mark.parametrize("requested", [{"effort": "high", "summary": "auto"}, {"summary": "auto"}]) + def test_reasoning_effort_object_mistral_cannot_take_is_rejected(self, local_model_cost_map, requested): + import litellm + + with pytest.raises(litellm.UnsupportedParamsError, match=r"\['reasoning_effort'\]"): + litellm.get_optional_params( + model="devstral-latest", + custom_llm_provider="mistral", + reasoning_effort=requested, + ) + def test_reasoning_effort_is_forwarded_verbatim_when_the_map_declares_no_levels( self, local_model_cost_map, monkeypatch ): @@ -1291,7 +1321,9 @@ async def test_prompt_cache_key_is_dropped_from_an_async_codestral_call_to_anoth prompt_cache_key="session-1", drop_params=True, max_retries=0, - client=AsyncOpenAI(api_key="azure-key", base_url=AZURE_HOSTED_MISTRAL_API_BASE, http_client=httpx.AsyncClient()), + client=AsyncOpenAI( + api_key="azure-key", base_url=AZURE_HOSTED_MISTRAL_API_BASE, http_client=httpx.AsyncClient() + ), ) assert "prompt_cache_key" not in json.loads(azure_route.calls.last.request.content) @@ -1323,6 +1355,30 @@ async def test_prompt_cache_key_rejection_for_an_async_codestral_call_names_code assert not azure_route.called +def test_prompt_cache_key_rejection_for_a_sync_codestral_call_is_an_unsupported_params_error( + respx_mock: respx.MockRouter, +) -> None: + import litellm + + azure_route: Final = respx_mock.post(f"{AZURE_HOSTED_MISTRAL_API_BASE}/chat/completions").mock( + return_value=_chat_completion_reply() + ) + + with pytest.raises( + litellm.UnsupportedParamsError, match=r"^litellm.UnsupportedParamsError: codestral does not support" + ): + litellm.completion( + model="codestral/codestral-latest", + messages=[{"role": "user", "content": "hi"}], + api_base=AZURE_HOSTED_MISTRAL_API_BASE, + api_key="azure-key", + prompt_cache_key="session-1", + max_retries=0, + ) + + assert not azure_route.called + + def test_prompt_cache_key_for_another_host_is_dropped_under_drop_params() -> None: request: Final = _transform_with_prompt_cache_key({"api_base": AZURE_HOSTED_MISTRAL_API_BASE, "drop_params": True}) From 4ef06b92a72627aea458a586ff7c44d4a22d62e5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 01:21:07 -0700 Subject: [PATCH 8/8] fix(mistral): keep async codestral file uploads working by not rewriting the caller's file blocks --- litellm/llms/mistral/chat/transformation.py | 37 ++++++++++++------- litellm/types/llms/mistral.py | 4 ++ .../test_mistral_chat_transformation.py | 30 +++++++++++++++ 3 files changed, 57 insertions(+), 14 deletions(-) diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index 236308ab7b5..abb81b11249 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -35,6 +35,7 @@ from litellm.router_utils.reasoning_effort_capability import ( from litellm.secret_managers.main import get_secret_str from litellm.types.llms.mistral import ( MistralCacheKeyParams, + MistralFilePart, MistralReasoningEffortParams, MistralRequestRouting, MistralThinkingBlock, @@ -69,6 +70,7 @@ _NO_TOOL_CALL_TURN: Final[tuple[int, frozenset[str]]] = (-1, frozenset()) _MISTRAL_REQUEST_ROUTING: Final = TypeAdapter(MistralRequestRouting) _MISTRAL_CACHE_KEY_PARAMS: Final = TypeAdapter(MistralCacheKeyParams) _MISTRAL_REASONING_EFFORT_PARAMS: Final = TypeAdapter(MistralReasoningEffortParams) +_MISTRAL_FILE_PART: Final = TypeAdapter(MistralFilePart) def _reasoning_level_and_summary(reasoning_effort: object) -> tuple[object, object]: @@ -438,24 +440,31 @@ class MistralConfig(OpenAIGPTConfig): messages = self._handle_message_with_file(messages) return messages + @staticmethod + def _with_mistral_file_id(block: Mapping[str, object]) -> Mapping[str, object]: + file_part: Final = block.get("file") if block.get("type") == "file" else None + file_id: Final = ( + _MISTRAL_FILE_PART.validate_python(file_part).get("file_id") if isinstance(file_part, Mapping) else None + ) + if not file_id: + return block + return {**{key: value for key, value in block.items() if key != "file"}, "file_id": file_id} + + @classmethod + def _with_mistral_file_ids(cls, message: AllMessageValues) -> AllMessageValues: + content: Final = message.get("content") + if not isinstance(content, list): + return message + blocks: Final[tuple[Mapping[str, object], ...]] = tuple(content) + return cast( # cast-ok: only file blocks change, to the file_id shape the Mistral API takes + AllMessageValues, {**message, "content": [cls._with_mistral_file_id(block) for block in blocks]} + ) + def _handle_message_with_file(self, messages: list[AllMessageValues]) -> list[AllMessageValues]: """ Mistral API supports only 'file_id' in message content with type 'file'. """ - for m in messages: - _content_block = m.get("content") - if _content_block and isinstance(_content_block, list): - if any(c.get("type") == "file" for c in _content_block): - # If file content is present, we get file_id from 'file' attribute of content block - # then replace 'file' with 'file_id' and assign the value of 'file_id' attribute to it. - file_contents = [c for c in _content_block if c.get("type") == "file"] - for file_content in file_contents: - file_id = file_content.get("file", {}).get("file_id") - if file_id: - # Replace 'file' with 'file_id' - file_content["file_id"] = file_id # pyright: ignore[reportGeneralTypeIssues] # legacy in-place rewrite of the block shape - file_content.pop("file", None) - return messages + return [self._with_mistral_file_ids(m) for m in messages] def _add_reasoning_system_prompt_if_needed( self, messages: list[AllMessageValues], optional_params: dict diff --git a/litellm/types/llms/mistral.py b/litellm/types/llms/mistral.py index 4c0fd9c0aee..03e31af6ffc 100644 --- a/litellm/types/llms/mistral.py +++ b/litellm/types/llms/mistral.py @@ -37,3 +37,7 @@ class MistralCacheKeyParams(TypedDict, total=False): class MistralReasoningEffortParams(TypedDict, total=False): reasoning_effort: ReadOnly[object] + + +class MistralFilePart(TypedDict, total=False): + file_id: ReadOnly[object] diff --git a/tests/unit/llms/mistral/test_mistral_chat_transformation.py b/tests/unit/llms/mistral/test_mistral_chat_transformation.py index b7f892e9ce5..3a78a28165c 100644 --- a/tests/unit/llms/mistral/test_mistral_chat_transformation.py +++ b/tests/unit/llms/mistral/test_mistral_chat_transformation.py @@ -1355,6 +1355,36 @@ async def test_prompt_cache_key_rejection_for_an_async_codestral_call_names_code assert not azure_route.called +@pytest.mark.asyncio +async def test_async_codestral_call_sends_an_uploaded_file_by_its_id(respx_mock: respx.MockRouter) -> None: + import litellm + + codestral_api_base: Final = "https://codestral.mistral.ai/v1" + codestral_route: Final = respx_mock.post(f"{codestral_api_base}/chat/completions").mock( + return_value=_chat_completion_reply() + ) + + await litellm.acompletion( + model="codestral/codestral-latest", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "Review this file"}, + {"type": "file", "file": {"file_id": "file-12345"}}, + ], + } + ], + api_base=codestral_api_base, + api_key="codestral-key", + max_retries=0, + client=AsyncOpenAI(api_key="codestral-key", base_url=codestral_api_base, http_client=httpx.AsyncClient()), + ) + + sent_content: Final = json.loads(codestral_route.calls.last.request.content)["messages"][0]["content"] + assert sent_content[1] == {"type": "file", "file_id": "file-12345"} + + def test_prompt_cache_key_rejection_for_a_sync_codestral_call_is_an_unsupported_params_error( respx_mock: respx.MockRouter, ) -> None: