diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index 657fd3abfbc..b3ccf612a0e 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -80,6 +80,7 @@ OPTIONAL_KWARGS_KEYS: Final = ( "github_copilot_user_session", "fireworks_forward_user_id", "responses_api_version", + "allowed_openai_params", PROVIDER_AFFINITY_HEADER_KWARG_KEY, } ) diff --git a/litellm/llms/anthropic/pass_through/adapters/handler.py b/litellm/llms/anthropic/pass_through/adapters/handler.py index 01e781485a0..a241e8d31d5 100644 --- a/litellm/llms/anthropic/pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/adapters/handler.py @@ -646,12 +646,14 @@ class LiteLLMMessagesToCompletionTransformationHandler: request_data["output_config"] = extra_kwargs["output_config"] custom_llm_provider: Final = extra_kwargs.get("custom_llm_provider") + api_base: Final = extra_kwargs.get("api_base") ( openai_request, tool_name_mapping, ) = ANTHROPIC_ADAPTER.translate_completion_input_params_with_tool_mapping( request_data, custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None, + api_base=api_base if isinstance(api_base, str) else None, drop_params=drops_unsupported_params(extra_kwargs.get("drop_params")), ) diff --git a/litellm/llms/anthropic/pass_through/adapters/transformation.py b/litellm/llms/anthropic/pass_through/adapters/transformation.py index 9b1297dbd75..18db57139fd 100644 --- a/litellm/llms/anthropic/pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/adapters/transformation.py @@ -24,6 +24,7 @@ OPENAI_MAX_TOOL_NAME_LENGTH: Final = 64 TOOL_NAME_HASH_LENGTH: Final = 8 TOOL_NAME_PREFIX_LENGTH: Final = OPENAI_MAX_TOOL_NAME_LENGTH - TOOL_NAME_HASH_LENGTH - 1 # 55 PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND: Final = frozenset({"litellm_proxy"}) +PROVIDERS_SERVED_BY_THE_MISTRAL_CHAT_CONFIG: Final = frozenset({"mistral", "codestral"}) _COMPACTION_BLOCK: Final = TypeAdapter(dict[str, JsonValue]) @@ -252,7 +253,12 @@ class AnthropicAdapter: return result def translate_completion_input_params_with_tool_mapping( - self, kwargs, *, custom_llm_provider: str | None = None, drop_params: bool = True + self, + kwargs, + *, + custom_llm_provider: str | None = None, + api_base: str | None = None, + drop_params: bool = True, ) -> tuple[ChatCompletionRequest | None, dict[str, str]]: """ Translate Anthropic request params to OpenAI format, returning tool name mapping. @@ -286,6 +292,7 @@ class AnthropicAdapter: ) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( anthropic_message_request=request_body, custom_llm_provider=custom_llm_provider, + api_base=api_base, drop_params=drop_params, ) @@ -995,7 +1002,9 @@ class LiteLLMAnthropicMessagesAdapter: ) @staticmethod - def _supports_prompt_cache_key(model: str | None, custom_llm_provider: str | None) -> bool: + def _supports_prompt_cache_key( + prompt_cache_key: str, model: str | None, custom_llm_provider: str | None, api_base: str | None + ) -> bool: if not model or not custom_llm_provider: return False if custom_llm_provider in PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND: @@ -1003,7 +1012,17 @@ class LiteLLMAnthropicMessagesAdapter: supported_params: Final = litellm.get_supported_openai_params( model=model, custom_llm_provider=custom_llm_provider ) - return "prompt_cache_key" in (supported_params or ()) + if "prompt_cache_key" not in (supported_params or ()): + return False + if custom_llm_provider not in PROVIDERS_SERVED_BY_THE_MISTRAL_CHAT_CONFIG: + return True + _, _, _, resolved_api_base = litellm.get_llm_provider( + model=model, custom_llm_provider=custom_llm_provider, api_base=api_base + ) + mistral_config: Final = litellm.MistralConfig() + return mistral_config.takes_prompt_cache_key_value( + prompt_cache_key + ) and mistral_config.accepts_prompt_cache_key_at(resolved_api_base) @staticmethod def _target_declares_reasoning_effort(model: str, custom_llm_provider: str | None) -> bool: @@ -1036,6 +1055,7 @@ class LiteLLMAnthropicMessagesAdapter: new_kwargs: ChatCompletionRequest, *, custom_llm_provider: str | None = None, + api_base: str | None = None, ) -> None: """Translate metadata fields from Anthropic request to OpenAI request.""" if "metadata" in anthropic_message_request: @@ -1044,7 +1064,7 @@ class LiteLLMAnthropicMessagesAdapter: new_kwargs["user"] = metadata["user_id"] prompt_cache_key: Final = prompt_cache_key_from_user_id(metadata["user_id"]) if prompt_cache_key is not None and self._supports_prompt_cache_key( - anthropic_message_request.get("model"), custom_llm_provider + prompt_cache_key, anthropic_message_request.get("model"), custom_llm_provider, api_base ): new_kwargs["prompt_cache_key"] = prompt_cache_key @@ -1235,6 +1255,7 @@ class LiteLLMAnthropicMessagesAdapter: anthropic_message_request: AnthropicMessagesRequest, *, custom_llm_provider: str | None = None, + api_base: str | None = None, preserve_midturn_system: bool = False, drop_params: bool = True, ) -> tuple[ChatCompletionRequest, dict[str, str]]: @@ -1273,6 +1294,7 @@ class LiteLLMAnthropicMessagesAdapter: anthropic_message_request=anthropic_message_request, new_kwargs=new_kwargs, custom_llm_provider=custom_llm_provider, + api_base=api_base, ) ## CONVERT TOOL CHOICE self._translate_tool_choice_to_openai( diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index b8a6cb1beb9..abb81b11249 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -9,10 +9,15 @@ Docs - https://docs.mistral.ai/api/ from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping from itertools import accumulate, count from typing import TYPE_CHECKING, Final, Literal, cast, get_type_hints, overload +from urllib.parse import urlparse import httpx +from pydantic import TypeAdapter +import litellm from litellm._logging import verbose_logger +from litellm.exceptions import UnsupportedParamsError +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.prompt_templates.common_utils import ( handle_messages_with_content_list_to_str_conversion, @@ -28,7 +33,14 @@ from litellm.router_utils.reasoning_effort_capability import ( nearest_declared_reasoning_effort, ) from litellm.secret_managers.main import get_secret_str -from litellm.types.llms.mistral import MistralThinkingBlock, MistralToolCallMessage +from litellm.types.llms.mistral import ( + MistralCacheKeyParams, + MistralFilePart, + MistralReasoningEffortParams, + MistralRequestRouting, + MistralThinkingBlock, + MistralToolCallMessage, +) from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse, ModelResponseStream from litellm.utils import convert_to_model_response_object, supports_reasoning @@ -55,6 +67,17 @@ def _accepted_reasoning_effort(model: str, requested: str, custom_llm_provider: _NO_TOOL_CALL_TURN: Final[tuple[int, frozenset[str]]] = (-1, frozenset()) +_MISTRAL_REQUEST_ROUTING: Final = TypeAdapter(MistralRequestRouting) +_MISTRAL_CACHE_KEY_PARAMS: Final = TypeAdapter(MistralCacheKeyParams) +_MISTRAL_REASONING_EFFORT_PARAMS: Final = TypeAdapter(MistralReasoningEffortParams) +_MISTRAL_FILE_PART: Final = TypeAdapter(MistralFilePart) + + +def _reasoning_level_and_summary(reasoning_effort: object) -> tuple[object, object]: + requested: Final = as_str_mapping(reasoning_effort) + if requested is None: + return reasoning_effort, None + return requested.get("effort"), requested.get("summary") class MistralConfig(OpenAIGPTConfig): @@ -82,6 +105,8 @@ class MistralConfig(OpenAIGPTConfig): - `response_format` (object or null): An object specifying the format that the model must output. Setting to { "type": "json_object" } enables JSON mode, which guarantees the message the model generates is in JSON. When using JSON mode you MUST also instruct the model to produce JSON yourself with a system or a user message. """ + _is_base_class = False + temperature: int | None = None top_p: int | None = None max_tokens: int | None = None @@ -122,6 +147,7 @@ class MistralConfig(OpenAIGPTConfig): accepts_reasoning_effort: Final = is_magistral or supports_reasoning( model=model, custom_llm_provider=self.custom_llm_provider ) + accepts_prompt_cache_key: Final = self.custom_llm_provider == "mistral" return [ "stream", "temperature", @@ -136,6 +162,7 @@ class MistralConfig(OpenAIGPTConfig): "parallel_tool_calls", *(("thinking",) if is_magistral else ()), *(("reasoning_effort",) if accepts_reasoning_effort else ()), + *(("prompt_cache_key",) if accepts_prompt_cache_key else ()), ] def _map_tool_choice(self, tool_choice: str) -> str: @@ -202,15 +229,84 @@ class MistralConfig(OpenAIGPTConfig): optional_params["extra_body"] = {"random_seed": value} if param == "response_format": optional_params["response_format"] = value - if param == "reasoning_effort" and "magistral" not in model.lower(): - optional_params["reasoning_effort"] = _accepted_reasoning_effort(model, value, self.custom_llm_provider) + if ( + param == "reasoning_effort" + and "magistral" not in model.lower() + and ( + reasoning_level := self._reasoning_level_to_send( + model, _MISTRAL_REASONING_EFFORT_PARAMS.validate_python(non_default_params), drop_params + ) + ) + is not None + ): + optional_params["reasoning_effort"] = reasoning_level if param in ("reasoning_effort", "thinking") and "magistral" in model.lower(): # Flag that we need to add reasoning system prompt optional_params["_add_reasoning_prompt"] = True - if param == "parallel_tool_calls": - optional_params["parallel_tool_calls"] = value + if param in ("parallel_tool_calls", "prompt_cache_key"): + optional_params[param] = value return optional_params + def _reasoning_level_to_send( + self, model: str, requested: MistralReasoningEffortParams, drop_params: bool + ) -> str | None: + level, summary = _reasoning_level_and_summary(requested.get("reasoning_effort")) + sendable: Final = ( + _accepted_reasoning_effort(model, level, self.custom_llm_provider) if isinstance(level, str) else None + ) + if sendable is not None and summary is None: + return sendable + if drop_params or litellm.drop_params is True: + return sendable + raise UnsupportedParamsError( + message=( + f"{self.custom_llm_provider} does not support parameters: ['reasoning_effort'], for model={model}, " + "since the Mistral API takes it only as a level such as 'high', with no reasoning summary. " + "To drop it, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n" + ), + llm_provider=self.custom_llm_provider, + model=model, + ) + + def accepts_prompt_cache_key_at(self, api_base: str | None) -> bool: + resolved_api_base, _ = self._get_openai_compatible_provider_info(api_base, None) + hostname: Final = urlparse(resolved_api_base).hostname or "" + return hostname == "mistral.ai" or hostname.endswith(".mistral.ai") + + @staticmethod + def takes_prompt_cache_key_value(prompt_cache_key: object) -> bool: + return isinstance(prompt_cache_key, str) and prompt_cache_key.isascii() + + def _prompt_cache_key_rejection(self, prompt_cache_key: object, api_base: str | None) -> str | None: + if not self.accepts_prompt_cache_key_at(api_base): + return "since only the Mistral API (*.mistral.ai) takes it" + if not self.takes_prompt_cache_key_value(prompt_cache_key): + return "since the Mistral API takes it only as a string of ASCII characters" + return None + + def _prompt_cache_key_withheld(self, model: str, prompt_cache_key: object, routing: MistralRequestRouting) -> bool: + requested_api_base: Final = routing.get("api_base") + api_base: Final = requested_api_base if isinstance(requested_api_base, str) else None + allowed_openai_params: Final = routing.get("allowed_openai_params") + if isinstance(allowed_openai_params, (list, tuple)) and "prompt_cache_key" in allowed_openai_params: + return False + rejection: Final = self._prompt_cache_key_rejection(prompt_cache_key, api_base) + if rejection is None: + return False + if litellm.drop_params is True or routing.get("drop_params") is True: + return True + requested_provider: Final = routing.get("custom_llm_provider") + provider: Final = requested_provider if isinstance(requested_provider, str) else self.custom_llm_provider + raise UnsupportedParamsError( + message=( + f"{provider} does not support parameters: ['prompt_cache_key'], for model={model}, {rejection}. " + "To send it anyway, pass `allowed_openai_params=['prompt_cache_key']`. To drop it, set " + "`litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n" + ), + llm_provider=provider, + model=model, + ) + def _get_openai_compatible_provider_info(self, api_base: str | None, api_key: str | None) -> tuple[str, str | None]: # mistral is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.mistral.ai api_base = ( @@ -344,24 +440,31 @@ class MistralConfig(OpenAIGPTConfig): messages = self._handle_message_with_file(messages) return messages + @staticmethod + def _with_mistral_file_id(block: Mapping[str, object]) -> Mapping[str, object]: + file_part: Final = block.get("file") if block.get("type") == "file" else None + file_id: Final = ( + _MISTRAL_FILE_PART.validate_python(file_part).get("file_id") if isinstance(file_part, Mapping) else None + ) + if not file_id: + return block + return {**{key: value for key, value in block.items() if key != "file"}, "file_id": file_id} + + @classmethod + def _with_mistral_file_ids(cls, message: AllMessageValues) -> AllMessageValues: + content: Final = message.get("content") + if not isinstance(content, list): + return message + blocks: Final[tuple[Mapping[str, object], ...]] = tuple(content) + return cast( # cast-ok: only file blocks change, to the file_id shape the Mistral API takes + AllMessageValues, {**message, "content": [cls._with_mistral_file_id(block) for block in blocks]} + ) + def _handle_message_with_file(self, messages: list[AllMessageValues]) -> list[AllMessageValues]: """ Mistral API supports only 'file_id' in message content with type 'file'. """ - for m in messages: - _content_block = m.get("content") - if _content_block and isinstance(_content_block, list): - if any(c.get("type") == "file" for c in _content_block): - # If file content is present, we get file_id from 'file' attribute of content block - # then replace 'file' with 'file_id' and assign the value of 'file_id' attribute to it. - file_contents = [c for c in _content_block if c.get("type") == "file"] - for file_content in file_contents: - file_id = file_content.get("file", {}).get("file_id") - if file_id: - # Replace 'file' with 'file_id' - file_content["file_id"] = file_id # pyright: ignore[reportGeneralTypeIssues] # legacy in-place rewrite of the block shape - file_content.pop("file", None) - return messages + return [self._with_mistral_file_ids(m) for m in messages] def _add_reasoning_system_prompt_if_needed( self, messages: list[AllMessageValues], optional_params: dict @@ -633,7 +736,16 @@ class MistralConfig(OpenAIGPTConfig): if "magistral" in model.lower() and optional_params.get("_add_reasoning_prompt", False): messages = self._add_reasoning_system_prompt_if_needed(messages, optional_params) - upstream_params: Final = {key: value for key, value in optional_params.items() if key != "client_metadata"} + cache_key_params: Final = _MISTRAL_CACHE_KEY_PARAMS.validate_python(optional_params) + withheld_params: Final = ( + ("client_metadata", "prompt_cache_key") + if "prompt_cache_key" in cache_key_params + and self._prompt_cache_key_withheld( + model, cache_key_params["prompt_cache_key"], _MISTRAL_REQUEST_ROUTING.validate_python(litellm_params) + ) + else ("client_metadata",) + ) + upstream_params: Final = {key: value for key, value in optional_params.items() if key not in withheld_params} # Call parent transform_request which handles _transform_messages return super().transform_request( diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 73f40f5e0e4..4e8009eeace 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -27,6 +27,7 @@ import litellm from litellm import LlmProviders from litellm._logging import verbose_logger from litellm.constants import DEFAULT_MAX_RETRIES +from litellm.exceptions import UnsupportedParamsError from litellm.files.types import FileContentStreamingResult from litellm.litellm_core_utils.core_helpers import set_provider_response_headers_in_hidden_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -942,8 +943,8 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): litellm.remove_index_from_tool_calls(messages=messages) else: raise e - except OpenAIError as e: - raise e + except (OpenAIError, UnsupportedParamsError): + raise except Exception as e: status_code: Final = getattr(e, "status_code", 500) error_headers = getattr(e, "headers", None) diff --git a/litellm/main.py b/litellm/main.py index 63375292077..ed1ecb51900 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5992,6 +5992,7 @@ def completion( PROVIDER_AFFINITY_HEADER_KWARG_KEY, "fireworks_forward_user_id", "responses_api_version", + "allowed_openai_params", ) if key in kwargs }, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 15eba7f8af8..fc2ba753a46 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -42139,9 +42139,14 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 3e-07, + "reasoning_effort_levels": [ + "none", + "high" + ], "source": "https://docs.mistral.ai/models/devstral-small-2-25-12", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, @@ -42154,9 +42159,14 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, + "reasoning_effort_levels": [ + "none", + "high" + ], "source": "https://mistral.ai/news/devstral-2-vibe-cli", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, @@ -42169,9 +42179,14 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, + "reasoning_effort_levels": [ + "none", + "high" + ], "source": "https://mistral.ai/news/devstral-2-vibe-cli", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, @@ -42742,8 +42757,13 @@ "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 3e-07, + "reasoning_effort_levels": [ + "none", + "high" + ], "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, @@ -67033,9 +67053,14 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, + "reasoning_effort_levels": [ + "none", + "high" + ], "source": "https://mistral.ai/news/devstral-2-vibe-cli", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, diff --git a/litellm/types/llms/mistral.py b/litellm/types/llms/mistral.py index e64aa9c13ab..03e31af6ffc 100644 --- a/litellm/types/llms/mistral.py +++ b/litellm/types/llms/mistral.py @@ -1,6 +1,6 @@ from typing import Literal -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict class FunctionCall(TypedDict): @@ -22,3 +22,22 @@ class MistralTextBlock(TypedDict): class MistralThinkingBlock(TypedDict): type: Literal["thinking"] thinking: list[MistralTextBlock] + + +class MistralRequestRouting(TypedDict, total=False): + api_base: ReadOnly[object] + drop_params: ReadOnly[object] + allowed_openai_params: ReadOnly[object] + custom_llm_provider: ReadOnly[object] + + +class MistralCacheKeyParams(TypedDict, total=False): + prompt_cache_key: ReadOnly[object] + + +class MistralReasoningEffortParams(TypedDict, total=False): + reasoning_effort: ReadOnly[object] + + +class MistralFilePart(TypedDict, total=False): + file_id: ReadOnly[object] diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 15eba7f8af8..fc2ba753a46 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -42139,9 +42139,14 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 3e-07, + "reasoning_effort_levels": [ + "none", + "high" + ], "source": "https://docs.mistral.ai/models/devstral-small-2-25-12", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, @@ -42154,9 +42159,14 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, + "reasoning_effort_levels": [ + "none", + "high" + ], "source": "https://mistral.ai/news/devstral-2-vibe-cli", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, @@ -42169,9 +42179,14 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, + "reasoning_effort_levels": [ + "none", + "high" + ], "source": "https://mistral.ai/news/devstral-2-vibe-cli", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, @@ -42742,8 +42757,13 @@ "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 3e-07, + "reasoning_effort_levels": [ + "none", + "high" + ], "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, @@ -67033,9 +67053,14 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, + "reasoning_effort_levels": [ + "none", + "high" + ], "source": "https://mistral.ai/news/devstral-2-vibe-cli", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_prompt_cache_key.py b/tests/unit/llms/anthropic/pass_through/adapters/test_handler_prompt_cache_key.py index c31d85be0a8..e15ee361067 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_prompt_cache_key.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_handler_prompt_cache_key.py @@ -1,6 +1,7 @@ import json import os import sys +from typing import Final import pytest @@ -70,3 +71,70 @@ def test_prepare_completion_kwargs_keeps_prompt_cache_key_through_responses_rero ) assert completion_kwargs["model"] == "openai/responses/gpt-5.6-luna" assert completion_kwargs["prompt_cache_key"] == "session-abc" + + +@pytest.mark.parametrize( + ("api_base", "azure_api_base_env", "derives_key"), + [ + (None, None, True), + ("https://api.mistral.ai/v1", None, True), + ("https://my-resource.openai.azure.com/openai/v1", None, False), + (None, "https://my-resource.services.ai.azure.com/openai", False), + ], +) +def test_prepare_completion_kwargs_derives_prompt_cache_key_only_for_a_mistral_hosted_endpoint( + monkeypatch: pytest.MonkeyPatch, api_base: str | None, azure_api_base_env: str | None, derives_key: bool +) -> None: + if azure_api_base_env is None: + monkeypatch.delenv("MISTRAL_AZURE_API_BASE", raising=False) + else: + monkeypatch.setenv("MISTRAL_AZURE_API_BASE", azure_api_base_env) + + completion_kwargs: Final = _prepare( + "mistral/mistral-small-latest", {"custom_llm_provider": "mistral", "api_base": api_base} + ) + + assert completion_kwargs["user"] == CLAUDE_CODE_USER_ID + assert ("prompt_cache_key" in completion_kwargs) is derives_key + + +@pytest.mark.parametrize( + ("codestral_api_base_env", "derives_key"), + [(None, True), ("https://my-resource.services.ai.azure.com/openai/v1", False)], +) +def test_prepare_completion_kwargs_derives_prompt_cache_key_for_codestral_only_on_its_mistral_host( + monkeypatch: pytest.MonkeyPatch, codestral_api_base_env: str | None, derives_key: bool +) -> None: + monkeypatch.delenv("MISTRAL_AZURE_API_BASE", raising=False) + if codestral_api_base_env is None: + monkeypatch.delenv("CODESTRAL_API_BASE", raising=False) + else: + monkeypatch.setenv("CODESTRAL_API_BASE", codestral_api_base_env) + + completion_kwargs: Final = _prepare("codestral/codestral-latest", {"custom_llm_provider": "codestral"}) + + assert ("prompt_cache_key" in completion_kwargs) is derives_key + + +def test_prepare_completion_kwargs_derives_no_prompt_cache_key_mistral_cannot_take_from_a_non_ascii_session() -> None: + non_ascii_user_id: Final = json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": "sitzung-ü"}) + + mistral_kwargs: Final = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( + max_tokens=1024, + messages=MESSAGES, + model="mistral/mistral-small-latest", + metadata={"user_id": non_ascii_user_id}, + thinking=None, + extra_kwargs={"custom_llm_provider": "mistral"}, + )[0] + openai_kwargs: Final = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( + max_tokens=1024, + messages=MESSAGES, + model="openai/gpt-5.6-luna", + metadata={"user_id": non_ascii_user_id}, + thinking=None, + extra_kwargs={"custom_llm_provider": "openai"}, + )[0] + + assert "prompt_cache_key" not in mistral_kwargs + assert openai_kwargs["prompt_cache_key"] == "sitzung-ü" diff --git a/tests/unit/llms/mistral/test_mistral_chat_transformation.py b/tests/unit/llms/mistral/test_mistral_chat_transformation.py index 10f57be33fe..3a78a28165c 100644 --- a/tests/unit/llms/mistral/test_mistral_chat_transformation.py +++ b/tests/unit/llms/mistral/test_mistral_chat_transformation.py @@ -1,7 +1,13 @@ -from typing import List, cast +import json +from collections.abc import Mapping +from typing import Final, List, cast from unittest.mock import MagicMock, patch +from urllib.parse import urlparse +import httpx import pytest +import respx +from openai import AsyncOpenAI from litellm.litellm_core_utils.prompt_templates.common_utils import TOOL_RESULT_IMAGE_BOUNDARY from litellm.llms.mistral.chat.transformation import ( @@ -43,21 +49,15 @@ class TestMistralReasoningSupport: mistral_config = MistralConfig() # Test magistral model supports reasoning parameters - supported_params = mistral_config.get_supported_openai_params( - "mistral/magistral-medium-2506" - ) + supported_params = mistral_config.get_supported_openai_params("mistral/magistral-medium-2506") assert "reasoning_effort" in supported_params assert "thinking" in supported_params - supported_params_reasoning = mistral_config.get_supported_openai_params( - "mistral/mistral-medium-latest" - ) + supported_params_reasoning = mistral_config.get_supported_openai_params("mistral/mistral-medium-latest") assert "reasoning_effort" in supported_params_reasoning assert "thinking" not in supported_params_reasoning - supported_params_normal = mistral_config.get_supported_openai_params( - "mistral/mistral-large-latest" - ) + supported_params_normal = mistral_config.get_supported_openai_params("mistral/mistral-large-latest") assert "reasoning_effort" not in supported_params_normal assert "thinking" not in supported_params_normal @@ -115,6 +115,36 @@ class TestMistralReasoningSupport: ) assert optional_params["reasoning_effort"] == sent + @pytest.mark.parametrize( + ("requested", "drop_params", "sent"), + [ + ({"effort": "medium"}, False, {"reasoning_effort": "high"}), + ({"effort": "high", "summary": "auto"}, True, {"reasoning_effort": "high"}), + ({"summary": "auto"}, True, {}), + ], + ) + def test_reasoning_effort_object_is_sent_as_its_level(self, local_model_cost_map, requested, drop_params, sent): + import litellm + + optional_params = litellm.get_optional_params( + model="mistral-small-latest", + custom_llm_provider="mistral", + reasoning_effort=requested, + drop_params=drop_params, + ) + assert {key: optional_params[key] for key in ("reasoning_effort",) if key in optional_params} == sent + + @pytest.mark.parametrize("requested", [{"effort": "high", "summary": "auto"}, {"summary": "auto"}]) + def test_reasoning_effort_object_mistral_cannot_take_is_rejected(self, local_model_cost_map, requested): + import litellm + + with pytest.raises(litellm.UnsupportedParamsError, match=r"\['reasoning_effort'\]"): + litellm.get_optional_params( + model="devstral-latest", + custom_llm_provider="mistral", + reasoning_effort=requested, + ) + def test_reasoning_effort_is_forwarded_verbatim_when_the_map_declares_no_levels( self, local_model_cost_map, monkeypatch ): @@ -192,9 +222,7 @@ class TestMistralReasoningSupport: messages = [{"role": "user", "content": "What is 2+2?"}] optional_params = {"_add_reasoning_prompt": True} - result = mistral_config._add_reasoning_system_prompt_if_needed( - messages, optional_params - ) + result = mistral_config._add_reasoning_system_prompt_if_needed(messages, optional_params) # Should add a new system message at the beginning assert len(result) == 2 @@ -216,9 +244,7 @@ class TestMistralReasoningSupport: ] optional_params = {"_add_reasoning_prompt": True} - result = mistral_config._add_reasoning_system_prompt_if_needed( - messages, optional_params - ) + result = mistral_config._add_reasoning_system_prompt_if_needed(messages, optional_params) # Should modify existing system message assert len(result) == 2 @@ -249,9 +275,7 @@ class TestMistralReasoningSupport: ] optional_params = {"_add_reasoning_prompt": True} - result = mistral_config._add_reasoning_system_prompt_if_needed( - messages, optional_params - ) + result = mistral_config._add_reasoning_system_prompt_if_needed(messages, optional_params) # Should modify existing system message preserving list format assert len(result) == 2 @@ -264,10 +288,7 @@ class TestMistralReasoningSupport: # Original content should be preserved assert "You are a helpful assistant." in result[0]["content"][1]["text"] - assert ( - "You always provide detailed explanations." - in result[0]["content"][2]["text"] - ) + assert "You always provide detailed explanations." in result[0]["content"][2]["text"] assert result[1]["role"] == "user" @@ -285,9 +306,7 @@ class TestMistralReasoningSupport: ] string_params = {"_add_reasoning_prompt": True} - string_result = mistral_config._add_reasoning_system_prompt_if_needed( - string_messages, string_params - ) + string_result = mistral_config._add_reasoning_system_prompt_if_needed(string_messages, string_params) assert isinstance(string_result[0]["content"], str) assert "" in string_result[0]["content"] assert "You are helpful." in string_result[0]["content"] @@ -302,9 +321,7 @@ class TestMistralReasoningSupport: ] list_params = {"_add_reasoning_prompt": True} - list_result = mistral_config._add_reasoning_system_prompt_if_needed( - list_messages, list_params - ) + list_result = mistral_config._add_reasoning_system_prompt_if_needed(list_messages, list_params) assert isinstance(list_result[0]["content"], list) assert list_result[0]["content"][0]["type"] == "text" assert "" in list_result[0]["content"][0]["text"] @@ -317,9 +334,7 @@ class TestMistralReasoningSupport: messages = [{"role": "user", "content": "What is 2+2?"}] optional_params = {} - result = mistral_config._add_reasoning_system_prompt_if_needed( - messages, optional_params - ) + result = mistral_config._add_reasoning_system_prompt_if_needed(messages, optional_params) # Should return messages unchanged assert result == messages @@ -442,9 +457,7 @@ class TestMistralReasoningSupport: def test_mistral_streaming_chunk_preserves_thinking_blocks(): """Ensure streaming chunks keep magistral reasoning content.""" - iterator = MistralChatResponseIterator( - streaming_response=iter([]), sync_stream=True, json_mode=False - ) + iterator = MistralChatResponseIterator(streaming_response=iter([]), sync_stream=True, json_mode=False) streamed_chunk = { "id": "chunk-1", @@ -521,9 +534,7 @@ class TestMistralParallelToolCalls: def test_get_supported_openai_params_includes_parallel_tool_calls(self): """Test that parallel_tool_calls is in supported parameters.""" mistral_config = MistralConfig() - supported_params = mistral_config.get_supported_openai_params( - "mistral/mistral-large-latest" - ) + supported_params = mistral_config.get_supported_openai_params("mistral/mistral-large-latest") assert "parallel_tool_calls" in supported_params def test_transform_request_preserves_parallel_tool_calls(self): @@ -689,9 +700,7 @@ class TestMistralEmptyContentHandling: result = MistralConfig._handle_empty_content_response(response_data) - assert ( - result["choices"][0]["message"]["content"] == "Hello, how can I help you?" - ) + assert result["choices"][0]["message"]["content"] == "Hello, how can I help you?" def test_handle_empty_content_response_handles_multiple_choices(self): """Test that only the first choice is processed for empty content.""" @@ -928,9 +937,7 @@ def test_mistral_transform_request_keeps_an_empty_tool_message_after_its_tool_ca { "role": "assistant", "content": "", - "tool_calls": [ - {"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}} - ], + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}}], }, {"role": "tool", "tool_call_id": "call_1", "content": ""}, ], @@ -954,9 +961,7 @@ def test_mistral_transform_request_sends_a_null_tool_reply_as_empty_text(reply): { "role": "assistant", "content": "", - "tool_calls": [ - {"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}} - ], + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}}], }, {"role": "tool", "tool_call_id": "call_1", **reply}, ], @@ -1011,9 +1016,7 @@ def test_mistral_transform_request_hoists_tool_message_image(): { "role": "assistant", "content": "", - "tool_calls": [ - {"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}} - ], + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}}], }, { "role": "tool", @@ -1162,3 +1165,274 @@ def test_mistral_transform_request_drops_an_empty_tool_message_whose_call_id_is_ ) assert [m.get("role") for m in request["messages"]] == ["user", "assistant"] + + +@pytest.mark.parametrize( + ("model", "custom_llm_provider"), + [ + ("mistral-large-latest", "mistral"), + ("mistral-large-4", "mistral"), + ("a-mistral-model-missing-from-the-cost-map", "mistral"), + ("codestral-latest", "codestral"), + ], +) +def test_prompt_cache_key_reaches_the_mistral_request_body( + local_model_cost_map: None, model: str, custom_llm_provider: str +) -> None: + import litellm + + optional_params: Final = litellm.get_optional_params( + model=model, custom_llm_provider=custom_llm_provider, prompt_cache_key="session-1" + ) + request: Final = MistralConfig().transform_request( + model=model, + messages=[{"role": "user", "content": "hi"}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert request["prompt_cache_key"] == "session-1" + + +AZURE_HOSTED_MISTRAL_API_BASE: Final = "https://my-resource.services.ai.azure.com/openai/v1" + + +def _transform_with_prompt_cache_key( + litellm_params: dict[str, object], prompt_cache_key: object = "session-1" +) -> Mapping[str, object]: + return cast( + Mapping[str, object], + MistralConfig().transform_request( + model="mistral-small-latest", + messages=[{"role": "user", "content": "hi"}], + optional_params={"prompt_cache_key": prompt_cache_key, "temperature": 0.2}, + litellm_params=litellm_params, + headers={}, + ), + ) + + +@pytest.mark.parametrize("api_base", ["https://api.mistral.ai/v1", "https://codestral.mistral.ai/v1"]) +def test_prompt_cache_key_reaches_a_mistral_hosted_endpoint(api_base: str) -> None: + request: Final = _transform_with_prompt_cache_key({"api_base": api_base}) + + assert request["prompt_cache_key"] == "session-1" + + +@pytest.mark.parametrize("api_base", [AZURE_HOSTED_MISTRAL_API_BASE, "https://mistral.ai.example.com/v1"]) +def test_prompt_cache_key_for_another_host_is_rejected_without_drop_params(api_base: str) -> None: + import litellm + + with pytest.raises(litellm.UnsupportedParamsError, match="prompt_cache_key") as rejection: + _transform_with_prompt_cache_key({"api_base": api_base}) + + upstream_host: Final = urlparse(api_base).hostname or api_base + assert upstream_host not in str(rejection.value) + + +UNSENDABLE_PROMPT_CACHE_KEYS: Final = ("Büro-Sitzung-42", "日本", 123, ["session-1"]) + + +@pytest.mark.parametrize("prompt_cache_key", UNSENDABLE_PROMPT_CACHE_KEYS) +def test_prompt_cache_key_the_mistral_api_cannot_take_is_rejected_without_drop_params( + prompt_cache_key: object, +) -> None: + import litellm + + with pytest.raises(litellm.UnsupportedParamsError, match="ASCII"): + _transform_with_prompt_cache_key({"api_base": "https://api.mistral.ai/v1"}, prompt_cache_key) + + +@pytest.mark.parametrize("prompt_cache_key", UNSENDABLE_PROMPT_CACHE_KEYS) +def test_prompt_cache_key_the_mistral_api_cannot_take_is_dropped_under_drop_params(prompt_cache_key: object) -> None: + request: Final = _transform_with_prompt_cache_key( + {"api_base": "https://api.mistral.ai/v1", "drop_params": True}, prompt_cache_key + ) + + assert "prompt_cache_key" not in request + assert request["temperature"] == 0.2 + + +def test_prompt_cache_key_with_ascii_punctuation_reaches_the_mistral_api() -> None: + request: Final = _transform_with_prompt_cache_key({}, "team a/session:42") + + assert request["prompt_cache_key"] == "team a/session:42" + + +def test_prompt_cache_key_allowlisted_for_another_purpose_is_still_rejected_for_another_host() -> None: + import litellm + + with pytest.raises(litellm.UnsupportedParamsError, match="allowed_openai_params"): + _transform_with_prompt_cache_key( + {"api_base": AZURE_HOSTED_MISTRAL_API_BASE, "allowed_openai_params": ["reasoning_effort"]} + ) + + +def _chat_completion_reply() -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 0, + "model": "mistral-small-latest", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + + +def test_allowlisted_prompt_cache_key_reaches_a_gateway_in_front_of_mistral(respx_mock: respx.MockRouter) -> None: + import litellm + + gateway_route: Final = respx_mock.post("https://gateway.example.com/mistral/v1/chat/completions").mock( + return_value=_chat_completion_reply() + ) + + litellm.completion( + model="mistral/mistral-small-latest", + messages=[{"role": "user", "content": "hi"}], + api_base="https://gateway.example.com/mistral/v1", + api_key="gateway-key", + prompt_cache_key="session-1", + allowed_openai_params=["prompt_cache_key"], + max_retries=0, + ) + + assert json.loads(gateway_route.calls.last.request.content)["prompt_cache_key"] == "session-1" + + +@pytest.mark.asyncio +async def test_prompt_cache_key_is_dropped_from_an_async_codestral_call_to_another_host( + respx_mock: respx.MockRouter, +) -> None: + import litellm + + azure_route: Final = respx_mock.post(f"{AZURE_HOSTED_MISTRAL_API_BASE}/chat/completions").mock( + return_value=_chat_completion_reply() + ) + + await litellm.acompletion( + model="codestral/codestral-latest", + messages=[{"role": "user", "content": "hi"}], + api_base=AZURE_HOSTED_MISTRAL_API_BASE, + api_key="azure-key", + prompt_cache_key="session-1", + drop_params=True, + max_retries=0, + client=AsyncOpenAI( + api_key="azure-key", base_url=AZURE_HOSTED_MISTRAL_API_BASE, http_client=httpx.AsyncClient() + ), + ) + + assert "prompt_cache_key" not in json.loads(azure_route.calls.last.request.content) + + +@pytest.mark.asyncio +async def test_prompt_cache_key_rejection_for_an_async_codestral_call_names_codestral( + respx_mock: respx.MockRouter, +) -> None: + import litellm + + azure_route: Final = respx_mock.post(f"{AZURE_HOSTED_MISTRAL_API_BASE}/chat/completions").mock( + return_value=_chat_completion_reply() + ) + + with pytest.raises(litellm.BadRequestError, match=r"codestral does not support parameters: \['prompt_cache_key'\]"): + await litellm.acompletion( + model="codestral/codestral-latest", + messages=[{"role": "user", "content": "hi"}], + api_base=AZURE_HOSTED_MISTRAL_API_BASE, + api_key="azure-key", + prompt_cache_key="session-1", + max_retries=0, + client=AsyncOpenAI( + api_key="azure-key", base_url=AZURE_HOSTED_MISTRAL_API_BASE, http_client=httpx.AsyncClient() + ), + ) + + assert not azure_route.called + + +@pytest.mark.asyncio +async def test_async_codestral_call_sends_an_uploaded_file_by_its_id(respx_mock: respx.MockRouter) -> None: + import litellm + + codestral_api_base: Final = "https://codestral.mistral.ai/v1" + codestral_route: Final = respx_mock.post(f"{codestral_api_base}/chat/completions").mock( + return_value=_chat_completion_reply() + ) + + await litellm.acompletion( + model="codestral/codestral-latest", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "Review this file"}, + {"type": "file", "file": {"file_id": "file-12345"}}, + ], + } + ], + api_base=codestral_api_base, + api_key="codestral-key", + max_retries=0, + client=AsyncOpenAI(api_key="codestral-key", base_url=codestral_api_base, http_client=httpx.AsyncClient()), + ) + + sent_content: Final = json.loads(codestral_route.calls.last.request.content)["messages"][0]["content"] + assert sent_content[1] == {"type": "file", "file_id": "file-12345"} + + +def test_prompt_cache_key_rejection_for_a_sync_codestral_call_is_an_unsupported_params_error( + respx_mock: respx.MockRouter, +) -> None: + import litellm + + azure_route: Final = respx_mock.post(f"{AZURE_HOSTED_MISTRAL_API_BASE}/chat/completions").mock( + return_value=_chat_completion_reply() + ) + + with pytest.raises( + litellm.UnsupportedParamsError, match=r"^litellm.UnsupportedParamsError: codestral does not support" + ): + litellm.completion( + model="codestral/codestral-latest", + messages=[{"role": "user", "content": "hi"}], + api_base=AZURE_HOSTED_MISTRAL_API_BASE, + api_key="azure-key", + prompt_cache_key="session-1", + max_retries=0, + ) + + assert not azure_route.called + + +def test_prompt_cache_key_for_another_host_is_dropped_under_drop_params() -> None: + request: Final = _transform_with_prompt_cache_key({"api_base": AZURE_HOSTED_MISTRAL_API_BASE, "drop_params": True}) + + assert "prompt_cache_key" not in request + assert request["temperature"] == 0.2 + + +def test_prompt_cache_key_for_the_azure_endpoint_set_in_the_environment_is_dropped_under_drop_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MISTRAL_AZURE_API_BASE", "https://my-resource.services.ai.azure.com/openai") + + request: Final = _transform_with_prompt_cache_key({"drop_params": True}) + + assert "prompt_cache_key" not in request + + +def test_prompt_cache_key_stays_unsupported_for_vertex_hosted_mistral(local_model_cost_map: None) -> None: + import litellm + + with pytest.raises(litellm.UnsupportedParamsError): + litellm.get_optional_params(model="mistral-medium-3", custom_llm_provider="vertex_ai", prompt_cache_key="s") + + dropped: Final = litellm.get_optional_params( + model="mistral-medium-3", custom_llm_provider="vertex_ai", prompt_cache_key="s", drop_params=True + ) + assert "prompt_cache_key" not in dropped