From ff7bdb629031c77dd40f570887f9c999565756cb Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 16 Aug 2025 00:41:11 -0700 Subject: [PATCH] fix(mistral/chat/transformation.py): handle empty message content for mistral calls Fixes https://github.com/BerriAI/litellm/issues/13355 --- litellm/llms/mistral/chat/transformation.py | 95 +++++-- .../llms/openai/chat/gpt_transformation.py | 1 + tests/llm_translation/test_mistral_api.py | 15 + .../test_mistral_chat_transformation.py | 264 ++++++++++-------- 4 files changed, 232 insertions(+), 143 deletions(-) diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index b38a4982471..dda497dac86 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -6,7 +6,18 @@ Why separate file? Make it easy to see how transformation works Docs - https://docs.mistral.ai/api/ """ -from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, cast, overload +from typing import ( + Any, + Coroutine, + List, + Literal, + Optional, + Tuple, + Union, + cast, + get_type_hints, + overload, +) import httpx @@ -145,7 +156,9 @@ class MistralConfig(OpenAIGPTConfig): for param, value in non_default_params.items(): if param == "max_tokens": optional_params["max_tokens"] = value - if param == "max_completion_tokens": # max_completion_tokens should take priority + if ( + param == "max_completion_tokens" + ): # max_completion_tokens should take priority optional_params["max_tokens"] = value if param == "tools": # Clean tools to remove problematic schema fields for Mistral API @@ -159,7 +172,9 @@ class MistralConfig(OpenAIGPTConfig): if param == "stop": optional_params["stop"] = value if param == "tool_choice" and isinstance(value, str): - optional_params["tool_choice"] = self._map_tool_choice(tool_choice=value) + optional_params["tool_choice"] = self._map_tool_choice( + tool_choice=value + ) if param == "seed": optional_params["extra_body"] = {"random_seed": value} if param == "response_format": @@ -185,7 +200,9 @@ class MistralConfig(OpenAIGPTConfig): ) # type: ignore # if api_base does not end with /v1 we add it - if api_base is not None and not api_base.endswith("/v1"): # Mistral always needs a /v1 at the end + if api_base is not None and not api_base.endswith( + "/v1" + ): # Mistral always needs a /v1 at the end api_base = api_base + "/v1" dynamic_api_key = ( api_key @@ -194,10 +211,12 @@ class MistralConfig(OpenAIGPTConfig): ) return api_base, dynamic_api_key + # fmt: off + @overload def _transform_messages( self, messages: List[AllMessageValues], model: str, is_async: Literal[True] - ) -> Coroutine[Any, Any, List[AllMessageValues]]: + ) -> Coroutine[Any, Any, List[AllMessageValues]]: ... @overload @@ -206,8 +225,9 @@ class MistralConfig(OpenAIGPTConfig): messages: List[AllMessageValues], model: str, is_async: Literal[False] = False, - ) -> List[AllMessageValues]: + ) -> List[AllMessageValues]: ... + # fmt: on def _transform_messages( self, messages: List[AllMessageValues], model: str, is_async: bool = False @@ -239,6 +259,8 @@ class MistralConfig(OpenAIGPTConfig): for m in messages: m = MistralConfig._handle_name_in_message(m) m = MistralConfig._handle_tool_call_message(m) + if MistralConfig._is_empty_assistant_message(m): + continue m = strip_none_values_from_message(m) # prevents 'extra_forbidden' error new_messages.append(m) @@ -269,20 +291,30 @@ class MistralConfig(OpenAIGPTConfig): # Handle both string and list content, preserving original format if isinstance(existing_content, str): # String content - prepend reasoning prompt - new_content: Union[str, list] = f"{reasoning_prompt}\n\n{existing_content}" + new_content: Union[str, list] = ( + f"{reasoning_prompt}\n\n{existing_content}" + ) elif isinstance(existing_content, list): # List content - prepend reasoning prompt as text block - new_content = [{"type": "text", "text": reasoning_prompt + "\n\n"}] + existing_content + new_content = [ + {"type": "text", "text": reasoning_prompt + "\n\n"} + ] + existing_content else: # Fallback for any other type - convert to string new_content = f"{reasoning_prompt}\n\n{str(existing_content)}" - messages[i] = cast(AllMessageValues, {**msg, "content": new_content}) + messages[i] = cast( + AllMessageValues, {**msg, "content": new_content} + ) break else: # Add new system message with reasoning instructions reasoning_message: AllMessageValues = cast( - AllMessageValues, {"role": "system", "content": self._get_mistral_reasoning_system_prompt()} + AllMessageValues, + { + "role": "system", + "content": self._get_mistral_reasoning_system_prompt(), + }, ) messages = [reasoning_message] + messages @@ -294,32 +326,34 @@ class MistralConfig(OpenAIGPTConfig): def _clean_tool_schema_for_mistral(cls, tools: list) -> list: """ Clean tool schemas to remove fields that cause issues with Mistral API. - + Removes: - $id and $schema fields (cause grammar validation errors) - additionalProperties=False (causes OpenAI API schema errors) - strict field (not supported by Mistral) - + Args: tools: List of tool definitions max_depth: Maximum recursion depth for schema cleaning (default: 10) - + Returns: Cleaned tools list """ if not tools: return tools - + import copy from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH from litellm.utils import _remove_json_schema_refs cleaned_tools = copy.deepcopy(tools) - + # Apply all cleaning functions with max_depth protection - cleaned_tools = _remove_json_schema_refs(cleaned_tools, max_depth=DEFAULT_MAX_RECURSE_DEPTH) - + cleaned_tools = _remove_json_schema_refs( + cleaned_tools, max_depth=DEFAULT_MAX_RECURSE_DEPTH + ) + return cleaned_tools @classmethod @@ -360,6 +394,25 @@ class MistralConfig(OpenAIGPTConfig): message["tool_calls"] = mistral_tool_calls # type: ignore return message + @classmethod + def _is_empty_assistant_message(cls, message: AllMessageValues) -> bool: + """ + Mistral API does not support empty string in assistant content. + """ + from litellm.types.llms.openai import ChatCompletionAssistantMessage + + set_keys = get_type_hints(ChatCompletionAssistantMessage).keys() + + all_expected_values_are_empty = True + for key in set_keys: + if key != "role" and message.get(key) is not None: + if key == "content" and message.get(key) == "": + continue + else: + all_expected_values_are_empty = False + break + return all_expected_values_are_empty + @staticmethod def _handle_empty_content_response(response_data: dict) -> dict: """ @@ -396,8 +449,12 @@ class MistralConfig(OpenAIGPTConfig): dict: The transformed request. Sent as the body of the API call. """ # Add reasoning system prompt if needed (for magistral models) - if "magistral" in model.lower() and optional_params.get("_add_reasoning_prompt", False): - messages = self._add_reasoning_system_prompt_if_needed(messages, optional_params) + if "magistral" in model.lower() and optional_params.get( + "_add_reasoning_prompt", False + ): + messages = self._add_reasoning_system_prompt_if_needed( + messages, optional_params + ) # Call parent transform_request which handles _transform_messages return super().transform_request( diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 396d59145ff..6e97fde3fe8 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -348,6 +348,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): for message in messages: message_content = message.get("content") message_role = message.get("role") + if ( message_role == "user" and message_content diff --git a/tests/llm_translation/test_mistral_api.py b/tests/llm_translation/test_mistral_api.py index 8cf704fbe89..04a92434208 100644 --- a/tests/llm_translation/test_mistral_api.py +++ b/tests/llm_translation/test_mistral_api.py @@ -37,3 +37,18 @@ class TestMistralCompletion(BaseLLMChatTest): def test_tool_call_no_arguments(self, tool_call_no_arguments): """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833""" pass + + +def test_empty_str_in_assistant_content(): + """Test that empty string in assistant content is converted to None.""" + litellm._turn_on_debug() + response = litellm.completion( + model="mistral/mistral-medium-latest", + messages=[ + {"role": "user", "content": "Hello, how are you?"}, + {"role": "assistant", "content": ""}, + {"role": "user", "content": "Hi again"}, + ], + max_tokens=10, + ) + assert response.choices[0].message.content is not None diff --git a/tests/test_litellm/llms/mistral/test_mistral_chat_transformation.py b/tests/test_litellm/llms/mistral/test_mistral_chat_transformation.py index 3e4d3ddb304..dd48bae3922 100644 --- a/tests/test_litellm/llms/mistral/test_mistral_chat_transformation.py +++ b/tests/test_litellm/llms/mistral/test_mistral_chat_transformation.py @@ -40,21 +40,25 @@ class TestMistralReasoningSupport: def test_get_supported_openai_params_magistral_model(self): """Test that magistral models support reasoning parameters.""" mistral_config = MistralConfig() - + # Test magistral model supports reasoning parameters - supported_params = mistral_config.get_supported_openai_params("mistral/magistral-medium-2506") + supported_params = mistral_config.get_supported_openai_params( + "mistral/magistral-medium-2506" + ) assert "reasoning_effort" in supported_params assert "thinking" in supported_params - + # Test non-magistral model doesn't include reasoning parameters - supported_params_normal = mistral_config.get_supported_openai_params("mistral/mistral-large-latest") + supported_params_normal = mistral_config.get_supported_openai_params( + "mistral/mistral-large-latest" + ) assert "reasoning_effort" not in supported_params_normal assert "thinking" not in supported_params_normal def test_map_openai_params_reasoning_effort(self): """Test that reasoning_effort parameter is properly mapped for magistral models.""" mistral_config = MistralConfig() - + # Test reasoning_effort mapping for magistral model optional_params = {} result = mistral_config.map_openai_params( @@ -63,9 +67,9 @@ class TestMistralReasoningSupport: model="mistral/magistral-medium-2506", drop_params=False, ) - + assert result.get("_add_reasoning_prompt") is True - + # Test reasoning_effort ignored for non-magistral model optional_params_normal = {} result_normal = mistral_config.map_openai_params( @@ -74,13 +78,13 @@ class TestMistralReasoningSupport: model="mistral/mistral-large-latest", drop_params=False, ) - + assert "_add_reasoning_prompt" not in result_normal def test_map_openai_params_thinking(self): """Test that thinking parameter is properly mapped for magistral models.""" mistral_config = MistralConfig() - + # Test thinking mapping for magistral model optional_params = {} result = mistral_config.map_openai_params( @@ -89,7 +93,7 @@ class TestMistralReasoningSupport: model="mistral/magistral-small-2506", drop_params=False, ) - + assert result.get("_add_reasoning_prompt") is True def test_get_mistral_reasoning_system_prompt(self): @@ -101,109 +105,123 @@ class TestMistralReasoningSupport: def test_add_reasoning_system_prompt_no_existing_system_message(self): """Test adding reasoning system prompt when no system message exists.""" mistral_config = MistralConfig() - - messages = [ - {"role": "user", "content": "What is 2+2?"} - ] + + messages = [{"role": "user", "content": "What is 2+2?"}] optional_params = {"_add_reasoning_prompt": True} - - result = mistral_config._add_reasoning_system_prompt_if_needed(messages, optional_params) - + + result = mistral_config._add_reasoning_system_prompt_if_needed( + messages, optional_params + ) + # Should add a new system message at the beginning assert len(result) == 2 assert result[0]["role"] == "system" assert "" in result[0]["content"] assert result[1]["role"] == "user" assert result[1]["content"] == "What is 2+2?" - + # Should remove the internal flag assert "_add_reasoning_prompt" not in optional_params def test_add_reasoning_system_prompt_with_existing_system_message(self): """Test adding reasoning system prompt when system message already exists.""" mistral_config = MistralConfig() - + messages = [ {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What is 2+2?"} + {"role": "user", "content": "What is 2+2?"}, ] optional_params = {"_add_reasoning_prompt": True} - - result = mistral_config._add_reasoning_system_prompt_if_needed(messages, optional_params) - + + result = mistral_config._add_reasoning_system_prompt_if_needed( + messages, optional_params + ) + # Should modify existing system message assert len(result) == 2 assert result[0]["role"] == "system" assert "" in result[0]["content"] assert "You are a helpful assistant." in result[0]["content"] assert result[1]["role"] == "user" - + # Should remove the internal flag assert "_add_reasoning_prompt" not in optional_params def test_add_reasoning_system_prompt_with_existing_list_content(self): """Test adding reasoning system prompt when system message has list content.""" mistral_config = MistralConfig() - + messages = [ { - "role": "system", + "role": "system", "content": [ {"type": "text", "text": "You are a helpful assistant."}, - {"type": "text", "text": "You always provide detailed explanations."} - ] + { + "type": "text", + "text": "You always provide detailed explanations.", + }, + ], }, - {"role": "user", "content": "What is 2+2?"} + {"role": "user", "content": "What is 2+2?"}, ] optional_params = {"_add_reasoning_prompt": True} - - result = mistral_config._add_reasoning_system_prompt_if_needed(messages, optional_params) - + + result = mistral_config._add_reasoning_system_prompt_if_needed( + messages, optional_params + ) + # Should modify existing system message preserving list format assert len(result) == 2 assert result[0]["role"] == "system" assert isinstance(result[0]["content"], list) - + # First item should be the reasoning prompt assert result[0]["content"][0]["type"] == "text" assert "" in result[0]["content"][0]["text"] - + # Original content should be preserved assert "You are a helpful assistant." in result[0]["content"][1]["text"] - assert "You always provide detailed explanations." in result[0]["content"][2]["text"] - + assert ( + "You always provide detailed explanations." + in result[0]["content"][2]["text"] + ) + assert result[1]["role"] == "user" - + # Should remove the internal flag assert "_add_reasoning_prompt" not in optional_params def test_add_reasoning_system_prompt_preserves_content_types(self): """Test that reasoning prompt preserves original content types (string vs list).""" mistral_config = MistralConfig() - + # Test with string content string_messages = [ {"role": "system", "content": "You are helpful."}, - {"role": "user", "content": "Hello"} + {"role": "user", "content": "Hello"}, ] string_params = {"_add_reasoning_prompt": True} - - string_result = mistral_config._add_reasoning_system_prompt_if_needed(string_messages, string_params) + + string_result = mistral_config._add_reasoning_system_prompt_if_needed( + string_messages, string_params + ) assert isinstance(string_result[0]["content"], str) assert "" in string_result[0]["content"] assert "You are helpful." in string_result[0]["content"] - + # Test with list content list_messages = [ { - "role": "system", - "content": [{"type": "text", "text": "You are helpful."}] + "role": "system", + "content": [{"type": "text", "text": "You are helpful."}], }, - {"role": "user", "content": "Hello"} + {"role": "user", "content": "Hello"}, ] list_params = {"_add_reasoning_prompt": True} - - list_result = mistral_config._add_reasoning_system_prompt_if_needed(list_messages, list_params) + + list_result = mistral_config._add_reasoning_system_prompt_if_needed( + list_messages, list_params + ) assert isinstance(list_result[0]["content"], list) assert list_result[0]["content"][0]["type"] == "text" assert "" in list_result[0]["content"][0]["text"] @@ -212,14 +230,14 @@ class TestMistralReasoningSupport: def test_add_reasoning_system_prompt_no_flag(self): """Test that no modification happens when _add_reasoning_prompt flag is not set.""" mistral_config = MistralConfig() - - messages = [ - {"role": "user", "content": "What is 2+2?"} - ] + + messages = [{"role": "user", "content": "What is 2+2?"}] optional_params = {} - - result = mistral_config._add_reasoning_system_prompt_if_needed(messages, optional_params) - + + result = mistral_config._add_reasoning_system_prompt_if_needed( + messages, optional_params + ) + # Should return messages unchanged assert result == messages assert len(result) == 1 @@ -227,46 +245,42 @@ class TestMistralReasoningSupport: def test_transform_request_magistral_with_reasoning(self): """Test transform_request method for magistral model with reasoning.""" mistral_config = MistralConfig() - - messages = [ - {"role": "user", "content": "What is 15 * 7?"} - ] + + messages = [{"role": "user", "content": "What is 15 * 7?"}] optional_params = {"_add_reasoning_prompt": True} - + result = mistral_config.transform_request( model="mistral/magistral-medium-2506", messages=messages, optional_params=optional_params, litellm_params={}, - headers={} + headers={}, ) - + # Should have added system message assert len(result["messages"]) == 2 assert result["messages"][0]["role"] == "system" assert "" in result["messages"][0]["content"] assert result["messages"][1]["role"] == "user" - + # Should remove internal flag from optional_params assert "_add_reasoning_prompt" not in result def test_transform_request_magistral_without_reasoning(self): """Test transform_request method for magistral model without reasoning.""" mistral_config = MistralConfig() - - messages = [ - {"role": "user", "content": "What is 15 * 7?"} - ] + + messages = [{"role": "user", "content": "What is 15 * 7?"}] optional_params = {} - + result = mistral_config.transform_request( model="mistral/magistral-medium-2506", messages=messages, optional_params=optional_params, litellm_params={}, - headers={} + headers={}, ) - + # Should not modify messages assert len(result["messages"]) == 1 assert result["messages"][0]["role"] == "user" @@ -274,20 +288,18 @@ class TestMistralReasoningSupport: def test_transform_request_non_magistral_with_reasoning_params(self): """Test that non-magistral models ignore reasoning parameters.""" mistral_config = MistralConfig() - - messages = [ - {"role": "user", "content": "What is 15 * 7?"} - ] + + messages = [{"role": "user", "content": "What is 15 * 7?"}] optional_params = {"_add_reasoning_prompt": True} - + result = mistral_config.transform_request( model="mistral/mistral-large-latest", messages=messages, optional_params=optional_params, litellm_params={}, - headers={} + headers={}, ) - + # Should not add system message for non-magistral models assert len(result["messages"]) == 1 assert result["messages"][0]["role"] == "user" @@ -295,15 +307,15 @@ class TestMistralReasoningSupport: def test_case_insensitive_magistral_detection(self): """Test that magistral model detection is case-insensitive.""" mistral_config = MistralConfig() - + # Test various case combinations models_to_test = [ "mistral/Magistral-medium-2506", "mistral/MAGISTRAL-MEDIUM-2506", "mistral/magistral-SMALL-2506", - "MaGiStRaL-medium-2506" + "MaGiStRaL-medium-2506", ] - + for model in models_to_test: supported_params = mistral_config.get_supported_openai_params(model) assert "reasoning_effort" in supported_params, f"Failed for model: {model}" @@ -311,7 +323,7 @@ class TestMistralReasoningSupport: def test_end_to_end_reasoning_workflow(self): """Test the complete workflow from parameter to system prompt injection.""" mistral_config = MistralConfig() - + # Step 1: Map parameters optional_params = {} mapped_params = mistral_config.map_openai_params( @@ -320,23 +332,21 @@ class TestMistralReasoningSupport: model="mistral/magistral-medium-2506", drop_params=False, ) - + assert mapped_params.get("_add_reasoning_prompt") is True assert mapped_params.get("temperature") == 0.7 - + # Step 2: Transform request - messages = [ - {"role": "user", "content": "Solve for x: 2x + 5 = 13"} - ] - + messages = [{"role": "user", "content": "Solve for x: 2x + 5 = 13"}] + result = mistral_config.transform_request( model="mistral/magistral-medium-2506", messages=messages, optional_params=mapped_params, litellm_params={}, - headers={} + headers={}, ) - + # Verify final result assert len(result["messages"]) == 2 assert result["messages"][0]["role"] == "system" @@ -347,7 +357,6 @@ class TestMistralReasoningSupport: assert "_add_reasoning_prompt" not in result - class TestMistralNameHandling: """Test suite for Mistral name handling in messages.""" @@ -363,7 +372,11 @@ class TestMistralNameHandling: def test_handle_name_in_message_tool_role_valid_name_keeps_name(self): """Test that valid name is kept for tool messages.""" # Test with normal function name - tool_message = {"role": "tool", "content": "Function result", "name": "get_weather"} + tool_message = { + "role": "tool", + "content": "Function result", + "name": "get_weather", + } result = MistralConfig._handle_name_in_message(tool_message) assert "name" in result assert result["name"] == "get_weather" @@ -386,26 +399,26 @@ class TestMistralParallelToolCalls: def test_get_supported_openai_params_includes_parallel_tool_calls(self): """Test that parallel_tool_calls is in supported parameters.""" mistral_config = MistralConfig() - supported_params = mistral_config.get_supported_openai_params("mistral/mistral-large-latest") + supported_params = mistral_config.get_supported_openai_params( + "mistral/mistral-large-latest" + ) assert "parallel_tool_calls" in supported_params def test_transform_request_preserves_parallel_tool_calls(self): """Test that transform_request preserves parallel_tool_calls parameter.""" mistral_config = MistralConfig() - - messages = [ - {"role": "user", "content": "What's the weather like?"} - ] + + messages = [{"role": "user", "content": "What's the weather like?"}] optional_params = {"parallel_tool_calls": True} - + result = mistral_config.transform_request( model="mistral/mistral-large-latest", messages=messages, optional_params=optional_params, litellm_params={}, - headers={} + headers={}, ) - + assert result.get("parallel_tool_calls") is True assert len(result["messages"]) == 1 assert result["messages"][0]["role"] == "user" @@ -419,17 +432,14 @@ class TestMistralEmptyContentHandling: response_data = { "choices": [ { - "message": { - "content": "", - "role": "assistant" - }, - "finish_reason": "stop" + "message": {"content": "", "role": "assistant"}, + "finish_reason": "stop", } ] } - + result = MistralConfig._handle_empty_content_response(response_data) - + assert result["choices"][0]["message"]["content"] is None def test_handle_empty_content_response_preserves_actual_content(self): @@ -439,41 +449,47 @@ class TestMistralEmptyContentHandling: { "message": { "content": "Hello, how can I help you?", - "role": "assistant" + "role": "assistant", }, - "finish_reason": "stop" + "finish_reason": "stop", } ] } - + result = MistralConfig._handle_empty_content_response(response_data) - - assert result["choices"][0]["message"]["content"] == "Hello, how can I help you?" + + assert ( + result["choices"][0]["message"]["content"] == "Hello, how can I help you?" + ) def test_handle_empty_content_response_handles_multiple_choices(self): """Test that only the first choice is processed for empty content.""" response_data = { "choices": [ { - "message": { - "content": "", - "role": "assistant" - }, - "finish_reason": "stop" + "message": {"content": "", "role": "assistant"}, + "finish_reason": "stop", }, { - "message": { - "content": "", - "role": "assistant" - }, - "finish_reason": "stop" - } + "message": {"content": "", "role": "assistant"}, + "finish_reason": "stop", + }, ] } - + result = MistralConfig._handle_empty_content_response(response_data) - + # Only first choice should be converted to None assert result["choices"][0]["message"]["content"] is None # Second choice should remain as empty string - assert result["choices"][1]["message"]["content"] is None \ No newline at end of file + assert result["choices"][1]["message"]["content"] is None + + def test_is_empty_assistant_message(self): + """Test that is_empty_assistant_message returns True for empty assistant message.""" + message = {"role": "assistant", "content": ""} + assert MistralConfig._is_empty_assistant_message(message) is True + + def test_is_empty_assistant_message_with_content(self): + """Test that is_empty_assistant_message returns False for assistant message with content.""" + message = {"role": "assistant", "content": "Hello"} + assert MistralConfig._is_empty_assistant_message(message) is False