From b83b1686c227d1d24e269edd59de25e5ec303d4c Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 16 Aug 2025 00:23:37 -0700 Subject: [PATCH 1/7] feat(support-allowed_openai_params-for-responses-api): Fixes https://github.com/BerriAI/litellm/issues/13559 --- .../llms/azure/responses/transformation.py | 5 + .../llms/base_llm/responses/transformation.py | 6 ++ .../llms/openai/responses/transformation.py | 15 ++- litellm/responses/main.py | 86 +++++++++------ litellm/responses/utils.py | 100 +++++++++++++----- litellm/types/llms/openai.py | 4 + tests/llm_translation/test_optional_params.py | 29 +++++ 7 files changed, 183 insertions(+), 62 deletions(-) diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index e3d37c8a15a..a31ccacf345 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -6,6 +6,7 @@ from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfi from litellm.types.llms.openai import * from litellm.types.responses.main import * from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -16,6 +17,10 @@ else: class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): + @property + def custom_llm_provider(self) -> LlmProviders: + return LlmProviders.AZURE + def validate_environment( self, headers: dict, model: str, litellm_params: Optional[GenericLiteLLMParams] ) -> dict: diff --git a/litellm/llms/base_llm/responses/transformation.py b/litellm/llms/base_llm/responses/transformation.py index e2f89da5e86..4da4f7652e0 100644 --- a/litellm/llms/base_llm/responses/transformation.py +++ b/litellm/llms/base_llm/responses/transformation.py @@ -12,6 +12,7 @@ from litellm.types.llms.openai import ( ) from litellm.types.responses.main import * from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -29,6 +30,11 @@ class BaseResponsesAPIConfig(ABC): def __init__(self): pass + @property + @abstractmethod + def custom_llm_provider(self) -> LlmProviders: + pass + @classmethod def get_config(cls): return { diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 501941fdc59..40aae5a1cec 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -13,6 +13,7 @@ from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import * from litellm.types.responses.main import * from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders from ..common_utils import OpenAIError @@ -25,6 +26,10 @@ else: class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): + @property + def custom_llm_provider(self) -> LlmProviders: + return LlmProviders.OPENAI + def get_supported_openai_params(self, model: str) -> list: """ All OpenAI Responses API params are supported @@ -85,8 +90,10 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): ) return final_request_params - - def _validate_input_param(self, input: Union[str, ResponseInputParam]) -> Union[str, ResponseInputParam]: + + def _validate_input_param( + self, input: Union[str, ResponseInputParam] + ) -> Union[str, ResponseInputParam]: """ Ensure all input fields if pydantic are converted to dict @@ -114,7 +121,9 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): """No transform applied since outputs are in OpenAI spec already""" try: raw_response_json = raw_response.json() - raw_response_json["created_at"] = _safe_convert_created_field(raw_response_json["created_at"]) + raw_response_json["created_at"] = _safe_convert_created_field( + raw_response_json["created_at"] + ) except Exception: raise OpenAIError( message=raw_response.text, status_code=raw_response.status_code diff --git a/litellm/responses/main.py b/litellm/responses/main.py index b5f18cf9e1a..ebd0ad4f42d 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -1,7 +1,7 @@ import asyncio import contextvars from functools import partial -from typing import Any, Coroutine, Dict, Iterable, List, Literal, Optional, Union +from typing import Any, Coroutine, Dict, Iterable, List, Literal, Optional, Union, cast import httpx @@ -89,6 +89,7 @@ def mock_responses_api_response( } ) + async def aresponses_api_with_mcp( input: Union[str, ResponseInputParam], model: str, @@ -122,7 +123,7 @@ async def aresponses_api_with_mcp( ) -> Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator]: """ Async version of responses API with MCP integration. - + When MCP tools with server_url="litellm_proxy" are provided, this function will: 1. Get available tools from the MCP server manager 2. Insert the tools into the messages/input @@ -134,19 +135,25 @@ async def aresponses_api_with_mcp( ) # Parse MCP tools and separate from other tools - mcp_tools_with_litellm_proxy, other_tools = LiteLLM_Proxy_MCP_Handler._parse_mcp_tools(tools) - + mcp_tools_with_litellm_proxy, other_tools = ( + LiteLLM_Proxy_MCP_Handler._parse_mcp_tools(tools) + ) + # Get available tools from MCP manager if we have MCP tools openai_tools = [] mcp_tools_fetched = [] if mcp_tools_with_litellm_proxy: user_api_key_auth = kwargs.get("user_api_key_auth") - mcp_tools_fetched = await LiteLLM_Proxy_MCP_Handler._get_mcp_tools_from_manager(user_api_key_auth) - openai_tools = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai(mcp_tools_fetched) - + mcp_tools_fetched = await LiteLLM_Proxy_MCP_Handler._get_mcp_tools_from_manager( + user_api_key_auth + ) + openai_tools = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai( + mcp_tools_fetched + ) + # Combine with other tools all_tools = openai_tools + other_tools if (openai_tools or other_tools) else None - + # Prepare call parameters for reuse call_params = { "include": include, @@ -172,7 +179,7 @@ async def aresponses_api_with_mcp( "custom_llm_provider": custom_llm_provider, **kwargs, } - + # Make initial response API call # TODO: if should auto-execute is True, then this first response should not be streamed response = await aresponses( @@ -180,45 +187,54 @@ async def aresponses_api_with_mcp( model=model, tools=all_tools, previous_response_id=previous_response_id, - **call_params + **call_params, ) - + # Check if we need to auto-execute tool calls (only for non-streaming responses) - if (mcp_tools_with_litellm_proxy and - isinstance(response, ResponsesAPIResponse) and - LiteLLM_Proxy_MCP_Handler._should_auto_execute_tools(mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy)): # type: ignore - tool_calls = LiteLLM_Proxy_MCP_Handler._extract_tool_calls_from_response(response=response) - + if ( + mcp_tools_with_litellm_proxy + and isinstance(response, ResponsesAPIResponse) + and LiteLLM_Proxy_MCP_Handler._should_auto_execute_tools( + mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy + ) + ): # type: ignore + tool_calls = LiteLLM_Proxy_MCP_Handler._extract_tool_calls_from_response( + response=response + ) + if tool_calls: - user_api_key_auth = kwargs.get("litellm_metadata", {}).get("user_api_key_auth") - tool_results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls(tool_calls=tool_calls, user_api_key_auth=user_api_key_auth) - + user_api_key_auth = kwargs.get("litellm_metadata", {}).get( + "user_api_key_auth" + ) + tool_results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( + tool_calls=tool_calls, user_api_key_auth=user_api_key_auth + ) + if tool_results: follow_up_input = LiteLLM_Proxy_MCP_Handler._create_follow_up_input( - response=response, - tool_results=tool_results, - original_input=input + response=response, tool_results=tool_results, original_input=input ) - + final_response = await LiteLLM_Proxy_MCP_Handler._make_follow_up_call( follow_up_input=follow_up_input, model=model, all_tools=all_tools, response_id=response.id, - **call_params + **call_params, ) - + # Add custom output elements to the final response if isinstance(final_response, ResponsesAPIResponse): - final_response = LiteLLM_Proxy_MCP_Handler._add_mcp_output_elements_to_response( - response=final_response, - mcp_tools_fetched=mcp_tools_fetched, - tool_results=tool_results + final_response = ( + LiteLLM_Proxy_MCP_Handler._add_mcp_output_elements_to_response( + response=final_response, + mcp_tools_fetched=mcp_tools_fetched, + tool_results=tool_results, + ) ) return final_response - - return response + return response @client @@ -319,7 +335,9 @@ async def aresponses( ) if response is None: - raise ValueError(f"Got an unexpected None response from the Responses API: {response}") + raise ValueError( + f"Got an unexpected None response from the Responses API: {response}" + ) return response except Exception as e: @@ -363,6 +381,7 @@ def responses( extra_body: Optional[Dict[str, Any]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, # LiteLLM specific params, + allowed_openai_params: Optional[List[str]] = None, custom_llm_provider: Optional[str] = None, **kwargs, ): @@ -373,7 +392,7 @@ def responses( from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, ) - + local_vars = locals() try: litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore @@ -445,6 +464,7 @@ def responses( model=model, responses_api_provider_config=responses_api_provider_config, response_api_optional_params=response_api_optional_params, + allowed_openai_params=allowed_openai_params, ) ) diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index b50c84ad463..2b3d178c260 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -1,5 +1,5 @@ import base64 -from typing import Any, Dict, Optional, Union, cast, get_type_hints, overload +from typing import Any, Dict, List, Optional, Union, cast, get_type_hints, overload import litellm from litellm._logging import verbose_logger @@ -16,11 +16,38 @@ from litellm.types.utils import SpecialEnums, Usage class ResponsesAPIRequestUtils: """Helper utils for constructing ResponseAPI requests""" + @staticmethod + def _check_valid_arg( + supported_params: Optional[List[str]], + non_default_params: Dict, + drop_params: Optional[bool], + custom_llm_provider: Optional[str], + model: str, + ): + + if supported_params is None: + return + unsupported_params = {} + for k in non_default_params.keys(): + if k not in supported_params: + unsupported_params[k] = non_default_params[k] + if unsupported_params: + if litellm.drop_params is True or ( + drop_params is not None and drop_params is True + ): + pass + else: + raise litellm.UnsupportedParamsError( + status_code=500, + message=f"{custom_llm_provider} does not support parameters: {unsupported_params}, for model={model}. To drop these, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n", + ) + @staticmethod def get_optional_params_responses_api( model: str, responses_api_provider_config: BaseResponsesAPIConfig, response_api_optional_params: ResponsesAPIOptionalRequestParams, + allowed_openai_params: Optional[List[str]] = None, ) -> Dict: """ Get optional parameters for the responses API. @@ -33,25 +60,23 @@ class ResponsesAPIRequestUtils: Returns: A dictionary of supported parameters for the responses API """ - # Remove None values and internal parameters + from litellm.utils import _apply_openai_param_overrides + # Remove None values and internal parameters # Get supported parameters for the model supported_params = responses_api_provider_config.get_supported_openai_params( model ) + non_default_params = cast(Dict, response_api_optional_params) # Check for unsupported parameters - unsupported_params = [ - param - for param in response_api_optional_params - if param not in supported_params - ] - - if unsupported_params: - raise litellm.UnsupportedParamsError( - model=model, - message=f"The following parameters are not supported for model {model}: {', '.join(unsupported_params)}", - ) + ResponsesAPIRequestUtils._check_valid_arg( + supported_params=supported_params + (allowed_openai_params or []), + non_default_params=non_default_params, + drop_params=litellm.drop_params, + custom_llm_provider=responses_api_provider_config.custom_llm_provider, + model=model, + ) # Map parameters to provider-specific format mapped_params = responses_api_provider_config.map_openai_params( @@ -60,6 +85,13 @@ class ResponsesAPIRequestUtils: drop_params=litellm.drop_params, ) + # add any allowed_openai_params to the mapped_params + mapped_params = _apply_openai_param_overrides( + optional_params=mapped_params, + non_default_params=non_default_params, + allowed_openai_params=allowed_openai_params or [], + ) + return mapped_params @staticmethod @@ -75,34 +107,48 @@ class ResponsesAPIRequestUtils: Returns: ResponsesAPIOptionalRequestParams instance with only the valid parameters """ + from litellm.utils import PreProcessNonDefaultParams + valid_keys = get_type_hints(ResponsesAPIOptionalRequestParams).keys() - filtered_params = { - k: v for k, v in params.items() if k in valid_keys and v is not None - } + custom_llm_provider = params.pop("custom_llm_provider", None) + special_params = params.pop("kwargs") + + additional_drop_params = params.pop("additional_drop_params", None) + non_default_params = ( + PreProcessNonDefaultParams.base_pre_process_non_default_params( + passed_params=params, + special_params=special_params, + custom_llm_provider=custom_llm_provider, + additional_drop_params=additional_drop_params, + default_param_values={k: None for k in valid_keys}, + additional_endpoint_specific_params=["input"], + ) + ) # decode previous_response_id if it's a litellm encoded id - if "previous_response_id" in filtered_params: + if "previous_response_id" in non_default_params: decoded_previous_response_id = ResponsesAPIRequestUtils.decode_previous_response_id_to_original_previous_response_id( - filtered_params["previous_response_id"] + non_default_params["previous_response_id"] ) - filtered_params["previous_response_id"] = decoded_previous_response_id + non_default_params["previous_response_id"] = decoded_previous_response_id - if "metadata" in filtered_params: + if "metadata" in non_default_params: from litellm.utils import add_openai_metadata - filtered_params["metadata"] = add_openai_metadata( - filtered_params["metadata"] + non_default_params["metadata"] = add_openai_metadata( + non_default_params["metadata"] ) - return cast(ResponsesAPIOptionalRequestParams, filtered_params) - + return cast(ResponsesAPIOptionalRequestParams, non_default_params) + + # fmt: off @overload @staticmethod def _update_responses_api_response_id_with_model_id( responses_api_response: ResponsesAPIResponse, custom_llm_provider: Optional[str], litellm_metadata: Optional[Dict[str, Any]] = None, - ) -> ResponsesAPIResponse: + ) -> ResponsesAPIResponse: ... @overload @@ -111,9 +157,11 @@ class ResponsesAPIRequestUtils: responses_api_response: Dict[str, Any], custom_llm_provider: Optional[str], litellm_metadata: Optional[Dict[str, Any]] = None, - ) -> Dict[str, Any]: + ) -> Dict[str, Any]: ... + # fmt: on + @staticmethod def _update_responses_api_response_id_with_model_id( responses_api_response: Union[ResponsesAPIResponse, Dict[str, Any]], diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index f2f64fa3c5d..db35037801d 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -968,6 +968,10 @@ class ResponsesAPIOptionalRequestParams(TypedDict, total=False): service_tier: Optional[str] safety_identifier: Optional[str] prompt: Optional[PromptObject] + max_tool_calls: Optional[int] + prompt_cache_key: Optional[str] + stream_options: Optional[dict] + top_logprobs: Optional[int] class ResponsesAPIRequestParams(ResponsesAPIOptionalRequestParams, total=False): diff --git a/tests/llm_translation/test_optional_params.py b/tests/llm_translation/test_optional_params.py index 52f6d99fdf5..0e1e34362b0 100644 --- a/tests/llm_translation/test_optional_params.py +++ b/tests/llm_translation/test_optional_params.py @@ -1562,3 +1562,32 @@ def test_optional_params_image_gen_with_aspect_ratio(): aspect_ratio="16:9", ) assert optional_params["aspect_ratio"] == "16:9" + + +def test_optional_params_responses_api_allowed_openai_params(): + from litellm import responses + from unittest.mock import patch, MagicMock + from litellm.llms.custom_httpx.http_handler import HTTPHandler + + client = HTTPHandler() + + with patch.object(client, "post") as mock_post: + try: + response = litellm.responses( + model="openai/o1-pro", + input="Tell me a three sentence bedtime story about a unicorn.", + max_output_tokens=100, + top_logprobs=10, + allowed_openai_params=["top_logprobs"], + client=client, + ) + except Exception as e: + import traceback + + traceback.print_exc() + print("error: ", e) + + mock_post.assert_called_once() + request_body = mock_post.call_args.kwargs + print("request_body: ", request_body) + assert "top_logprobs" in request_body["json"] From ff7bdb629031c77dd40f570887f9c999565756cb Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 16 Aug 2025 00:41:11 -0700 Subject: [PATCH 2/7] fix(mistral/chat/transformation.py): handle empty message content for mistral calls Fixes https://github.com/BerriAI/litellm/issues/13355 --- litellm/llms/mistral/chat/transformation.py | 95 +++++-- .../llms/openai/chat/gpt_transformation.py | 1 + tests/llm_translation/test_mistral_api.py | 15 + .../test_mistral_chat_transformation.py | 264 ++++++++++-------- 4 files changed, 232 insertions(+), 143 deletions(-) diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index b38a4982471..dda497dac86 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -6,7 +6,18 @@ Why separate file? Make it easy to see how transformation works Docs - https://docs.mistral.ai/api/ """ -from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, cast, overload +from typing import ( + Any, + Coroutine, + List, + Literal, + Optional, + Tuple, + Union, + cast, + get_type_hints, + overload, +) import httpx @@ -145,7 +156,9 @@ class MistralConfig(OpenAIGPTConfig): for param, value in non_default_params.items(): if param == "max_tokens": optional_params["max_tokens"] = value - if param == "max_completion_tokens": # max_completion_tokens should take priority + if ( + param == "max_completion_tokens" + ): # max_completion_tokens should take priority optional_params["max_tokens"] = value if param == "tools": # Clean tools to remove problematic schema fields for Mistral API @@ -159,7 +172,9 @@ class MistralConfig(OpenAIGPTConfig): if param == "stop": optional_params["stop"] = value if param == "tool_choice" and isinstance(value, str): - optional_params["tool_choice"] = self._map_tool_choice(tool_choice=value) + optional_params["tool_choice"] = self._map_tool_choice( + tool_choice=value + ) if param == "seed": optional_params["extra_body"] = {"random_seed": value} if param == "response_format": @@ -185,7 +200,9 @@ class MistralConfig(OpenAIGPTConfig): ) # type: ignore # if api_base does not end with /v1 we add it - if api_base is not None and not api_base.endswith("/v1"): # Mistral always needs a /v1 at the end + if api_base is not None and not api_base.endswith( + "/v1" + ): # Mistral always needs a /v1 at the end api_base = api_base + "/v1" dynamic_api_key = ( api_key @@ -194,10 +211,12 @@ class MistralConfig(OpenAIGPTConfig): ) return api_base, dynamic_api_key + # fmt: off + @overload def _transform_messages( self, messages: List[AllMessageValues], model: str, is_async: Literal[True] - ) -> Coroutine[Any, Any, List[AllMessageValues]]: + ) -> Coroutine[Any, Any, List[AllMessageValues]]: ... @overload @@ -206,8 +225,9 @@ class MistralConfig(OpenAIGPTConfig): messages: List[AllMessageValues], model: str, is_async: Literal[False] = False, - ) -> List[AllMessageValues]: + ) -> List[AllMessageValues]: ... + # fmt: on def _transform_messages( self, messages: List[AllMessageValues], model: str, is_async: bool = False @@ -239,6 +259,8 @@ class MistralConfig(OpenAIGPTConfig): for m in messages: m = MistralConfig._handle_name_in_message(m) m = MistralConfig._handle_tool_call_message(m) + if MistralConfig._is_empty_assistant_message(m): + continue m = strip_none_values_from_message(m) # prevents 'extra_forbidden' error new_messages.append(m) @@ -269,20 +291,30 @@ class MistralConfig(OpenAIGPTConfig): # Handle both string and list content, preserving original format if isinstance(existing_content, str): # String content - prepend reasoning prompt - new_content: Union[str, list] = f"{reasoning_prompt}\n\n{existing_content}" + new_content: Union[str, list] = ( + f"{reasoning_prompt}\n\n{existing_content}" + ) elif isinstance(existing_content, list): # List content - prepend reasoning prompt as text block - new_content = [{"type": "text", "text": reasoning_prompt + "\n\n"}] + existing_content + new_content = [ + {"type": "text", "text": reasoning_prompt + "\n\n"} + ] + existing_content else: # Fallback for any other type - convert to string new_content = f"{reasoning_prompt}\n\n{str(existing_content)}" - messages[i] = cast(AllMessageValues, {**msg, "content": new_content}) + messages[i] = cast( + AllMessageValues, {**msg, "content": new_content} + ) break else: # Add new system message with reasoning instructions reasoning_message: AllMessageValues = cast( - AllMessageValues, {"role": "system", "content": self._get_mistral_reasoning_system_prompt()} + AllMessageValues, + { + "role": "system", + "content": self._get_mistral_reasoning_system_prompt(), + }, ) messages = [reasoning_message] + messages @@ -294,32 +326,34 @@ class MistralConfig(OpenAIGPTConfig): def _clean_tool_schema_for_mistral(cls, tools: list) -> list: """ Clean tool schemas to remove fields that cause issues with Mistral API. - + Removes: - $id and $schema fields (cause grammar validation errors) - additionalProperties=False (causes OpenAI API schema errors) - strict field (not supported by Mistral) - + Args: tools: List of tool definitions max_depth: Maximum recursion depth for schema cleaning (default: 10) - + Returns: Cleaned tools list """ if not tools: return tools - + import copy from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH from litellm.utils import _remove_json_schema_refs cleaned_tools = copy.deepcopy(tools) - + # Apply all cleaning functions with max_depth protection - cleaned_tools = _remove_json_schema_refs(cleaned_tools, max_depth=DEFAULT_MAX_RECURSE_DEPTH) - + cleaned_tools = _remove_json_schema_refs( + cleaned_tools, max_depth=DEFAULT_MAX_RECURSE_DEPTH + ) + return cleaned_tools @classmethod @@ -360,6 +394,25 @@ class MistralConfig(OpenAIGPTConfig): message["tool_calls"] = mistral_tool_calls # type: ignore return message + @classmethod + def _is_empty_assistant_message(cls, message: AllMessageValues) -> bool: + """ + Mistral API does not support empty string in assistant content. + """ + from litellm.types.llms.openai import ChatCompletionAssistantMessage + + set_keys = get_type_hints(ChatCompletionAssistantMessage).keys() + + all_expected_values_are_empty = True + for key in set_keys: + if key != "role" and message.get(key) is not None: + if key == "content" and message.get(key) == "": + continue + else: + all_expected_values_are_empty = False + break + return all_expected_values_are_empty + @staticmethod def _handle_empty_content_response(response_data: dict) -> dict: """ @@ -396,8 +449,12 @@ class MistralConfig(OpenAIGPTConfig): dict: The transformed request. Sent as the body of the API call. """ # Add reasoning system prompt if needed (for magistral models) - if "magistral" in model.lower() and optional_params.get("_add_reasoning_prompt", False): - messages = self._add_reasoning_system_prompt_if_needed(messages, optional_params) + if "magistral" in model.lower() and optional_params.get( + "_add_reasoning_prompt", False + ): + messages = self._add_reasoning_system_prompt_if_needed( + messages, optional_params + ) # Call parent transform_request which handles _transform_messages return super().transform_request( diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 396d59145ff..6e97fde3fe8 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -348,6 +348,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): for message in messages: message_content = message.get("content") message_role = message.get("role") + if ( message_role == "user" and message_content diff --git a/tests/llm_translation/test_mistral_api.py b/tests/llm_translation/test_mistral_api.py index 8cf704fbe89..04a92434208 100644 --- a/tests/llm_translation/test_mistral_api.py +++ b/tests/llm_translation/test_mistral_api.py @@ -37,3 +37,18 @@ class TestMistralCompletion(BaseLLMChatTest): def test_tool_call_no_arguments(self, tool_call_no_arguments): """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833""" pass + + +def test_empty_str_in_assistant_content(): + """Test that empty string in assistant content is converted to None.""" + litellm._turn_on_debug() + response = litellm.completion( + model="mistral/mistral-medium-latest", + messages=[ + {"role": "user", "content": "Hello, how are you?"}, + {"role": "assistant", "content": ""}, + {"role": "user", "content": "Hi again"}, + ], + max_tokens=10, + ) + assert response.choices[0].message.content is not None diff --git a/tests/test_litellm/llms/mistral/test_mistral_chat_transformation.py b/tests/test_litellm/llms/mistral/test_mistral_chat_transformation.py index 3e4d3ddb304..dd48bae3922 100644 --- a/tests/test_litellm/llms/mistral/test_mistral_chat_transformation.py +++ b/tests/test_litellm/llms/mistral/test_mistral_chat_transformation.py @@ -40,21 +40,25 @@ class TestMistralReasoningSupport: def test_get_supported_openai_params_magistral_model(self): """Test that magistral models support reasoning parameters.""" mistral_config = MistralConfig() - + # Test magistral model supports reasoning parameters - supported_params = mistral_config.get_supported_openai_params("mistral/magistral-medium-2506") + supported_params = mistral_config.get_supported_openai_params( + "mistral/magistral-medium-2506" + ) assert "reasoning_effort" in supported_params assert "thinking" in supported_params - + # Test non-magistral model doesn't include reasoning parameters - supported_params_normal = mistral_config.get_supported_openai_params("mistral/mistral-large-latest") + supported_params_normal = mistral_config.get_supported_openai_params( + "mistral/mistral-large-latest" + ) assert "reasoning_effort" not in supported_params_normal assert "thinking" not in supported_params_normal def test_map_openai_params_reasoning_effort(self): """Test that reasoning_effort parameter is properly mapped for magistral models.""" mistral_config = MistralConfig() - + # Test reasoning_effort mapping for magistral model optional_params = {} result = mistral_config.map_openai_params( @@ -63,9 +67,9 @@ class TestMistralReasoningSupport: model="mistral/magistral-medium-2506", drop_params=False, ) - + assert result.get("_add_reasoning_prompt") is True - + # Test reasoning_effort ignored for non-magistral model optional_params_normal = {} result_normal = mistral_config.map_openai_params( @@ -74,13 +78,13 @@ class TestMistralReasoningSupport: model="mistral/mistral-large-latest", drop_params=False, ) - + assert "_add_reasoning_prompt" not in result_normal def test_map_openai_params_thinking(self): """Test that thinking parameter is properly mapped for magistral models.""" mistral_config = MistralConfig() - + # Test thinking mapping for magistral model optional_params = {} result = mistral_config.map_openai_params( @@ -89,7 +93,7 @@ class TestMistralReasoningSupport: model="mistral/magistral-small-2506", drop_params=False, ) - + assert result.get("_add_reasoning_prompt") is True def test_get_mistral_reasoning_system_prompt(self): @@ -101,109 +105,123 @@ class TestMistralReasoningSupport: def test_add_reasoning_system_prompt_no_existing_system_message(self): """Test adding reasoning system prompt when no system message exists.""" mistral_config = MistralConfig() - - messages = [ - {"role": "user", "content": "What is 2+2?"} - ] + + messages = [{"role": "user", "content": "What is 2+2?"}] optional_params = {"_add_reasoning_prompt": True} - - result = mistral_config._add_reasoning_system_prompt_if_needed(messages, optional_params) - + + result = mistral_config._add_reasoning_system_prompt_if_needed( + messages, optional_params + ) + # Should add a new system message at the beginning assert len(result) == 2 assert result[0]["role"] == "system" assert "" in result[0]["content"] assert result[1]["role"] == "user" assert result[1]["content"] == "What is 2+2?" - + # Should remove the internal flag assert "_add_reasoning_prompt" not in optional_params def test_add_reasoning_system_prompt_with_existing_system_message(self): """Test adding reasoning system prompt when system message already exists.""" mistral_config = MistralConfig() - + messages = [ {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What is 2+2?"} + {"role": "user", "content": "What is 2+2?"}, ] optional_params = {"_add_reasoning_prompt": True} - - result = mistral_config._add_reasoning_system_prompt_if_needed(messages, optional_params) - + + result = mistral_config._add_reasoning_system_prompt_if_needed( + messages, optional_params + ) + # Should modify existing system message assert len(result) == 2 assert result[0]["role"] == "system" assert "" in result[0]["content"] assert "You are a helpful assistant." in result[0]["content"] assert result[1]["role"] == "user" - + # Should remove the internal flag assert "_add_reasoning_prompt" not in optional_params def test_add_reasoning_system_prompt_with_existing_list_content(self): """Test adding reasoning system prompt when system message has list content.""" mistral_config = MistralConfig() - + messages = [ { - "role": "system", + "role": "system", "content": [ {"type": "text", "text": "You are a helpful assistant."}, - {"type": "text", "text": "You always provide detailed explanations."} - ] + { + "type": "text", + "text": "You always provide detailed explanations.", + }, + ], }, - {"role": "user", "content": "What is 2+2?"} + {"role": "user", "content": "What is 2+2?"}, ] optional_params = {"_add_reasoning_prompt": True} - - result = mistral_config._add_reasoning_system_prompt_if_needed(messages, optional_params) - + + result = mistral_config._add_reasoning_system_prompt_if_needed( + messages, optional_params + ) + # Should modify existing system message preserving list format assert len(result) == 2 assert result[0]["role"] == "system" assert isinstance(result[0]["content"], list) - + # First item should be the reasoning prompt assert result[0]["content"][0]["type"] == "text" assert "" in result[0]["content"][0]["text"] - + # Original content should be preserved assert "You are a helpful assistant." in result[0]["content"][1]["text"] - assert "You always provide detailed explanations." in result[0]["content"][2]["text"] - + assert ( + "You always provide detailed explanations." + in result[0]["content"][2]["text"] + ) + assert result[1]["role"] == "user" - + # Should remove the internal flag assert "_add_reasoning_prompt" not in optional_params def test_add_reasoning_system_prompt_preserves_content_types(self): """Test that reasoning prompt preserves original content types (string vs list).""" mistral_config = MistralConfig() - + # Test with string content string_messages = [ {"role": "system", "content": "You are helpful."}, - {"role": "user", "content": "Hello"} + {"role": "user", "content": "Hello"}, ] string_params = {"_add_reasoning_prompt": True} - - string_result = mistral_config._add_reasoning_system_prompt_if_needed(string_messages, string_params) + + string_result = mistral_config._add_reasoning_system_prompt_if_needed( + string_messages, string_params + ) assert isinstance(string_result[0]["content"], str) assert "" in string_result[0]["content"] assert "You are helpful." in string_result[0]["content"] - + # Test with list content list_messages = [ { - "role": "system", - "content": [{"type": "text", "text": "You are helpful."}] + "role": "system", + "content": [{"type": "text", "text": "You are helpful."}], }, - {"role": "user", "content": "Hello"} + {"role": "user", "content": "Hello"}, ] list_params = {"_add_reasoning_prompt": True} - - list_result = mistral_config._add_reasoning_system_prompt_if_needed(list_messages, list_params) + + list_result = mistral_config._add_reasoning_system_prompt_if_needed( + list_messages, list_params + ) assert isinstance(list_result[0]["content"], list) assert list_result[0]["content"][0]["type"] == "text" assert "" in list_result[0]["content"][0]["text"] @@ -212,14 +230,14 @@ class TestMistralReasoningSupport: def test_add_reasoning_system_prompt_no_flag(self): """Test that no modification happens when _add_reasoning_prompt flag is not set.""" mistral_config = MistralConfig() - - messages = [ - {"role": "user", "content": "What is 2+2?"} - ] + + messages = [{"role": "user", "content": "What is 2+2?"}] optional_params = {} - - result = mistral_config._add_reasoning_system_prompt_if_needed(messages, optional_params) - + + result = mistral_config._add_reasoning_system_prompt_if_needed( + messages, optional_params + ) + # Should return messages unchanged assert result == messages assert len(result) == 1 @@ -227,46 +245,42 @@ class TestMistralReasoningSupport: def test_transform_request_magistral_with_reasoning(self): """Test transform_request method for magistral model with reasoning.""" mistral_config = MistralConfig() - - messages = [ - {"role": "user", "content": "What is 15 * 7?"} - ] + + messages = [{"role": "user", "content": "What is 15 * 7?"}] optional_params = {"_add_reasoning_prompt": True} - + result = mistral_config.transform_request( model="mistral/magistral-medium-2506", messages=messages, optional_params=optional_params, litellm_params={}, - headers={} + headers={}, ) - + # Should have added system message assert len(result["messages"]) == 2 assert result["messages"][0]["role"] == "system" assert "" in result["messages"][0]["content"] assert result["messages"][1]["role"] == "user" - + # Should remove internal flag from optional_params assert "_add_reasoning_prompt" not in result def test_transform_request_magistral_without_reasoning(self): """Test transform_request method for magistral model without reasoning.""" mistral_config = MistralConfig() - - messages = [ - {"role": "user", "content": "What is 15 * 7?"} - ] + + messages = [{"role": "user", "content": "What is 15 * 7?"}] optional_params = {} - + result = mistral_config.transform_request( model="mistral/magistral-medium-2506", messages=messages, optional_params=optional_params, litellm_params={}, - headers={} + headers={}, ) - + # Should not modify messages assert len(result["messages"]) == 1 assert result["messages"][0]["role"] == "user" @@ -274,20 +288,18 @@ class TestMistralReasoningSupport: def test_transform_request_non_magistral_with_reasoning_params(self): """Test that non-magistral models ignore reasoning parameters.""" mistral_config = MistralConfig() - - messages = [ - {"role": "user", "content": "What is 15 * 7?"} - ] + + messages = [{"role": "user", "content": "What is 15 * 7?"}] optional_params = {"_add_reasoning_prompt": True} - + result = mistral_config.transform_request( model="mistral/mistral-large-latest", messages=messages, optional_params=optional_params, litellm_params={}, - headers={} + headers={}, ) - + # Should not add system message for non-magistral models assert len(result["messages"]) == 1 assert result["messages"][0]["role"] == "user" @@ -295,15 +307,15 @@ class TestMistralReasoningSupport: def test_case_insensitive_magistral_detection(self): """Test that magistral model detection is case-insensitive.""" mistral_config = MistralConfig() - + # Test various case combinations models_to_test = [ "mistral/Magistral-medium-2506", "mistral/MAGISTRAL-MEDIUM-2506", "mistral/magistral-SMALL-2506", - "MaGiStRaL-medium-2506" + "MaGiStRaL-medium-2506", ] - + for model in models_to_test: supported_params = mistral_config.get_supported_openai_params(model) assert "reasoning_effort" in supported_params, f"Failed for model: {model}" @@ -311,7 +323,7 @@ class TestMistralReasoningSupport: def test_end_to_end_reasoning_workflow(self): """Test the complete workflow from parameter to system prompt injection.""" mistral_config = MistralConfig() - + # Step 1: Map parameters optional_params = {} mapped_params = mistral_config.map_openai_params( @@ -320,23 +332,21 @@ class TestMistralReasoningSupport: model="mistral/magistral-medium-2506", drop_params=False, ) - + assert mapped_params.get("_add_reasoning_prompt") is True assert mapped_params.get("temperature") == 0.7 - + # Step 2: Transform request - messages = [ - {"role": "user", "content": "Solve for x: 2x + 5 = 13"} - ] - + messages = [{"role": "user", "content": "Solve for x: 2x + 5 = 13"}] + result = mistral_config.transform_request( model="mistral/magistral-medium-2506", messages=messages, optional_params=mapped_params, litellm_params={}, - headers={} + headers={}, ) - + # Verify final result assert len(result["messages"]) == 2 assert result["messages"][0]["role"] == "system" @@ -347,7 +357,6 @@ class TestMistralReasoningSupport: assert "_add_reasoning_prompt" not in result - class TestMistralNameHandling: """Test suite for Mistral name handling in messages.""" @@ -363,7 +372,11 @@ class TestMistralNameHandling: def test_handle_name_in_message_tool_role_valid_name_keeps_name(self): """Test that valid name is kept for tool messages.""" # Test with normal function name - tool_message = {"role": "tool", "content": "Function result", "name": "get_weather"} + tool_message = { + "role": "tool", + "content": "Function result", + "name": "get_weather", + } result = MistralConfig._handle_name_in_message(tool_message) assert "name" in result assert result["name"] == "get_weather" @@ -386,26 +399,26 @@ class TestMistralParallelToolCalls: def test_get_supported_openai_params_includes_parallel_tool_calls(self): """Test that parallel_tool_calls is in supported parameters.""" mistral_config = MistralConfig() - supported_params = mistral_config.get_supported_openai_params("mistral/mistral-large-latest") + supported_params = mistral_config.get_supported_openai_params( + "mistral/mistral-large-latest" + ) assert "parallel_tool_calls" in supported_params def test_transform_request_preserves_parallel_tool_calls(self): """Test that transform_request preserves parallel_tool_calls parameter.""" mistral_config = MistralConfig() - - messages = [ - {"role": "user", "content": "What's the weather like?"} - ] + + messages = [{"role": "user", "content": "What's the weather like?"}] optional_params = {"parallel_tool_calls": True} - + result = mistral_config.transform_request( model="mistral/mistral-large-latest", messages=messages, optional_params=optional_params, litellm_params={}, - headers={} + headers={}, ) - + assert result.get("parallel_tool_calls") is True assert len(result["messages"]) == 1 assert result["messages"][0]["role"] == "user" @@ -419,17 +432,14 @@ class TestMistralEmptyContentHandling: response_data = { "choices": [ { - "message": { - "content": "", - "role": "assistant" - }, - "finish_reason": "stop" + "message": {"content": "", "role": "assistant"}, + "finish_reason": "stop", } ] } - + result = MistralConfig._handle_empty_content_response(response_data) - + assert result["choices"][0]["message"]["content"] is None def test_handle_empty_content_response_preserves_actual_content(self): @@ -439,41 +449,47 @@ class TestMistralEmptyContentHandling: { "message": { "content": "Hello, how can I help you?", - "role": "assistant" + "role": "assistant", }, - "finish_reason": "stop" + "finish_reason": "stop", } ] } - + result = MistralConfig._handle_empty_content_response(response_data) - - assert result["choices"][0]["message"]["content"] == "Hello, how can I help you?" + + assert ( + result["choices"][0]["message"]["content"] == "Hello, how can I help you?" + ) def test_handle_empty_content_response_handles_multiple_choices(self): """Test that only the first choice is processed for empty content.""" response_data = { "choices": [ { - "message": { - "content": "", - "role": "assistant" - }, - "finish_reason": "stop" + "message": {"content": "", "role": "assistant"}, + "finish_reason": "stop", }, { - "message": { - "content": "", - "role": "assistant" - }, - "finish_reason": "stop" - } + "message": {"content": "", "role": "assistant"}, + "finish_reason": "stop", + }, ] } - + result = MistralConfig._handle_empty_content_response(response_data) - + # Only first choice should be converted to None assert result["choices"][0]["message"]["content"] is None # Second choice should remain as empty string - assert result["choices"][1]["message"]["content"] is None \ No newline at end of file + assert result["choices"][1]["message"]["content"] is None + + def test_is_empty_assistant_message(self): + """Test that is_empty_assistant_message returns True for empty assistant message.""" + message = {"role": "assistant", "content": ""} + assert MistralConfig._is_empty_assistant_message(message) is True + + def test_is_empty_assistant_message_with_content(self): + """Test that is_empty_assistant_message returns False for assistant message with content.""" + message = {"role": "assistant", "content": "Hello"} + assert MistralConfig._is_empty_assistant_message(message) is False From 675d73fb9b80de90f56669b550ea22d5d9b26c9c Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 16 Aug 2025 01:16:01 -0700 Subject: [PATCH 3/7] fix(mistral/chat/transformation.py): Support new mistral thinking block Closes https://github.com/BerriAI/litellm/issues/13416 --- litellm/llms/mistral/chat/transformation.py | 64 +++++++++- litellm/types/llms/mistral.py | 10 ++ .../test_mistral_chat_transformation.py | 110 ++++++++++++++++++ 3 files changed, 181 insertions(+), 3 deletions(-) diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index dda497dac86..385057bb45a 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -28,7 +28,11 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( ) from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.secret_managers.main import get_secret_str -from litellm.types.llms.mistral import MistralToolCallMessage +from litellm.types.llms.mistral import ( + MistralTextBlock, + MistralThinkingBlock, + MistralToolCallMessage, +) from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse from litellm.utils import convert_to_model_response_object @@ -433,6 +437,58 @@ class MistralConfig(OpenAIGPTConfig): choice["message"]["content"] = None return response_data + @staticmethod + def _convert_thinking_block_to_reasoning_content( + thinking_blocks: MistralThinkingBlock, + ) -> str: + """ + Convert Mistral thinking blocks to reasoning content. + """ + return "\n".join( + [block.get("text", "") for block in thinking_blocks["thinking"]] + ) + + @staticmethod + def _handle_content_list_to_str_conversion(response_data: dict) -> dict: + """ + Handle Mistral's content list format and extract thinking content. + + Map mistral's content list to string and extract thinking blocks: + - Thinking block -> reasoning_content field + - Text block -> content field + """ + + if response_data.get("choices") and len(response_data["choices"]) > 0: + for choice in response_data["choices"]: + if choice.get("message") and choice["message"].get("content"): + content = choice["message"]["content"] + + # Only process if content is a list + if isinstance(content, list): + thinking_content = "" + text_content = "" + + # Process each content block + for block in content: + if block.get("type") == "thinking": + thinking_blocks = block.get("thinking", []) + thinking_texts = [] + for thinking_block in thinking_blocks: + if thinking_block.get("type") == "text": + thinking_texts.append( + thinking_block.get("text", "") + ) + thinking_content = "\n".join(thinking_texts) + elif block.get("type") == "text": + text_content = block.get("text", "") + + # Set the extracted content + choice["message"]["content"] = text_content + if thinking_content: + choice["message"]["reasoning_content"] = thinking_content + + return response_data + def transform_request( self, model: str, @@ -481,14 +537,16 @@ class MistralConfig(OpenAIGPTConfig): ) -> ModelResponse: """ Transform the raw response from Mistral API. - Handles Mistral-specific behavior like converting empty string content to None. + Handles Mistral-specific behavior like converting empty string content to None + and extracting thinking content from content lists. """ logging_obj.post_call(original_response=raw_response.text) logging_obj.model_call_details["response_headers"] = raw_response.headers - # Handle Mistral-specific empty string content conversion to None + # Handle Mistral-specific response transformations response_data = raw_response.json() response_data = self._handle_empty_content_response(response_data) + response_data = self._handle_content_list_to_str_conversion(response_data) final_response_obj = cast( ModelResponse, diff --git a/litellm/types/llms/mistral.py b/litellm/types/llms/mistral.py index e9563a9ae3f..375120e5376 100644 --- a/litellm/types/llms/mistral.py +++ b/litellm/types/llms/mistral.py @@ -10,3 +10,13 @@ class MistralToolCallMessage(TypedDict): id: Optional[str] type: Literal["function"] function: Optional[FunctionCall] + + +class MistralTextBlock(TypedDict): + type: Literal["text"] + text: str + + +class MistralThinkingBlock(TypedDict): + type: Literal["thinking"] + thinking: List[MistralTextBlock] diff --git a/tests/test_litellm/llms/mistral/test_mistral_chat_transformation.py b/tests/test_litellm/llms/mistral/test_mistral_chat_transformation.py index dd48bae3922..d1859c2c5ba 100644 --- a/tests/test_litellm/llms/mistral/test_mistral_chat_transformation.py +++ b/tests/test_litellm/llms/mistral/test_mistral_chat_transformation.py @@ -9,6 +9,7 @@ sys.path.insert( ) # Adds the parent directory to the system path from litellm.llms.mistral.chat.transformation import MistralConfig +from litellm.types.utils import ModelResponse @pytest.mark.asyncio @@ -424,6 +425,115 @@ class TestMistralParallelToolCalls: assert result["messages"][0]["role"] == "user" +class TestMistralThinkingContentHandling: + """Test suite for Mistral thinking content response handling functionality.""" + + def test_transform_response_with_thinking_content(self): + """Test that Mistral responses with thinking content are correctly transformed.""" + import json + from unittest.mock import Mock + + import litellm + + # Raw response from Mistral with thinking content + raw_response_data = { + "id": "12a18e1439f24f95b9812a016e0af235", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": None, + "message": { + "content": [ + { + "type": "thinking", + "thinking": [ + { + "type": "text", + "text": "Well, the capital of France is a well-known fact. It's Paris. But just to be sure, I recall that Paris is indeed the capital city of France. I don't need to look it up because it's a common knowledge fact. But if I were unsure, I would double-check using a reliable source or a knowledge base. Since I'm confident about this, I can provide the answer directly.", + } + ], + }, + {"type": "text", "text": "The capital of France is Paris."}, + ], + "refusal": None, + "role": "assistant", + "annotations": None, + "audio": None, + "function_call": None, + "tool_calls": None, + }, + } + ], + "created": 1754654178, + "model": "magistral-medium-2507", + "object": "chat.completion", + "service_tier": None, + "system_fingerprint": None, + "usage": { + "completion_tokens": 93, + "prompt_tokens": 11, + "total_tokens": 104, + "completion_tokens_details": None, + "prompt_tokens_details": None, + }, + } + + # Mock httpx response + mock_response = Mock() + mock_response.json.return_value = raw_response_data + mock_response.headers = {} + mock_response.text = json.dumps(raw_response_data) + + # Mock logging object with proper attributes + mock_logging_obj = Mock() + mock_logging_obj.model_call_details = {} + + # Test the transformation + mistral_config = MistralConfig() + model_response = litellm.ModelResponse() + + # Test transform_response method + final_response = mistral_config.transform_response( + model="mistral/magistral-medium-2507", + raw_response=mock_response, + model_response=model_response, + logging_obj=mock_logging_obj, + request_data={}, + messages=[{"role": "user", "content": "What is the capital of France?"}], + optional_params={}, + litellm_params={}, + encoding=None, + ) + + # Verify the response structure + assert final_response is not None + assert len(final_response.choices) == 1 + choice = final_response.choices[0] + + # Verify message content + message = choice.message + assert message.role == "assistant" + + # The content should be processed - either as text or as thinking blocks + # Content could be the text part or the full content list + content_str = str(message.content) if message.content else "" + + # Verify the actual text content is preserved somewhere + assert "The capital of France is Paris." in content_str or ( + hasattr(message, "thinking_blocks") and message.thinking_blocks + ) + + # Verify usage information + assert final_response.usage.completion_tokens == 93 + assert final_response.usage.prompt_tokens == 11 + assert final_response.usage.total_tokens == 104 + + # Verify model and metadata + assert final_response.id == "12a18e1439f24f95b9812a016e0af235" + assert final_response.created == 1754654178 + + class TestMistralEmptyContentHandling: """Test suite for Mistral empty content response handling functionality.""" From 5b641380ab3df4c9c0435632c2e1658c05befa08 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 16 Aug 2025 01:49:54 -0700 Subject: [PATCH 4/7] fix(openai/responses/transformation.py): update supported openai params --- .../llms/openai/responses/transformation.py | 44 +++++++------------ .../test_openai_responses_transformation.py | 34 ++++++++++---- 2 files changed, 41 insertions(+), 37 deletions(-) diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 40aae5a1cec..e70cadddaf7 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Any, Dict, Optional, Union, cast +from typing import TYPE_CHECKING, Any, Dict, Optional, Union, cast, get_type_hints import httpx from pydantic import BaseModel @@ -34,34 +34,20 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): """ All OpenAI Responses API params are supported """ - return [ - "input", - "model", - "include", - "instructions", - "max_output_tokens", - "metadata", - "parallel_tool_calls", - "previous_response_id", - "reasoning", - "store", - "background", - "stream", - "prompt", - "temperature", - "text", - "tool_choice", - "tools", - "top_p", - "truncation", - "user", - "service_tier", - "safety_identifier", - "extra_headers", - "extra_query", - "extra_body", - "timeout", - ] + supported_params = get_type_hints(ResponsesAPIRequestParams).keys() + return list( + set( + [ + "input", + "model", + "extra_headers", + "extra_query", + "extra_body", + "timeout", + ] + + list(supported_params) + ) + ) def map_openai_params( self, diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py index 1ed2266d1f1..6d46a40f6c6 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py @@ -147,7 +147,7 @@ class TestOpenAIResponsesAPIConfig: assert result.type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED assert result.response.id == "resp_123" - + @pytest.mark.serial def test_validate_environment(self): """Test that validate_environment correctly sets the Authorization header""" @@ -292,27 +292,36 @@ class TestAzureResponsesAPIConfig: def test_azure_get_complete_url_with_version_types(self): """Test Azure get_complete_url with different API version types""" base_url = "https://litellm8397336933.openai.azure.com" - + # Test with preview version - should use openai/v1/responses result_preview = self.config.get_complete_url( api_base=base_url, litellm_params={"api_version": "preview"}, ) - assert result_preview == "https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version=preview" - - # Test with latest version - should use openai/v1/responses + assert ( + result_preview + == "https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version=preview" + ) + + # Test with latest version - should use openai/v1/responses result_latest = self.config.get_complete_url( api_base=base_url, litellm_params={"api_version": "latest"}, ) - assert result_latest == "https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version=latest" - + assert ( + result_latest + == "https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version=latest" + ) + # Test with date-based version - should use openai/responses result_date = self.config.get_complete_url( api_base=base_url, litellm_params={"api_version": "2025-01-01"}, ) - assert result_date == "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2025-01-01" + assert ( + result_date + == "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2025-01-01" + ) class TestTransformListInputItemsRequest: @@ -650,3 +659,12 @@ class TestTransformListInputItemsRequest: for key, value in params.items(): assert isinstance(key, str) assert value is not None + + +def test_get_supported_openai_params(): + config = OpenAIResponsesAPIConfig() + params = config.get_supported_openai_params("gpt-4o") + assert "temperature" in params + assert "stream" in params + assert "background" in params + assert "stream" in params From 549acbc82293091e3e543eb40eeef050c798581a Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 18 Aug 2025 19:16:53 -0700 Subject: [PATCH 5/7] fix: fix test --- litellm/responses/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 2b3d178c260..ac59d28a50d 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -111,7 +111,7 @@ class ResponsesAPIRequestUtils: valid_keys = get_type_hints(ResponsesAPIOptionalRequestParams).keys() custom_llm_provider = params.pop("custom_llm_provider", None) - special_params = params.pop("kwargs") + special_params = params.pop("kwargs", {}) additional_drop_params = params.pop("additional_drop_params", None) non_default_params = ( From 45d32dc90f7479ceca90e1de7369741cfbc2a40c Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 23 Aug 2025 10:51:17 -0700 Subject: [PATCH 6/7] fix: fix ruff check --- litellm/llms/mistral/chat/transformation.py | 6 +----- litellm/responses/main.py | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index 385057bb45a..3ba925c1ba5 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -28,11 +28,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( ) from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.secret_managers.main import get_secret_str -from litellm.types.llms.mistral import ( - MistralTextBlock, - MistralThinkingBlock, - MistralToolCallMessage, -) +from litellm.types.llms.mistral import MistralThinkingBlock, MistralToolCallMessage from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse from litellm.utils import convert_to_model_response_object diff --git a/litellm/responses/main.py b/litellm/responses/main.py index ebd0ad4f42d..0d7c4f732c6 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -1,7 +1,7 @@ import asyncio import contextvars from functools import partial -from typing import Any, Coroutine, Dict, Iterable, List, Literal, Optional, Union, cast +from typing import Any, Coroutine, Dict, Iterable, List, Literal, Optional, Union import httpx From 4d7433b5ada2cc97960b9b2b03814d8415cc8309 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 23 Aug 2025 11:13:39 -0700 Subject: [PATCH 7/7] test: reduce mistral direct tests b/c of rate limit errors --- tests/llm_translation/test_mistral_api.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/tests/llm_translation/test_mistral_api.py b/tests/llm_translation/test_mistral_api.py index 04a92434208..8cf704fbe89 100644 --- a/tests/llm_translation/test_mistral_api.py +++ b/tests/llm_translation/test_mistral_api.py @@ -37,18 +37,3 @@ class TestMistralCompletion(BaseLLMChatTest): def test_tool_call_no_arguments(self, tool_call_no_arguments): """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833""" pass - - -def test_empty_str_in_assistant_content(): - """Test that empty string in assistant content is converted to None.""" - litellm._turn_on_debug() - response = litellm.completion( - model="mistral/mistral-medium-latest", - messages=[ - {"role": "user", "content": "Hello, how are you?"}, - {"role": "assistant", "content": ""}, - {"role": "user", "content": "Hi again"}, - ], - max_tokens=10, - ) - assert response.choices[0].message.content is not None