scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-06 08:16:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

Merge pull request #13671 from BerriAI/litellm_dev_08_15_2025_p1

Browse source 
Responses API - support `allowed_openai_params` + Mistral - handle empty assistant content + support new mistral 'thinking' response block
This commit is contained in:
Krish Dholakia 2025-08-23 11:15:23 -07:00 • committed by GitHub
commit 06a2915e9f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 641 additions and 236 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
litellm/llms/azure/responses/transformation.py
View file

@ -6,6 +6,7 @@ from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfi
from litellm.types.llms.openai import *
from litellm.types.responses.main import *
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@ -16,6 +17,10 @@ else:
class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
@property
def custom_llm_provider(self) -> LlmProviders:
return LlmProviders.AZURE
def validate_environment(
self, headers: dict, model: str, litellm_params: Optional[GenericLiteLLMParams]
) -> dict:

6
litellm/llms/base_llm/responses/transformation.py
View file

@ -12,6 +12,7 @@ from litellm.types.llms.openai import (
)
from litellm.types.responses.main import *
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@ -29,6 +30,11 @@ class BaseResponsesAPIConfig(ABC):
def __init__(self):
pass
@property
@abstractmethod
def custom_llm_provider(self) -> LlmProviders:
pass
@classmethod
def get_config(cls):
return {

155
litellm/llms/mistral/chat/transformation.py
View file

@ -6,7 +6,18 @@ Why separate file? Make it easy to see how transformation works
Docs - https://docs.mistral.ai/api/
"""
from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, cast, overload
from typing import (
Any,
Coroutine,
List,
Literal,
Optional,
Tuple,
Union,
cast,
get_type_hints,
overload,
)
import httpx
@ -17,7 +28,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
)
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.mistral import MistralToolCallMessage
from litellm.types.llms.mistral import MistralThinkingBlock, MistralToolCallMessage
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponse
from litellm.utils import convert_to_model_response_object
@ -145,7 +156,9 @@ class MistralConfig(OpenAIGPTConfig):
for param, value in non_default_params.items():
if param == "max_tokens":
optional_params["max_tokens"] = value
if param == "max_completion_tokens": # max_completion_tokens should take priority
if (
param == "max_completion_tokens"
): # max_completion_tokens should take priority
optional_params["max_tokens"] = value
if param == "tools":
# Clean tools to remove problematic schema fields for Mistral API
@ -159,7 +172,9 @@ class MistralConfig(OpenAIGPTConfig):
if param == "stop":
optional_params["stop"] = value
if param == "tool_choice" and isinstance(value, str):
optional_params["tool_choice"] = self._map_tool_choice(tool_choice=value)
optional_params["tool_choice"] = self._map_tool_choice(
tool_choice=value
)
if param == "seed":
optional_params["extra_body"] = {"random_seed": value}
if param == "response_format":
@ -185,7 +200,9 @@ class MistralConfig(OpenAIGPTConfig):
) # type: ignore
# if api_base does not end with /v1 we add it
if api_base is not None and not api_base.endswith("/v1"): # Mistral always needs a /v1 at the end
if api_base is not None and not api_base.endswith(
"/v1"
): # Mistral always needs a /v1 at the end
api_base = api_base + "/v1"
dynamic_api_key = (
api_key
@ -194,10 +211,12 @@ class MistralConfig(OpenAIGPTConfig):
)
return api_base, dynamic_api_key
# fmt: off
@overload
def _transform_messages(
self, messages: List[AllMessageValues], model: str, is_async: Literal[True]
) -> Coroutine[Any, Any, List[AllMessageValues]]:
) -> Coroutine[Any, Any, List[AllMessageValues]]:
...
@overload
@ -206,8 +225,9 @@ class MistralConfig(OpenAIGPTConfig):
messages: List[AllMessageValues],
model: str,
is_async: Literal[False] = False,
) -> List[AllMessageValues]:
) -> List[AllMessageValues]:
...
# fmt: on
def _transform_messages(
self, messages: List[AllMessageValues], model: str, is_async: bool = False
@ -241,6 +261,8 @@ class MistralConfig(OpenAIGPTConfig):
for m in messages:
m = MistralConfig._handle_name_in_message(m)
m = MistralConfig._handle_tool_call_message(m)
if MistralConfig._is_empty_assistant_message(m):
continue
m = strip_none_values_from_message(m) # prevents 'extra_forbidden' error
new_messages.append(m)
@ -316,20 +338,30 @@ class MistralConfig(OpenAIGPTConfig):
# Handle both string and list content, preserving original format
if isinstance(existing_content, str):
# String content - prepend reasoning prompt
new_content: Union[str, list] = f"{reasoning_prompt}\n\n{existing_content}"
new_content: Union[str, list] = (
f"{reasoning_prompt}\n\n{existing_content}"
)
elif isinstance(existing_content, list):
# List content - prepend reasoning prompt as text block
new_content = [{"type": "text", "text": reasoning_prompt + "\n\n"}] + existing_content
new_content = [
{"type": "text", "text": reasoning_prompt + "\n\n"}
] + existing_content
else:
# Fallback for any other type - convert to string
new_content = f"{reasoning_prompt}\n\n{str(existing_content)}"
messages[i] = cast(AllMessageValues, {**msg, "content": new_content})
messages[i] = cast(
AllMessageValues, {**msg, "content": new_content}
)
break
else:
# Add new system message with reasoning instructions
reasoning_message: AllMessageValues = cast(
AllMessageValues, {"role": "system", "content": self._get_mistral_reasoning_system_prompt()}
AllMessageValues,
{
"role": "system",
"content": self._get_mistral_reasoning_system_prompt(),
},
)
messages = [reasoning_message] + messages
@ -341,32 +373,34 @@ class MistralConfig(OpenAIGPTConfig):
def _clean_tool_schema_for_mistral(cls, tools: list) -> list:
"""
Clean tool schemas to remove fields that cause issues with Mistral API.
Removes:
- $id and $schema fields (cause grammar validation errors)
- additionalProperties=False (causes OpenAI API schema errors)
- strict field (not supported by Mistral)
Args:
tools: List of tool definitions
max_depth: Maximum recursion depth for schema cleaning (default: 10)
Returns:
Cleaned tools list
"""
if not tools:
return tools
import copy
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
from litellm.utils import _remove_json_schema_refs
cleaned_tools = copy.deepcopy(tools)
# Apply all cleaning functions with max_depth protection
cleaned_tools = _remove_json_schema_refs(cleaned_tools, max_depth=DEFAULT_MAX_RECURSE_DEPTH)
cleaned_tools = _remove_json_schema_refs(
cleaned_tools, max_depth=DEFAULT_MAX_RECURSE_DEPTH
)
return cleaned_tools
@classmethod
@ -407,6 +441,25 @@ class MistralConfig(OpenAIGPTConfig):
message["tool_calls"] = mistral_tool_calls # type: ignore
return message
@classmethod
def _is_empty_assistant_message(cls, message: AllMessageValues) -> bool:
"""
Mistral API does not support empty string in assistant content.
"""
from litellm.types.llms.openai import ChatCompletionAssistantMessage
set_keys = get_type_hints(ChatCompletionAssistantMessage).keys()
all_expected_values_are_empty = True
for key in set_keys:
if key != "role" and message.get(key) is not None:
if key == "content" and message.get(key) == "":
continue
else:
all_expected_values_are_empty = False
break
return all_expected_values_are_empty
@staticmethod
def _handle_empty_content_response(response_data: dict) -> dict:
"""
@ -427,6 +480,58 @@ class MistralConfig(OpenAIGPTConfig):
choice["message"]["content"] = None
return response_data
@staticmethod
def _convert_thinking_block_to_reasoning_content(
thinking_blocks: MistralThinkingBlock,
) -> str:
"""
Convert Mistral thinking blocks to reasoning content.
"""
return "\n".join(
[block.get("text", "") for block in thinking_blocks["thinking"]]
)
@staticmethod
def _handle_content_list_to_str_conversion(response_data: dict) -> dict:
"""
Handle Mistral's content list format and extract thinking content.
Map mistral's content list to string and extract thinking blocks:
- Thinking block -> reasoning_content field
- Text block -> content field
"""
if response_data.get("choices") and len(response_data["choices"]) > 0:
for choice in response_data["choices"]:
if choice.get("message") and choice["message"].get("content"):
content = choice["message"]["content"]
# Only process if content is a list
if isinstance(content, list):
thinking_content = ""
text_content = ""
# Process each content block
for block in content:
if block.get("type") == "thinking":
thinking_blocks = block.get("thinking", [])
thinking_texts = []
for thinking_block in thinking_blocks:
if thinking_block.get("type") == "text":
thinking_texts.append(
thinking_block.get("text", "")
)
thinking_content = "\n".join(thinking_texts)
elif block.get("type") == "text":
text_content = block.get("text", "")
# Set the extracted content
choice["message"]["content"] = text_content
if thinking_content:
choice["message"]["reasoning_content"] = thinking_content
return response_data
def transform_request(
self,
model: str,
@ -443,8 +548,12 @@ class MistralConfig(OpenAIGPTConfig):
dict: The transformed request. Sent as the body of the API call.
"""
# Add reasoning system prompt if needed (for magistral models)
if "magistral" in model.lower() and optional_params.get("_add_reasoning_prompt", False):
messages = self._add_reasoning_system_prompt_if_needed(messages, optional_params)
if "magistral" in model.lower() and optional_params.get(
"_add_reasoning_prompt", False
):
messages = self._add_reasoning_system_prompt_if_needed(
messages, optional_params
)
# Call parent transform_request which handles _transform_messages
return super().transform_request(
@ -471,14 +580,16 @@ class MistralConfig(OpenAIGPTConfig):
) -> ModelResponse:
"""
Transform the raw response from Mistral API.
Handles Mistral-specific behavior like converting empty string content to None.
Handles Mistral-specific behavior like converting empty string content to None
and extracting thinking content from content lists.
"""
logging_obj.post_call(original_response=raw_response.text)
logging_obj.model_call_details["response_headers"] = raw_response.headers
# Handle Mistral-specific empty string content conversion to None
# Handle Mistral-specific response transformations
response_data = raw_response.json()
response_data = self._handle_empty_content_response(response_data)
response_data = self._handle_content_list_to_str_conversion(response_data)
final_response_obj = cast(
ModelResponse,

1
litellm/llms/openai/chat/gpt_transformation.py
View file

@ -348,6 +348,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
for message in messages:
message_content = message.get("content")
message_role = message.get("role")
if (
message_role == "user"
and message_content

59
litellm/llms/openai/responses/transformation.py
View file

@ -1,4 +1,4 @@
from typing import TYPE_CHECKING, Any, Dict, Optional, Union, cast
from typing import TYPE_CHECKING, Any, Dict, Optional, Union, cast, get_type_hints
import httpx
from pydantic import BaseModel
@ -13,6 +13,7 @@ from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import *
from litellm.types.responses.main import *
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
from ..common_utils import OpenAIError
@ -25,38 +26,28 @@ else:
class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
@property
def custom_llm_provider(self) -> LlmProviders:
return LlmProviders.OPENAI
def get_supported_openai_params(self, model: str) -> list:
"""
All OpenAI Responses API params are supported
"""
return [
"input",
"model",
"include",
"instructions",
"max_output_tokens",
"metadata",
"parallel_tool_calls",
"previous_response_id",
"reasoning",
"store",
"background",
"stream",
"prompt",
"temperature",
"text",
"tool_choice",
"tools",
"top_p",
"truncation",
"user",
"service_tier",
"safety_identifier",
"extra_headers",
"extra_query",
"extra_body",
"timeout",
]
supported_params = get_type_hints(ResponsesAPIRequestParams).keys()
return list(
set(
[
"input",
"model",
"extra_headers",
"extra_query",
"extra_body",
"timeout",
]
+ list(supported_params)
)
)
def map_openai_params(
self,
@ -85,8 +76,10 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
)
return final_request_params
def _validate_input_param(self, input: Union[str, ResponseInputParam]) -> Union[str, ResponseInputParam]:
def _validate_input_param(
self, input: Union[str, ResponseInputParam]
) -> Union[str, ResponseInputParam]:
"""
Ensure all input fields if pydantic are converted to dict
@ -114,7 +107,9 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
"""No transform applied since outputs are in OpenAI spec already"""
try:
raw_response_json = raw_response.json()
raw_response_json["created_at"] = _safe_convert_created_field(raw_response_json["created_at"])
raw_response_json["created_at"] = _safe_convert_created_field(
raw_response_json["created_at"]
)
except Exception:
raise OpenAIError(
message=raw_response.text, status_code=raw_response.status_code

84
litellm/responses/main.py
View file

@ -89,6 +89,7 @@ def mock_responses_api_response(
}
)
async def aresponses_api_with_mcp(
input: Union[str, ResponseInputParam],
model: str,
@ -122,7 +123,7 @@ async def aresponses_api_with_mcp(
) -> Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator]:
"""
Async version of responses API with MCP integration.
When MCP tools with server_url="litellm_proxy" are provided, this function will:
1. Get available tools from the MCP server manager
2. Insert the tools into the messages/input
@ -134,19 +135,25 @@ async def aresponses_api_with_mcp(
)
# Parse MCP tools and separate from other tools
mcp_tools_with_litellm_proxy, other_tools = LiteLLM_Proxy_MCP_Handler._parse_mcp_tools(tools)
mcp_tools_with_litellm_proxy, other_tools = (
LiteLLM_Proxy_MCP_Handler._parse_mcp_tools(tools)
)
# Get available tools from MCP manager if we have MCP tools
openai_tools = []
mcp_tools_fetched = []
if mcp_tools_with_litellm_proxy:
user_api_key_auth = kwargs.get("user_api_key_auth")
mcp_tools_fetched = await LiteLLM_Proxy_MCP_Handler._get_mcp_tools_from_manager(user_api_key_auth)
openai_tools = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai(mcp_tools_fetched)
mcp_tools_fetched = await LiteLLM_Proxy_MCP_Handler._get_mcp_tools_from_manager(
user_api_key_auth
)
openai_tools = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai(
mcp_tools_fetched
)
# Combine with other tools
all_tools = openai_tools + other_tools if (openai_tools or other_tools) else None
# Prepare call parameters for reuse
call_params = {
"include": include,
@ -172,7 +179,7 @@ async def aresponses_api_with_mcp(
"custom_llm_provider": custom_llm_provider,
**kwargs,
}
# Make initial response API call
# TODO: if should auto-execute is True, then this first response should not be streamed
response = await aresponses(
@ -180,45 +187,54 @@ async def aresponses_api_with_mcp(
model=model,
tools=all_tools,
previous_response_id=previous_response_id,
**call_params
**call_params,
)
# Check if we need to auto-execute tool calls (only for non-streaming responses)
if (mcp_tools_with_litellm_proxy and
isinstance(response, ResponsesAPIResponse) and
LiteLLM_Proxy_MCP_Handler._should_auto_execute_tools(mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy)): # type: ignore
tool_calls = LiteLLM_Proxy_MCP_Handler._extract_tool_calls_from_response(response=response)
if (
mcp_tools_with_litellm_proxy
and isinstance(response, ResponsesAPIResponse)
and LiteLLM_Proxy_MCP_Handler._should_auto_execute_tools(
mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy
)
): # type: ignore
tool_calls = LiteLLM_Proxy_MCP_Handler._extract_tool_calls_from_response(
response=response
)
if tool_calls:
user_api_key_auth = kwargs.get("litellm_metadata", {}).get("user_api_key_auth")
tool_results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls(tool_calls=tool_calls, user_api_key_auth=user_api_key_auth)
user_api_key_auth = kwargs.get("litellm_metadata", {}).get(
"user_api_key_auth"
)
tool_results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls(
tool_calls=tool_calls, user_api_key_auth=user_api_key_auth
)
if tool_results:
follow_up_input = LiteLLM_Proxy_MCP_Handler._create_follow_up_input(
response=response,
tool_results=tool_results,
original_input=input
response=response, tool_results=tool_results, original_input=input
)
final_response = await LiteLLM_Proxy_MCP_Handler._make_follow_up_call(
follow_up_input=follow_up_input,
model=model,
all_tools=all_tools,
response_id=response.id,
**call_params
**call_params,
)
# Add custom output elements to the final response
if isinstance(final_response, ResponsesAPIResponse):
final_response = LiteLLM_Proxy_MCP_Handler._add_mcp_output_elements_to_response(
response=final_response,
mcp_tools_fetched=mcp_tools_fetched,
tool_results=tool_results
final_response = (
LiteLLM_Proxy_MCP_Handler._add_mcp_output_elements_to_response(
response=final_response,
mcp_tools_fetched=mcp_tools_fetched,
tool_results=tool_results,
)
)
return final_response
return response
return response
@client
@ -319,7 +335,9 @@ async def aresponses(
)
if response is None:
raise ValueError(f"Got an unexpected None response from the Responses API: {response}")
raise ValueError(
f"Got an unexpected None response from the Responses API: {response}"
)
return response
except Exception as e:
@ -363,6 +381,7 @@ def responses(
extra_body: Optional[Dict[str, Any]] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
# LiteLLM specific params,
allowed_openai_params: Optional[List[str]] = None,
custom_llm_provider: Optional[str] = None,
**kwargs,
):
@ -373,7 +392,7 @@ def responses(
from litellm.responses.mcp.litellm_proxy_mcp_handler import (
LiteLLM_Proxy_MCP_Handler,
)
local_vars = locals()
try:
litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore
@ -445,6 +464,7 @@ def responses(
model=model,
responses_api_provider_config=responses_api_provider_config,
response_api_optional_params=response_api_optional_params,
allowed_openai_params=allowed_openai_params,
)
)

100
litellm/responses/utils.py
View file

@ -1,5 +1,5 @@
import base64
from typing import Any, Dict, Optional, Union, cast, get_type_hints, overload
from typing import Any, Dict, List, Optional, Union, cast, get_type_hints, overload
import litellm
from litellm._logging import verbose_logger
@ -16,11 +16,38 @@ from litellm.types.utils import SpecialEnums, Usage
class ResponsesAPIRequestUtils:
"""Helper utils for constructing ResponseAPI requests"""
@staticmethod
def _check_valid_arg(
supported_params: Optional[List[str]],
non_default_params: Dict,
drop_params: Optional[bool],
custom_llm_provider: Optional[str],
model: str,
):
if supported_params is None:
return
unsupported_params = {}
for k in non_default_params.keys():
if k not in supported_params:
unsupported_params[k] = non_default_params[k]
if unsupported_params:
if litellm.drop_params is True or (
drop_params is not None and drop_params is True
):
pass
else:
raise litellm.UnsupportedParamsError(
status_code=500,
message=f"{custom_llm_provider} does not support parameters: {unsupported_params}, for model={model}. To drop these, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n",
)
@staticmethod
def get_optional_params_responses_api(
model: str,
responses_api_provider_config: BaseResponsesAPIConfig,
response_api_optional_params: ResponsesAPIOptionalRequestParams,
allowed_openai_params: Optional[List[str]] = None,
) -> Dict:
"""
Get optional parameters for the responses API.
@ -33,25 +60,23 @@ class ResponsesAPIRequestUtils:
Returns:
A dictionary of supported parameters for the responses API
"""
# Remove None values and internal parameters
from litellm.utils import _apply_openai_param_overrides
# Remove None values and internal parameters
# Get supported parameters for the model
supported_params = responses_api_provider_config.get_supported_openai_params(
model
)
non_default_params = cast(Dict, response_api_optional_params)
# Check for unsupported parameters
unsupported_params = [
param
for param in response_api_optional_params
if param not in supported_params
]
if unsupported_params:
raise litellm.UnsupportedParamsError(
model=model,
message=f"The following parameters are not supported for model {model}: {', '.join(unsupported_params)}",
)
ResponsesAPIRequestUtils._check_valid_arg(
supported_params=supported_params + (allowed_openai_params or []),
non_default_params=non_default_params,
drop_params=litellm.drop_params,
custom_llm_provider=responses_api_provider_config.custom_llm_provider,
model=model,
)
# Map parameters to provider-specific format
mapped_params = responses_api_provider_config.map_openai_params(
@ -60,6 +85,13 @@ class ResponsesAPIRequestUtils:
drop_params=litellm.drop_params,
)
# add any allowed_openai_params to the mapped_params
mapped_params = _apply_openai_param_overrides(
optional_params=mapped_params,
non_default_params=non_default_params,
allowed_openai_params=allowed_openai_params or [],
)
return mapped_params
@staticmethod
@ -75,34 +107,48 @@ class ResponsesAPIRequestUtils:
Returns:
ResponsesAPIOptionalRequestParams instance with only the valid parameters
"""
from litellm.utils import PreProcessNonDefaultParams
valid_keys = get_type_hints(ResponsesAPIOptionalRequestParams).keys()
filtered_params = {
k: v for k, v in params.items() if k in valid_keys and v is not None
}
custom_llm_provider = params.pop("custom_llm_provider", None)
special_params = params.pop("kwargs", {})
additional_drop_params = params.pop("additional_drop_params", None)
non_default_params = (
PreProcessNonDefaultParams.base_pre_process_non_default_params(
passed_params=params,
special_params=special_params,
custom_llm_provider=custom_llm_provider,
additional_drop_params=additional_drop_params,
default_param_values={k: None for k in valid_keys},
additional_endpoint_specific_params=["input"],
)
)
# decode previous_response_id if it's a litellm encoded id
if "previous_response_id" in filtered_params:
if "previous_response_id" in non_default_params:
decoded_previous_response_id = ResponsesAPIRequestUtils.decode_previous_response_id_to_original_previous_response_id(
filtered_params["previous_response_id"]
non_default_params["previous_response_id"]
)
filtered_params["previous_response_id"] = decoded_previous_response_id
non_default_params["previous_response_id"] = decoded_previous_response_id
if "metadata" in filtered_params:
if "metadata" in non_default_params:
from litellm.utils import add_openai_metadata
filtered_params["metadata"] = add_openai_metadata(
filtered_params["metadata"]
non_default_params["metadata"] = add_openai_metadata(
non_default_params["metadata"]
)
return cast(ResponsesAPIOptionalRequestParams, filtered_params)
return cast(ResponsesAPIOptionalRequestParams, non_default_params)
# fmt: off
@overload
@staticmethod
def _update_responses_api_response_id_with_model_id(
responses_api_response: ResponsesAPIResponse,
custom_llm_provider: Optional[str],
litellm_metadata: Optional[Dict[str, Any]] = None,
) -> ResponsesAPIResponse:
) -> ResponsesAPIResponse:
...
@overload
@ -111,9 +157,11 @@ class ResponsesAPIRequestUtils:
responses_api_response: Dict[str, Any],
custom_llm_provider: Optional[str],
litellm_metadata: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
) -> Dict[str, Any]:
...
# fmt: on
@staticmethod
def _update_responses_api_response_id_with_model_id(
responses_api_response: Union[ResponsesAPIResponse, Dict[str, Any]],

10
litellm/types/llms/mistral.py
View file

@ -10,3 +10,13 @@ class MistralToolCallMessage(TypedDict):
id: Optional[str]
type: Literal["function"]
function: Optional[FunctionCall]
class MistralTextBlock(TypedDict):
type: Literal["text"]
text: str
class MistralThinkingBlock(TypedDict):
type: Literal["thinking"]
thinking: List[MistralTextBlock]

4
litellm/types/llms/openai.py
View file

@ -974,6 +974,10 @@ class ResponsesAPIOptionalRequestParams(TypedDict, total=False):
service_tier: Optional[str]
safety_identifier: Optional[str]
prompt: Optional[PromptObject]
max_tool_calls: Optional[int]
prompt_cache_key: Optional[str]
stream_options: Optional[dict]
top_logprobs: Optional[int]
class ResponsesAPIRequestParams(ResponsesAPIOptionalRequestParams, total=False):

29
tests/llm_translation/test_optional_params.py
View file

@ -1562,3 +1562,32 @@ def test_optional_params_image_gen_with_aspect_ratio():
aspect_ratio="16:9",
)
assert optional_params["aspect_ratio"] == "16:9"
def test_optional_params_responses_api_allowed_openai_params():
from litellm import responses
from unittest.mock import patch, MagicMock
from litellm.llms.custom_httpx.http_handler import HTTPHandler
client = HTTPHandler()
with patch.object(client, "post") as mock_post:
try:
response = litellm.responses(
model="openai/o1-pro",
input="Tell me a three sentence bedtime story about a unicorn.",
max_output_tokens=100,
top_logprobs=10,
allowed_openai_params=["top_logprobs"],
client=client,
)
except Exception as e:
import traceback
traceback.print_exc()
print("error: ", e)
mock_post.assert_called_once()
request_body = mock_post.call_args.kwargs
print("request_body: ", request_body)
assert "top_logprobs" in request_body["json"]

374
tests/test_litellm/llms/mistral/test_mistral_chat_transformation.py
View file

@ -12,6 +12,7 @@ sys.path.insert(
) # Adds the parent directory to the system path
from litellm.llms.mistral.chat.transformation import MistralConfig
from litellm.types.utils import ModelResponse
@pytest.mark.asyncio
@ -43,21 +44,25 @@ class TestMistralReasoningSupport:
def test_get_supported_openai_params_magistral_model(self):
"""Test that magistral models support reasoning parameters."""
mistral_config = MistralConfig()
# Test magistral model supports reasoning parameters
supported_params = mistral_config.get_supported_openai_params("mistral/magistral-medium-2506")
supported_params = mistral_config.get_supported_openai_params(
"mistral/magistral-medium-2506"
)
assert "reasoning_effort" in supported_params
assert "thinking" in supported_params
# Test non-magistral model doesn't include reasoning parameters
supported_params_normal = mistral_config.get_supported_openai_params("mistral/mistral-large-latest")
supported_params_normal = mistral_config.get_supported_openai_params(
"mistral/mistral-large-latest"
)
assert "reasoning_effort" not in supported_params_normal
assert "thinking" not in supported_params_normal
def test_map_openai_params_reasoning_effort(self):
"""Test that reasoning_effort parameter is properly mapped for magistral models."""
mistral_config = MistralConfig()
# Test reasoning_effort mapping for magistral model
optional_params = {}
result = mistral_config.map_openai_params(
@ -66,9 +71,9 @@ class TestMistralReasoningSupport:
model="mistral/magistral-medium-2506",
drop_params=False,
)
assert result.get("_add_reasoning_prompt") is True
# Test reasoning_effort ignored for non-magistral model
optional_params_normal = {}
result_normal = mistral_config.map_openai_params(
@ -77,13 +82,13 @@ class TestMistralReasoningSupport:
model="mistral/mistral-large-latest",
drop_params=False,
)
assert "_add_reasoning_prompt" not in result_normal
def test_map_openai_params_thinking(self):
"""Test that thinking parameter is properly mapped for magistral models."""
mistral_config = MistralConfig()
# Test thinking mapping for magistral model
optional_params = {}
result = mistral_config.map_openai_params(
@ -92,7 +97,7 @@ class TestMistralReasoningSupport:
model="mistral/magistral-small-2506",
drop_params=False,
)
assert result.get("_add_reasoning_prompt") is True
def test_get_mistral_reasoning_system_prompt(self):
@ -104,109 +109,123 @@ class TestMistralReasoningSupport:
def test_add_reasoning_system_prompt_no_existing_system_message(self):
"""Test adding reasoning system prompt when no system message exists."""
mistral_config = MistralConfig()
messages = [
{"role": "user", "content": "What is 2+2?"}
]
messages = [{"role": "user", "content": "What is 2+2?"}]
optional_params = {"_add_reasoning_prompt": True}
result = mistral_config._add_reasoning_system_prompt_if_needed(messages, optional_params)
result = mistral_config._add_reasoning_system_prompt_if_needed(
messages, optional_params
)
# Should add a new system message at the beginning
assert len(result) == 2
assert result[0]["role"] == "system"
assert "<think>" in result[0]["content"]
assert result[1]["role"] == "user"
assert result[1]["content"] == "What is 2+2?"
# Should remove the internal flag
assert "_add_reasoning_prompt" not in optional_params
def test_add_reasoning_system_prompt_with_existing_system_message(self):
"""Test adding reasoning system prompt when system message already exists."""
mistral_config = MistralConfig()
messages = [
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "What is 2+2?"}
{"role": "user", "content": "What is 2+2?"},
]
optional_params = {"_add_reasoning_prompt": True}
result = mistral_config._add_reasoning_system_prompt_if_needed(messages, optional_params)
result = mistral_config._add_reasoning_system_prompt_if_needed(
messages, optional_params
)
# Should modify existing system message
assert len(result) == 2
assert result[0]["role"] == "system"
assert "<think>" in result[0]["content"]
assert "You are a helpful assistant." in result[0]["content"]
assert result[1]["role"] == "user"
# Should remove the internal flag
assert "_add_reasoning_prompt" not in optional_params
def test_add_reasoning_system_prompt_with_existing_list_content(self):
"""Test adding reasoning system prompt when system message has list content."""
mistral_config = MistralConfig()
messages = [
{
"role": "system",
"role": "system",
"content": [
{"type": "text", "text": "You are a helpful assistant."},
{"type": "text", "text": "You always provide detailed explanations."}
]
{
"type": "text",
"text": "You always provide detailed explanations.",
},
],
},
{"role": "user", "content": "What is 2+2?"}
{"role": "user", "content": "What is 2+2?"},
]
optional_params = {"_add_reasoning_prompt": True}
result = mistral_config._add_reasoning_system_prompt_if_needed(messages, optional_params)
result = mistral_config._add_reasoning_system_prompt_if_needed(
messages, optional_params
)
# Should modify existing system message preserving list format
assert len(result) == 2
assert result[0]["role"] == "system"
assert isinstance(result[0]["content"], list)
# First item should be the reasoning prompt
assert result[0]["content"][0]["type"] == "text"
assert "<think>" in result[0]["content"][0]["text"]
# Original content should be preserved
assert "You are a helpful assistant." in result[0]["content"][1]["text"]
assert "You always provide detailed explanations." in result[0]["content"][2]["text"]
assert (
"You always provide detailed explanations."
in result[0]["content"][2]["text"]
)
assert result[1]["role"] == "user"
# Should remove the internal flag
assert "_add_reasoning_prompt" not in optional_params
def test_add_reasoning_system_prompt_preserves_content_types(self):
"""Test that reasoning prompt preserves original content types (string vs list)."""
mistral_config = MistralConfig()
# Test with string content
string_messages = [
{"role": "system", "content": "You are helpful."},
{"role": "user", "content": "Hello"}
{"role": "user", "content": "Hello"},
]
string_params = {"_add_reasoning_prompt": True}
string_result = mistral_config._add_reasoning_system_prompt_if_needed(string_messages, string_params)
string_result = mistral_config._add_reasoning_system_prompt_if_needed(
string_messages, string_params
)
assert isinstance(string_result[0]["content"], str)
assert "<think>" in string_result[0]["content"]
assert "You are helpful." in string_result[0]["content"]
# Test with list content
list_messages = [
{
"role": "system",
"content": [{"type": "text", "text": "You are helpful."}]
"role": "system",
"content": [{"type": "text", "text": "You are helpful."}],
},
{"role": "user", "content": "Hello"}
{"role": "user", "content": "Hello"},
]
list_params = {"_add_reasoning_prompt": True}
list_result = mistral_config._add_reasoning_system_prompt_if_needed(list_messages, list_params)
list_result = mistral_config._add_reasoning_system_prompt_if_needed(
list_messages, list_params
)
assert isinstance(list_result[0]["content"], list)
assert list_result[0]["content"][0]["type"] == "text"
assert "<think>" in list_result[0]["content"][0]["text"]
@ -215,14 +234,14 @@ class TestMistralReasoningSupport:
def test_add_reasoning_system_prompt_no_flag(self):
"""Test that no modification happens when _add_reasoning_prompt flag is not set."""
mistral_config = MistralConfig()
messages = [
{"role": "user", "content": "What is 2+2?"}
]
messages = [{"role": "user", "content": "What is 2+2?"}]
optional_params = {}
result = mistral_config._add_reasoning_system_prompt_if_needed(messages, optional_params)
result = mistral_config._add_reasoning_system_prompt_if_needed(
messages, optional_params
)
# Should return messages unchanged
assert result == messages
assert len(result) == 1
@ -230,46 +249,42 @@ class TestMistralReasoningSupport:
def test_transform_request_magistral_with_reasoning(self):
"""Test transform_request method for magistral model with reasoning."""
mistral_config = MistralConfig()
messages = [
{"role": "user", "content": "What is 15 * 7?"}
]
messages = [{"role": "user", "content": "What is 15 * 7?"}]
optional_params = {"_add_reasoning_prompt": True}
result = mistral_config.transform_request(
model="mistral/magistral-medium-2506",
messages=messages,
optional_params=optional_params,
litellm_params={},
headers={}
headers={},
)
# Should have added system message
assert len(result["messages"]) == 2
assert result["messages"][0]["role"] == "system"
assert "<think>" in result["messages"][0]["content"]
assert result["messages"][1]["role"] == "user"
# Should remove internal flag from optional_params
assert "_add_reasoning_prompt" not in result
def test_transform_request_magistral_without_reasoning(self):
"""Test transform_request method for magistral model without reasoning."""
mistral_config = MistralConfig()
messages = [
{"role": "user", "content": "What is 15 * 7?"}
]
messages = [{"role": "user", "content": "What is 15 * 7?"}]
optional_params = {}
result = mistral_config.transform_request(
model="mistral/magistral-medium-2506",
messages=messages,
optional_params=optional_params,
litellm_params={},
headers={}
headers={},
)
# Should not modify messages
assert len(result["messages"]) == 1
assert result["messages"][0]["role"] == "user"
@ -277,20 +292,18 @@ class TestMistralReasoningSupport:
def test_transform_request_non_magistral_with_reasoning_params(self):
"""Test that non-magistral models ignore reasoning parameters."""
mistral_config = MistralConfig()
messages = [
{"role": "user", "content": "What is 15 * 7?"}
]
messages = [{"role": "user", "content": "What is 15 * 7?"}]
optional_params = {"_add_reasoning_prompt": True}
result = mistral_config.transform_request(
model="mistral/mistral-large-latest",
messages=messages,
optional_params=optional_params,
litellm_params={},
headers={}
headers={},
)
# Should not add system message for non-magistral models
assert len(result["messages"]) == 1
assert result["messages"][0]["role"] == "user"
@ -298,15 +311,15 @@ class TestMistralReasoningSupport:
def test_case_insensitive_magistral_detection(self):
"""Test that magistral model detection is case-insensitive."""
mistral_config = MistralConfig()
# Test various case combinations
models_to_test = [
"mistral/Magistral-medium-2506",
"mistral/MAGISTRAL-MEDIUM-2506",
"mistral/magistral-SMALL-2506",
"MaGiStRaL-medium-2506"
"MaGiStRaL-medium-2506",
]
for model in models_to_test:
supported_params = mistral_config.get_supported_openai_params(model)
assert "reasoning_effort" in supported_params, f"Failed for model: {model}"
@ -314,7 +327,7 @@ class TestMistralReasoningSupport:
def test_end_to_end_reasoning_workflow(self):
"""Test the complete workflow from parameter to system prompt injection."""
mistral_config = MistralConfig()
# Step 1: Map parameters
optional_params = {}
mapped_params = mistral_config.map_openai_params(
@ -323,23 +336,21 @@ class TestMistralReasoningSupport:
model="mistral/magistral-medium-2506",
drop_params=False,
)
assert mapped_params.get("_add_reasoning_prompt") is True
assert mapped_params.get("temperature") == 0.7
# Step 2: Transform request
messages = [
{"role": "user", "content": "Solve for x: 2x + 5 = 13"}
]
messages = [{"role": "user", "content": "Solve for x: 2x + 5 = 13"}]
result = mistral_config.transform_request(
model="mistral/magistral-medium-2506",
messages=messages,
optional_params=mapped_params,
litellm_params={},
headers={}
headers={},
)
# Verify final result
assert len(result["messages"]) == 2
assert result["messages"][0]["role"] == "system"
@ -350,7 +361,6 @@ class TestMistralReasoningSupport:
assert "_add_reasoning_prompt" not in result
class TestMistralNameHandling:
"""Test suite for Mistral name handling in messages."""
@ -366,7 +376,11 @@ class TestMistralNameHandling:
def test_handle_name_in_message_tool_role_valid_name_keeps_name(self):
"""Test that valid name is kept for tool messages."""
# Test with normal function name
tool_message = {"role": "tool", "content": "Function result", "name": "get_weather"}
tool_message = {
"role": "tool",
"content": "Function result",
"name": "get_weather",
}
result = MistralConfig._handle_name_in_message(tool_message)
assert "name" in result
assert result["name"] == "get_weather"
@ -389,31 +403,140 @@ class TestMistralParallelToolCalls:
def test_get_supported_openai_params_includes_parallel_tool_calls(self):
"""Test that parallel_tool_calls is in supported parameters."""
mistral_config = MistralConfig()
supported_params = mistral_config.get_supported_openai_params("mistral/mistral-large-latest")
supported_params = mistral_config.get_supported_openai_params(
"mistral/mistral-large-latest"
)
assert "parallel_tool_calls" in supported_params
def test_transform_request_preserves_parallel_tool_calls(self):
"""Test that transform_request preserves parallel_tool_calls parameter."""
mistral_config = MistralConfig()
messages = [
{"role": "user", "content": "What's the weather like?"}
]
messages = [{"role": "user", "content": "What's the weather like?"}]
optional_params = {"parallel_tool_calls": True}
result = mistral_config.transform_request(
model="mistral/mistral-large-latest",
messages=messages,
optional_params=optional_params,
litellm_params={},
headers={}
headers={},
)
assert result.get("parallel_tool_calls") is True
assert len(result["messages"]) == 1
assert result["messages"][0]["role"] == "user"
class TestMistralThinkingContentHandling:
"""Test suite for Mistral thinking content response handling functionality."""
def test_transform_response_with_thinking_content(self):
"""Test that Mistral responses with thinking content are correctly transformed."""
import json
from unittest.mock import Mock
import litellm
# Raw response from Mistral with thinking content
raw_response_data = {
"id": "12a18e1439f24f95b9812a016e0af235",
"choices": [
{
"finish_reason": "stop",
"index": 0,
"logprobs": None,
"message": {
"content": [
{
"type": "thinking",
"thinking": [
{
"type": "text",
"text": "Well, the capital of France is a well-known fact. It's Paris. But just to be sure, I recall that Paris is indeed the capital city of France. I don't need to look it up because it's a common knowledge fact. But if I were unsure, I would double-check using a reliable source or a knowledge base. Since I'm confident about this, I can provide the answer directly.",
}
],
},
{"type": "text", "text": "The capital of France is Paris."},
],
"refusal": None,
"role": "assistant",
"annotations": None,
"audio": None,
"function_call": None,
"tool_calls": None,
},
}
],
"created": 1754654178,
"model": "magistral-medium-2507",
"object": "chat.completion",
"service_tier": None,
"system_fingerprint": None,
"usage": {
"completion_tokens": 93,
"prompt_tokens": 11,
"total_tokens": 104,
"completion_tokens_details": None,
"prompt_tokens_details": None,
},
}
# Mock httpx response
mock_response = Mock()
mock_response.json.return_value = raw_response_data
mock_response.headers = {}
mock_response.text = json.dumps(raw_response_data)
# Mock logging object with proper attributes
mock_logging_obj = Mock()
mock_logging_obj.model_call_details = {}
# Test the transformation
mistral_config = MistralConfig()
model_response = litellm.ModelResponse()
# Test transform_response method
final_response = mistral_config.transform_response(
model="mistral/magistral-medium-2507",
raw_response=mock_response,
model_response=model_response,
logging_obj=mock_logging_obj,
request_data={},
messages=[{"role": "user", "content": "What is the capital of France?"}],
optional_params={},
litellm_params={},
encoding=None,
)
# Verify the response structure
assert final_response is not None
assert len(final_response.choices) == 1
choice = final_response.choices[0]
# Verify message content
message = choice.message
assert message.role == "assistant"
# The content should be processed - either as text or as thinking blocks
# Content could be the text part or the full content list
content_str = str(message.content) if message.content else ""
# Verify the actual text content is preserved somewhere
assert "The capital of France is Paris." in content_str or (
hasattr(message, "thinking_blocks") and message.thinking_blocks
)
# Verify usage information
assert final_response.usage.completion_tokens == 93
assert final_response.usage.prompt_tokens == 11
assert final_response.usage.total_tokens == 104
# Verify model and metadata
assert final_response.id == "12a18e1439f24f95b9812a016e0af235"
assert final_response.created == 1754654178
class TestMistralEmptyContentHandling:
"""Test suite for Mistral empty content response handling functionality."""
@ -422,17 +545,14 @@ class TestMistralEmptyContentHandling:
response_data = {
"choices": [
{
"message": {
"content": "",
"role": "assistant"
},
"finish_reason": "stop"
"message": {"content": "", "role": "assistant"},
"finish_reason": "stop",
}
]
}
result = MistralConfig._handle_empty_content_response(response_data)
assert result["choices"][0]["message"]["content"] is None
def test_handle_empty_content_response_preserves_actual_content(self):
@ -442,45 +562,51 @@ class TestMistralEmptyContentHandling:
{
"message": {
"content": "Hello, how can I help you?",
"role": "assistant"
"role": "assistant",
},
"finish_reason": "stop"
"finish_reason": "stop",
}
]
}
result = MistralConfig._handle_empty_content_response(response_data)
assert result["choices"][0]["message"]["content"] == "Hello, how can I help you?"
assert (
result["choices"][0]["message"]["content"] == "Hello, how can I help you?"
)
def test_handle_empty_content_response_handles_multiple_choices(self):
"""Test that only the first choice is processed for empty content."""
response_data = {
"choices": [
{
"message": {
"content": "",
"role": "assistant"
},
"finish_reason": "stop"
"message": {"content": "", "role": "assistant"},
"finish_reason": "stop",
},
{
"message": {
"content": "",
"role": "assistant"
},
"finish_reason": "stop"
}
"message": {"content": "", "role": "assistant"},
"finish_reason": "stop",
},
]
}
result = MistralConfig._handle_empty_content_response(response_data)
# Only first choice should be converted to None
assert result["choices"][0]["message"]["content"] is None
# Second choice should remain as empty string
assert result["choices"][1]["message"]["content"] is None
def test_is_empty_assistant_message(self):
"""Test that is_empty_assistant_message returns True for empty assistant message."""
message = {"role": "assistant", "content": ""}
assert MistralConfig._is_empty_assistant_message(message) is True
def test_is_empty_assistant_message_with_content(self):
"""Test that is_empty_assistant_message returns False for assistant message with content."""
message = {"role": "assistant", "content": "Hello"}
assert MistralConfig._is_empty_assistant_message(message) is False
class TestMistralFileHandling:
"""Test suite for Mistral file handling functionality."""
@ -551,4 +677,4 @@ class TestMistralFileHandling:
assert result[0]["content"][2]["type"] == "file"
# Check that file_ids are modified to match Mistral's expected format
assert result[0]["content"][1]["file_id"] == "file-12345" # type: ignore
assert result[0]["content"][2]["file_id"] == "file-67890" # type: ignore
assert result[0]["content"][2]["file_id"] == "file-67890" # type: ignore

50
tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py
View file

@ -283,6 +283,47 @@ class TestOpenAIResponsesAPIConfig:
assert result.type == "test"
class TestAzureResponsesAPIConfig:
def setup_method(self):
self.config = AzureOpenAIResponsesAPIConfig()
self.model = "gpt-4o"
self.logging_obj = MagicMock()
def test_azure_get_complete_url_with_version_types(self):
"""Test Azure get_complete_url with different API version types"""
base_url = "https://litellm8397336933.openai.azure.com"
# Test with preview version - should use openai/v1/responses
result_preview = self.config.get_complete_url(
api_base=base_url,
litellm_params={"api_version": "preview"},
)
assert (
result_preview
== "https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version=preview"
)
# Test with latest version - should use openai/v1/responses
result_latest = self.config.get_complete_url(
api_base=base_url,
litellm_params={"api_version": "latest"},
)
assert (
result_latest
== "https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version=latest"
)
# Test with date-based version - should use openai/responses
result_date = self.config.get_complete_url(
api_base=base_url,
litellm_params={"api_version": "2025-01-01"},
)
assert (
result_date
== "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2025-01-01"
)
class TestTransformListInputItemsRequest:
"""Test suite for transform_list_input_items_request function"""
@ -618,3 +659,12 @@ class TestTransformListInputItemsRequest:
for key, value in params.items():
assert isinstance(key, str)
assert value is not None
def test_get_supported_openai_params():
config = OpenAIResponsesAPIConfig()
params = config.get_supported_openai_params("gpt-4o")
assert "temperature" in params
assert "stream" in params
assert "background" in params
assert "stream" in params