diff --git a/litellm/llms/deepinfra/chat/transformation.py b/litellm/llms/deepinfra/chat/transformation.py index 09cdabcdd82..5198260a24b 100644 --- a/litellm/llms/deepinfra/chat/transformation.py +++ b/litellm/llms/deepinfra/chat/transformation.py @@ -1,9 +1,11 @@ -from typing import Optional, Tuple, Union +import json +from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, cast, overload import litellm from litellm.constants import MIN_NON_ZERO_TEMPERATURE from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import AllMessageValues class DeepInfraConfig(OpenAIGPTConfig): @@ -117,6 +119,79 @@ class DeepInfraConfig(OpenAIGPTConfig): optional_params[param] = value return optional_params + def _transform_tool_message_content(self, messages: List[AllMessageValues]) -> List[AllMessageValues]: + """ + Transform tool message content from array to string format for DeepInfra compatibility. + + DeepInfra requires tool message content to be a string, not an array. + This method converts tool message content from array format to string format. + + Example transformation: + - Input: {"role": "tool", "content": [{"type": "text", "text": "20"}]} + - Output: {"role": "tool", "content": "20"} + + Or if content is complex: + - Input: {"role": "tool", "content": [{"type": "text", "text": "result"}]} + - Output: {"role": "tool", "content": "[{\"type\": \"text\", \"text\": \"result\"}]"} + """ + for message in messages: + if message.get("role") == "tool": + content = message.get("content") + + # If content is a list/array, convert it to string + if isinstance(content, list): + # Check if it's a simple single text item + if ( + len(content) == 1 + and isinstance(content[0], dict) + and content[0].get("type") == "text" + and "text" in content[0] + ): + # Extract just the text value for simple cases + message["content"] = content[0]["text"] + else: + # For complex content, serialize the entire array as JSON string + message["content"] = json.dumps(content) + + return messages + + @overload + def _transform_messages( + self, messages: List[AllMessageValues], model: str, is_async: Literal[True] + ) -> Coroutine[Any, Any, List[AllMessageValues]]: + ... + + @overload + def _transform_messages( + self, messages: List[AllMessageValues], model: str, is_async: Literal[False] = False + ) -> List[AllMessageValues]: + ... + + def _transform_messages( + self, messages: List[AllMessageValues], model: str, is_async: bool = False + ) -> Union[List[AllMessageValues], Coroutine[Any, Any, List[AllMessageValues]]]: + """ + Transform messages for DeepInfra compatibility. + Handles both sync and async transformations. + """ + if is_async: + # For async case, create an async function that awaits parent and applies our transformation + async def _async_transform(): + # Call parent with is_async=True (literal) for async case + parent_result = super(DeepInfraConfig, self)._transform_messages( + messages=messages, model=model, is_async=cast(Literal[True], True) + ) + transformed_messages = await parent_result + return self._transform_tool_message_content(transformed_messages) + return _async_transform() + else: + # Call parent with is_async=False (literal) for sync case + parent_result = super()._transform_messages( + messages=messages, model=model, is_async=cast(Literal[False], False) + ) + # For sync case, parent_result is already the transformed messages + return self._transform_tool_message_content(parent_result) + def _get_openai_compatible_provider_info( self, api_base: Optional[str], api_key: Optional[str] ) -> Tuple[Optional[str], Optional[str]]: diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index 4c947b99ba3..642cb0cec2d 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -218,10 +218,10 @@ if MCP_AVAILABLE: from fastapi import HTTPException from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException - from litellm.proxy.proxy_server import add_litellm_data_to_request, proxy_config from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) + from litellm.proxy.proxy_server import add_litellm_data_to_request, proxy_config try: data = await request.json() @@ -252,7 +252,12 @@ if MCP_AVAILABLE: if mcp_server_auth_headers: data["mcp_server_auth_headers"] = mcp_server_auth_headers data["raw_headers"] = raw_headers_from_request - + + # Extract user_api_key_auth from metadata and add to top level + # call_mcp_tool expects user_api_key_auth as a top-level parameter + if "metadata" in data and "user_api_key_auth" in data["metadata"]: + data["user_api_key_auth"] = data["metadata"]["user_api_key_auth"] + result = await call_mcp_tool(**data) return result except BlockedPiiEntityError as e: diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index d1a78534dae..f16c115fed3 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1195,7 +1195,7 @@ class ProxyLogging: and _callback.__class__.async_pre_call_hook != CustomLogger.async_pre_call_hook ): - if call_type == "mcp_call" and user_api_key_dict is None: + if call_type == "call_mcp_tool" and user_api_key_dict is None: continue response = await _callback.async_pre_call_hook( diff --git a/tests/test_litellm/llms/deepinfra/test_deepinfra_chat_transformation.py b/tests/test_litellm/llms/deepinfra/test_deepinfra_chat_transformation.py index fc8cf6dc60f..49d55f920b5 100644 --- a/tests/test_litellm/llms/deepinfra/test_deepinfra_chat_transformation.py +++ b/tests/test_litellm/llms/deepinfra/test_deepinfra_chat_transformation.py @@ -24,3 +24,194 @@ def test_deepseek_supported_openai_params(): supported_openai_params = DeepInfraConfig().get_supported_openai_params(model="deepinfra/deepseek-ai/DeepSeek-V3.1") print(supported_openai_params) assert "reasoning_effort" in supported_openai_params + + +def test_deepinfra_tool_message_content_transformation(): + """ + Test that DeepInfra transforms tool message content from array to string. + + This fixes the issue where LibreChat sends tool messages with content as an array: + {"role": "tool", "content": [{"type": "text", "text": "20"}]} + + DeepInfra requires content to be a string, so we transform it to: + {"role": "tool", "content": "20"} + + Related to issue #13982 + """ + from litellm.llms.deepinfra.chat.transformation import DeepInfraConfig + + config = DeepInfraConfig() + + # Test case 1: Simple single text item in array (common case from LibreChat) + messages_with_array_content = [ + { + "role": "user", + "content": "Calculate 10 + 10" + }, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_123", + "type": "function", + "function": { + "name": "calculator", + "arguments": '{"input": "10 + 10"}' + } + } + ] + }, + { + "role": "tool", + "tool_call_id": "call_123", + "name": "calculator", + "content": [{"type": "text", "text": "20"}] # Array format from LibreChat + } + ] + + transformed_messages = config._transform_messages( + messages=messages_with_array_content, + model="deepinfra/Qwen/Qwen3-235B-A22B" + ) + + # Verify the tool message content was converted to string + tool_message = transformed_messages[2] + assert tool_message["role"] == "tool" + assert isinstance(tool_message["content"], str) + assert tool_message["content"] == "20" + print(f"✓ Test case 1 passed: {tool_message['content']}") + + # Test case 2: Complex content array (multiple items) + messages_with_complex_content = [ + { + "role": "user", + "content": "Test" + }, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_456", + "type": "function", + "function": {"name": "test", "arguments": "{}"} + } + ] + }, + { + "role": "tool", + "tool_call_id": "call_456", + "content": [ + {"type": "text", "text": "Result 1"}, + {"type": "text", "text": "Result 2"} + ] + } + ] + + transformed_messages_complex = config._transform_messages( + messages=messages_with_complex_content, + model="deepinfra/Qwen/Qwen3-235B-A22B" + ) + + tool_message_complex = transformed_messages_complex[2] + assert tool_message_complex["role"] == "tool" + assert isinstance(tool_message_complex["content"], str) + # For complex content, it should be JSON stringified + parsed_content = json.loads(tool_message_complex["content"]) + assert len(parsed_content) == 2 + assert parsed_content[0]["text"] == "Result 1" + print(f"✓ Test case 2 passed: {tool_message_complex['content']}") + + # Test case 3: Tool message with string content (should remain unchanged) + messages_with_string_content = [ + { + "role": "user", + "content": "Test" + }, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_789", + "type": "function", + "function": {"name": "test", "arguments": "{}"} + } + ] + }, + { + "role": "tool", + "tool_call_id": "call_789", + "content": "Simple string result" # Already a string + } + ] + + transformed_messages_string = config._transform_messages( + messages=messages_with_string_content, + model="deepinfra/Qwen/Qwen3-235B-A22B" + ) + + tool_message_string = transformed_messages_string[2] + assert tool_message_string["role"] == "tool" + assert isinstance(tool_message_string["content"], str) + assert tool_message_string["content"] == "Simple string result" + print(f"✓ Test case 3 passed: {tool_message_string['content']}") + + print("\n✅ All DeepInfra tool message transformation tests passed!") + + +@pytest.mark.asyncio +async def test_deepinfra_tool_message_content_transformation_async(): + """ + Test that DeepInfra transforms tool message content from array to string in async mode. + + This ensures the async path works correctly when is_async=True. + + Related to issue #13982 + """ + from litellm.llms.deepinfra.chat.transformation import DeepInfraConfig + + config = DeepInfraConfig() + + # Test async transformation with tool message containing array content + messages_with_array_content = [ + { + "role": "user", + "content": "Calculate 10 + 10" + }, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_123", + "type": "function", + "function": { + "name": "calculator", + "arguments": '{"input": "10 + 10"}' + } + } + ] + }, + { + "role": "tool", + "tool_call_id": "call_123", + "name": "calculator", + "content": [{"type": "text", "text": "20"}] # Array format from LibreChat + } + ] + + # Call with is_async=True + transformed_messages = await config._transform_messages( + messages=messages_with_array_content, + model="deepinfra/Qwen/Qwen3-235B-A22B", + is_async=True + ) + + # Verify the tool message content was converted to string + tool_message = transformed_messages[2] + assert tool_message["role"] == "tool" + assert isinstance(tool_message["content"], str) + assert tool_message["content"] == "20" + print(f"✓ Async test passed: {tool_message['content']}") + + print("\n✅ DeepInfra async tool message transformation test passed!")