From f3749709b8743fa00d8ae86629a76811e1223c46 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 4 Aug 2025 12:01:33 -0700 Subject: [PATCH] Bug Fix - Responses API raises error with Gemini Tool Calls in `input` (#13260) * add _transform_responses_api_function_call_to_chat_completion_message * test_responses_api_with_tool_calls * TestFunctionCallTransformation * fixes for responses API testing google ai studio * TestGoogleAIStudioResponsesAPITest * test_responses_api_with_tool_calls * test_responses_api_with_tool_calls * test_basic_openai_responses_streaming_delete_endpoint --- .../transformation.py | 64 ++++- .../base_responses_api.py | 80 ++++++- .../test_google_ai_studio_responses_api.py | 20 +- .../test_litellm_completion_responses.py | 221 +++++++++++++++++- 4 files changed, 377 insertions(+), 8 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index b8cde45c611..13791666044 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -22,6 +22,7 @@ from litellm.caching import InMemoryCache from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.types.llms.openai import ( AllMessageValues, + ChatCompletionImageUrlObject, ChatCompletionResponseMessage, ChatCompletionSystemMessage, ChatCompletionToolCallChunk, @@ -40,7 +41,6 @@ from litellm.types.llms.openai import ( ResponsesAPIOptionalRequestParams, ResponsesAPIResponse, ResponseTextConfig, - ChatCompletionImageUrlObject, ) from litellm.types.responses.main import ( GenericResponseOutputItem, @@ -264,6 +264,10 @@ class LiteLLMCompletionResponsesConfig: chat_completion_messages = LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message( input_item=_input ) + + ######################################################### + # If Input Item is a Tool Call Output, add it to the tool_call_output_messages list + ######################################################### if LiteLLMCompletionResponsesConfig._is_input_item_tool_call_output( input_item=_input ): @@ -316,6 +320,11 @@ class LiteLLMCompletionResponsesConfig: return LiteLLMCompletionResponsesConfig._transform_responses_api_tool_call_output_to_chat_completion_message( tool_call_output=input_item ) + elif LiteLLMCompletionResponsesConfig._is_input_item_function_call(input_item): + # handle function call input items + return LiteLLMCompletionResponsesConfig._transform_responses_api_function_call_to_chat_completion_message( + function_call=input_item + ) else: return [ GenericChatCompletionMessage( @@ -337,6 +346,13 @@ class LiteLLMCompletionResponsesConfig: "computer_call_output", ] + @staticmethod + def _is_input_item_function_call(input_item: Any) -> bool: + """ + Check if the input item is a function call + """ + return input_item.get("type") == "function_call" + @staticmethod def _transform_responses_api_tool_call_output_to_chat_completion_message( tool_call_output: Dict[str, Any], @@ -402,6 +418,52 @@ class LiteLLMCompletionResponsesConfig: return [tool_output_message] + @staticmethod + def _transform_responses_api_function_call_to_chat_completion_message( + function_call: Dict[str, Any], + ) -> List[ + Union[ + AllMessageValues, + GenericChatCompletionMessage, + ChatCompletionResponseMessage, + ] + ]: + """ + Transform a Responses API function_call into a Chat Completion message with tool calls + + Handles Input items of this type: + function_call: + ```json + { + "type": "function_call", + "arguments":"{\"location\": \"São Paulo, Brazil\"}", + "call_id": "call_v2wlBzrlTIFl9FxPeY774GHZ", + "name": "get_weather", + "id": "fc_685c42deefc0819a822b6936faaa30be0c76bc1491ab6619", + "status": "completed" + } + ``` + """ + # Create a tool call for the function call + tool_call = ChatCompletionToolCallChunk( + id=function_call.get("call_id") or function_call.get("id") or "", + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=function_call.get("name") or "", + arguments=function_call.get("arguments") or "", + ), + index=0, + ) + + # Create an assistant message with the tool call + chat_completion_response_message = ChatCompletionResponseMessage( + tool_calls=[tool_call], + role="assistant", + content=None, # Function calls don't have content + ) + + return [chat_completion_response_message] + @staticmethod def _transform_input_file_item_to_file_item(item: Dict[str, Any]) -> Dict[str, Any]: """ diff --git a/tests/llm_responses_api_testing/base_responses_api.py b/tests/llm_responses_api_testing/base_responses_api.py index 8087e0ead3c..a4ee9a8835a 100644 --- a/tests/llm_responses_api_testing/base_responses_api.py +++ b/tests/llm_responses_api_testing/base_responses_api.py @@ -25,6 +25,9 @@ from litellm.types.llms.openai import ( ResponseAPIUsage, IncompleteDetails, ) +from openai.types.responses.response_create_params import ( + ResponseInputParam, +) from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler @@ -184,8 +187,8 @@ class BaseResponsesAPITest(ABC): # basic test assert the usage seems reasonable print("response_completed_event.response.usage=", response_completed_event.response.usage) assert response_completed_event.response.usage.input_tokens > 0 and response_completed_event.response.usage.input_tokens < 100 - assert response_completed_event.response.usage.output_tokens > 0 and response_completed_event.response.usage.output_tokens < 1000 - assert response_completed_event.response.usage.total_tokens > 0 and response_completed_event.response.usage.total_tokens < 1000 + assert response_completed_event.response.usage.output_tokens > 0 and response_completed_event.response.usage.output_tokens < 2000 + assert response_completed_event.response.usage.total_tokens > 0 and response_completed_event.response.usage.total_tokens < 2000 # total tokens should be the sum of input and output tokens assert response_completed_event.response.usage.total_tokens == response_completed_event.response.usage.input_tokens + response_completed_event.response.usage.output_tokens @@ -229,6 +232,7 @@ class BaseResponsesAPITest(ABC): @pytest.mark.parametrize("sync_mode", [True, False]) + @pytest.mark.flaky(retries=3, delay=2) @pytest.mark.asyncio async def test_basic_openai_responses_streaming_delete_endpoint(self, sync_mode): #litellm._turn_on_debug() @@ -278,6 +282,7 @@ class BaseResponsesAPITest(ABC): ) @pytest.mark.parametrize("sync_mode", [False, True]) + @pytest.mark.flaky(retries=3, delay=2) @pytest.mark.asyncio async def test_basic_openai_responses_get_endpoint(self, sync_mode): litellm._turn_on_debug() @@ -318,6 +323,7 @@ class BaseResponsesAPITest(ABC): raise ValueError("response is not a ResponsesAPIResponse") @pytest.mark.asyncio + @pytest.mark.flaky(retries=3, delay=2) async def test_basic_openai_list_input_items_endpoint(self): """Test that calls the OpenAI List Input Items endpoint""" litellm._turn_on_debug() @@ -364,3 +370,73 @@ class BaseResponsesAPITest(ABC): # assert the response is not None assert response_1 is not None assert response_2 is not None + + @pytest.mark.asyncio + async def test_responses_api_with_tool_calls(self): + """Test that calls the Responses API with tool calls including function call and output""" + litellm._turn_on_debug() + litellm.set_verbose = True + base_completion_call_args = self.get_base_completion_call_args() + + # Define the input with message, function call, and function call output + input_data: ResponseInputParam = [ + { + "type": "message", + "role": "user", + "content": "How is the weather in São Paulo today ?" + }, + { + "type": "function_call", + "arguments": "{\"location\": \"São Paulo, Brazil\"}", + "call_id": "fc_1fe70e2a-a596-45ef-b72c-9b8567c460e5", + "name": "get_weather", + "id": "fc_1fe70e2a-a596-45ef-b72c-9b8567c460e5", + "status": "completed" + }, + { + "type": "function_call_output", + "call_id": "fc_1fe70e2a-a596-45ef-b72c-9b8567c460e5", + "output": "Rainy" + } + ] + + # Define the tools + tools = [ + { + "type": "function", + "name": "get_weather", + "description": "Get current temperature for a given location.", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "City and country e.g. Bogotá, Colombia" + } + }, + "required": ["location"], + "additionalProperties": False + } + } + ] + + try: + # Make the responses API call + response = await litellm.aresponses( + input=input_data, + store=False, + tools=tools, + **base_completion_call_args + ) + except litellm.InternalServerError: + pytest.skip("Skipping test due to litellm.InternalServerError") + + print("litellm response=", json.dumps(response, indent=4, default=str)) + + # Validate the response structure + validate_responses_api_response(response, final_chunk=True) + + # Additional assertions specific to tool calls + assert response is not None + assert "output" in response + assert len(response["output"]) > 0 diff --git a/tests/llm_responses_api_testing/test_google_ai_studio_responses_api.py b/tests/llm_responses_api_testing/test_google_ai_studio_responses_api.py index 0357f8c6496..81daaea238d 100644 --- a/tests/llm_responses_api_testing/test_google_ai_studio_responses_api.py +++ b/tests/llm_responses_api_testing/test_google_ai_studio_responses_api.py @@ -5,7 +5,7 @@ from unittest.mock import patch, AsyncMock sys.path.insert(0, os.path.abspath("../..")) import litellm import json - +from base_responses_api import BaseResponsesAPITest @pytest.mark.asyncio async def test_basic_google_ai_studio_responses_api_with_tools(): litellm._turn_on_debug() @@ -85,10 +85,22 @@ async def test_mock_basic_google_ai_studio_responses_api_with_tools(): assert call_kwargs["messages"][0]["content"] == "what is the latest version of supabase python package and when was it released?" assert call_kwargs["tools"] == [] # web search tools are converted to web_search_options, not kept as tools +class TestGoogleAIStudioResponsesAPITest(BaseResponsesAPITest): + def get_base_completion_call_args(self): + #litellm._turn_on_debug() + return { + "model": "gemini/gemini-2.5-flash-lite" + } + + async def test_basic_openai_responses_delete_endpoint(self, sync_mode=False): + pass + + async def test_basic_openai_responses_streaming_delete_endpoint(self, sync_mode=False): + pass - - - + async def test_basic_openai_responses_get_endpoint(self, sync_mode=False): + pass + diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 752b6ab8f4f..00c55b9f60f 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -8,7 +8,11 @@ sys.path.insert( from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) -from litellm.types.utils import ModelResponse, Choices, Message +from litellm.types.llms.openai import ( + ChatCompletionResponseMessage, + ChatCompletionToolMessage, +) +from litellm.types.utils import Choices, Message, ModelResponse class TestLiteLLMCompletionResponsesConfig: @@ -364,3 +368,218 @@ class TestLiteLLMCompletionResponsesConfig: item for item in responses_api_response.output if item.type == "message" ] assert len(message_items) == 2, "Should have two message items" + + + + +class TestFunctionCallTransformation: + """Test cases for function_call input transformation""" + + def test_function_call_detection(self): + """Test that function_call items are correctly detected""" + function_call_item = { + "type": "function_call", + "name": "get_weather", + "arguments": '{"location": "test"}', + "call_id": "test_id" + } + + function_call_output_item = { + "type": "function_call_output", + "call_id": "test_id", + "output": "result" + } + + regular_message = { + "type": "message", + "role": "user", + "content": "Hello" + } + + # Test function_call detection + assert LiteLLMCompletionResponsesConfig._is_input_item_function_call(function_call_item) + assert not LiteLLMCompletionResponsesConfig._is_input_item_function_call(function_call_output_item) + assert not LiteLLMCompletionResponsesConfig._is_input_item_function_call(regular_message) + + # Test function_call_output detection (should still work) + assert LiteLLMCompletionResponsesConfig._is_input_item_tool_call_output(function_call_output_item) + assert not LiteLLMCompletionResponsesConfig._is_input_item_tool_call_output(function_call_item) + assert not LiteLLMCompletionResponsesConfig._is_input_item_tool_call_output(regular_message) + + def test_function_call_transformation(self): + """Test that function_call items are correctly transformed to assistant messages with tool calls""" + function_call_item = { + "type": "function_call", + "name": "get_weather", + "arguments": '{"location": "São Paulo, Brazil"}', + "call_id": "call_123", + "id": "call_123", + "status": "completed" + } + + result = LiteLLMCompletionResponsesConfig._transform_responses_api_function_call_to_chat_completion_message( + function_call=function_call_item + ) + + assert len(result) == 1 + message = result[0] + + # Should be an assistant message + assert message.get("role") == "assistant" + assert message.get("content") is None # Function calls don't have content + + # Should have tool calls + tool_calls = message.get("tool_calls", []) + assert len(tool_calls) == 1 + + tool_call = tool_calls[0] + assert tool_call.get("id") == "call_123" + assert tool_call.get("type") == "function" + + function = tool_call.get("function", {}) + assert function.get("name") == "get_weather" + assert function.get("arguments") == '{"location": "São Paulo, Brazil"}' + + def test_complete_input_transformation_with_function_calls(self): + """Test the complete transformation with the exact input from the issue""" + test_input = [ + { + "type": "message", + "role": "user", + "content": "How is the weather in São Paulo today ?" + }, + { + "type": "function_call", + "arguments": '{"location": "São Paulo, Brazil"}', + "call_id": "call_1fe70e2a-a596-45ef-b72c-9b8567c460e5", + "name": "get_weather", + "id": "call_1fe70e2a-a596-45ef-b72c-9b8567c460e5", + "status": "completed" + }, + { + "type": "function_call_output", + "call_id": "call_1fe70e2a-a596-45ef-b72c-9b8567c460e5", + "output": "Rainy" + } + ] + + # This should not raise an error (previously would raise "Invalid content type: ") + messages = LiteLLMCompletionResponsesConfig._transform_response_input_param_to_chat_completion_message( + input=test_input + ) + + assert len(messages) == 3 + + # First message: user message + user_msg = messages[0] + assert user_msg.get("role") == "user" + assert user_msg.get("content") == "How is the weather in São Paulo today ?" + + # Second message: assistant message with tool call + assistant_msg = messages[1] + assert assistant_msg.get("role") == "assistant" + assert assistant_msg.get("tool_calls") is not None + assert len(assistant_msg.get("tool_calls", [])) == 1 + + tool_call = assistant_msg.get("tool_calls")[0] + assert tool_call.get("function", {}).get("name") == "get_weather" + + # Third message: tool output + tool_msg = messages[2] + assert tool_msg.get("role") == "tool" + assert tool_msg.get("content") == "Rainy" + assert tool_msg.get("tool_call_id") == "call_1fe70e2a-a596-45ef-b72c-9b8567c460e5" + + def test_complete_request_transformation_with_function_calls(self): + """Test the complete request transformation that would be used by the responses API""" + test_input = [ + { + "type": "message", + "role": "user", + "content": "How is the weather in São Paulo today ?" + }, + { + "type": "function_call", + "arguments": '{"location": "São Paulo, Brazil"}', + "call_id": "call_1fe70e2a-a596-45ef-b72c-9b8567c460e5", + "name": "get_weather", + "id": "call_1fe70e2a-a596-45ef-b72c-9b8567c460e5", + "status": "completed" + }, + { + "type": "function_call_output", + "call_id": "call_1fe70e2a-a596-45ef-b72c-9b8567c460e5", + "output": "Rainy" + } + ] + + tools = [ + { + "type": "function", + "name": "get_weather", + "description": "Get current temperature for a given location.", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "City and country e.g. Bogotá, Colombia" + } + }, + "required": ["location"], + "additionalProperties": False + } + } + ] + + responses_api_request = { + "store": False, + "tools": tools + } + + # This should work without errors for non-OpenAI models + result = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model="gemini/gemini-2.0-flash", + input=test_input, + responses_api_request=responses_api_request + ) + + assert "messages" in result + assert "model" in result + assert "tools" in result + + messages = result["messages"] + assert len(messages) == 3 + assert result["model"] == "gemini/gemini-2.0-flash" + + # Verify the structure is correct for chat completion + user_msg = messages[0] + assert user_msg["role"] == "user" + + assistant_msg = messages[1] + assert assistant_msg["role"] == "assistant" + assert "tool_calls" in assistant_msg + + tool_msg = messages[2] + assert tool_msg["role"] == "tool" + + def test_function_call_without_call_id_fallback_to_id(self): + """Test that function_call items can use 'id' field when 'call_id' is missing""" + function_call_item = { + "type": "function_call", + "name": "get_weather", + "arguments": '{"location": "test"}', + "id": "fallback_id" # Only has 'id', not 'call_id' + } + + result = LiteLLMCompletionResponsesConfig._transform_responses_api_function_call_to_chat_completion_message( + function_call=function_call_item + ) + + assert len(result) == 1 + message = result[0] + tool_calls = message.get("tool_calls", []) + assert len(tool_calls) == 1 + + tool_call = tool_calls[0] + assert tool_call.get("id") == "fallback_id" \ No newline at end of file