From 6d102ea5599a1ecf9c8fb823a88a18fc8131cd0c Mon Sep 17 00:00:00 2001 From: Tin Date: Tue, 21 Jul 2026 16:11:35 -0700 Subject: [PATCH] fix(litellm): bridge gpt-5.4+ chat requests with tools when reasoning defaults on OpenAI enables reasoning by default for gpt-5.4+ (unset reasoning_effort means medium server-side) and Chat Completions rejects function tools whenever reasoning is on, so a tools request without an explicit reasoning_effort 400d instead of auto-bridging to the Responses API; the bridge heuristic now treats unset effort as reasoning-active and honors the documented escape hatch by keeping explicit "none" on chat completions. The cursor input arm also gains the mirror of the messages-arm normalization: chat-nested tool envelopes, grammar formats, and object tool_choice flatten to the Responses dialect before dispatch --- litellm/main.py | 13 +- .../proxy/response_api_endpoints/endpoints.py | 49 +++++++ .../response_api_endpoints/test_endpoints.py | 136 ++++++++++++++++++ tests/test_litellm/test_main.py | 72 +++++++++- 4 files changed, 262 insertions(+), 8 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index acdec7385da..43d021ebe8b 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1022,7 +1022,12 @@ def responses_api_bridge_check( # ``reasoningSummary`` in ``extra_body``) must be bridged; Chat Completions rejects # those keys. # - # - gpt-5.4+: tools + reasoning_effort (original) or any reasoning-summary alias. + # - gpt-5.4+: function tools with reasoning active must be bridged. OpenAI enables + # reasoning by default for these models (unset reasoning_effort means medium + # server-side), and Chat Completions rejects tools whenever reasoning is on + # ("Function tools with reasoning_effort are not supported ... use /v1/responses + # or set reasoning_effort to 'none'"), so only an explicit ``"none"`` keeps the + # request chat-servable. # - Older GPT-5 names (e.g. ``gpt-5``, ``gpt-5.1``): bridge only when a reasoning # summary alias is present with ``reasoning_effort`` (tools alone stay on chat). if ( @@ -1030,8 +1035,10 @@ def responses_api_bridge_check( and model_info.get("mode") != "responses" and OpenAIGPT5Config.is_model_gpt_5_model(model) and not OpenAIGPT5Config.is_model_gpt_5_search_model(model) - and reasoning_effort is not None - and (reasoning_summary is not None or (OpenAIGPT5Config.is_model_gpt_5_4_plus_model(model) and tools)) + and ( + (reasoning_effort is not None and reasoning_summary is not None) + or (OpenAIGPT5Config.is_model_gpt_5_4_plus_model(model) and tools and reasoning_effort != "none") + ) ): model_info["mode"] = "responses" model = model.replace("responses/", "") diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index a980f85d406..8a678601284 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -66,6 +66,47 @@ def _nest_flat_chat_tool_choice(tool_choice: object) -> object: return tool_choice +def _flatten_chat_tool_for_responses(tool: object) -> object: + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + convert_custom_tool_format_to_responses_shape, + ) + + if not isinstance(tool, dict): + return tool + if tool.get("type") == "custom": + if isinstance(tool.get("custom"), dict): + payload = {k: tool["custom"][k] for k in _FLAT_CUSTOM_TOOL_KEYS if k in tool["custom"]} + elif "name" in tool: + payload = {k: tool[k] for k in _FLAT_CUSTOM_TOOL_KEYS if k in tool} + else: + return tool + if isinstance(payload.get("format"), dict): + payload = {**payload, "format": convert_custom_tool_format_to_responses_shape(payload["format"])} + return {"type": "custom", **payload} + if tool.get("type") == "function" and isinstance(tool.get("function"), dict): + return { + "type": "function", + **{k: tool["function"][k] for k in _FLAT_FUNCTION_TOOL_KEYS if k in tool["function"]}, + } + return tool + + +def _flatten_chat_tools_for_responses(tools: list) -> list: + return [_flatten_chat_tool_for_responses(tool) for tool in tools] + + +def _flatten_chat_tool_choice_for_responses(tool_choice: object) -> object: + if not isinstance(tool_choice, dict): + return tool_choice + choice_type = tool_choice.get("type") + if choice_type not in ("custom", "function"): + return tool_choice + nested = tool_choice.get(choice_type) + if isinstance(nested, dict) and isinstance(nested.get("name"), str): + return {"type": choice_type, "name": nested["name"]} + return tool_choice + + @router.post( "/v1/responses", dependencies=[Depends(user_api_key_auth)], @@ -444,6 +485,14 @@ async def cursor_chat_completions( # cache's key snapshot so later readers get an empty body data = {key: value for key, value in data.items() if key != "stream_options"} + tools = data.get("tools") + if isinstance(tools, list): + data = {**data, "tools": _flatten_chat_tools_for_responses(tools)} + tool_choice = data.get("tool_choice") + flattened_tool_choice = _flatten_chat_tool_choice_for_responses(tool_choice) + if flattened_tool_choice != tool_choice: + data = {**data, "tool_choice": flattened_tool_choice} + processor = ProxyBaseLLMRequestProcessing(data=data) def cursor_data_generator(response, user_api_key_dict, request_data, request=None): diff --git a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py index b8699d6ef8c..04f027ac4bb 100644 --- a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py @@ -1151,3 +1151,139 @@ class TestNestFlatChatToolChoice: 42, ): assert _nest_flat_chat_tool_choice(unchanged) == unchanged + + +class TestFlattenChatToolsForResponsesInputArm: + """ + Mirror of TestNestFlatChatToolShapeMatrix for the input arm: chat-nested shapes in a + Responses-shaped body must flatten to the Responses dialect, per level, idempotently. + """ + + FLAT_GRAMMAR = {"type": "grammar", "definition": "start: patch", "syntax": "lark"} + NESTED_GRAMMAR = {"type": "grammar", "grammar": {"definition": "start: patch", "syntax": "lark"}} + + @pytest.mark.parametrize("envelope", ["flat", "nested"]) + @pytest.mark.parametrize("format_shape", ["absent", "text", "flat_grammar", "nested_grammar"]) + def test_every_envelope_and_format_combination_lands_flat(self, envelope, format_shape): + from litellm.proxy.response_api_endpoints.endpoints import _flatten_chat_tools_for_responses + + format_value = { + "absent": None, + "text": {"type": "text"}, + "flat_grammar": self.FLAT_GRAMMAR, + "nested_grammar": self.NESTED_GRAMMAR, + }[format_shape] + payload = {"name": "ApplyPatch", "description": "V4A patch"} + if format_value is not None: + payload["format"] = format_value + tool = {"type": "custom", "custom": payload} if envelope == "nested" else {"type": "custom", **payload} + + canonical = {"type": "custom", "name": "ApplyPatch", "description": "V4A patch"} + if format_shape in ("flat_grammar", "nested_grammar"): + canonical["format"] = self.FLAT_GRAMMAR + elif format_shape == "text": + canonical["format"] = {"type": "text"} + + assert _flatten_chat_tools_for_responses([tool]) == [canonical] + + def test_nested_function_tool_is_flattened_and_flat_passes_through(self): + from litellm.proxy.response_api_endpoints.endpoints import _flatten_chat_tools_for_responses + + nested = {"type": "function", "function": {"name": "read_file", "parameters": {"type": "object"}}} + flat = {"type": "function", "name": "read_file", "parameters": {"type": "object"}} + assert _flatten_chat_tools_for_responses([nested]) == [flat] + assert _flatten_chat_tools_for_responses([flat]) == [flat] + + def test_unrecognized_entries_pass_through(self): + from litellm.proxy.response_api_endpoints.endpoints import _flatten_chat_tools_for_responses + + entries = [{"type": "web_search"}, {"type": "custom"}, "junk", None, {}] + assert _flatten_chat_tools_for_responses(entries) == entries + + +class TestFlattenChatToolChoiceForResponsesInputArm: + def test_nested_custom_and_function_tool_choice_flatten(self): + from litellm.proxy.response_api_endpoints.endpoints import _flatten_chat_tool_choice_for_responses + + assert _flatten_chat_tool_choice_for_responses({"type": "custom", "custom": {"name": "ApplyPatch"}}) == { + "type": "custom", + "name": "ApplyPatch", + } + assert _flatten_chat_tool_choice_for_responses({"type": "function", "function": {"name": "f"}}) == { + "type": "function", + "name": "f", + } + + def test_flat_and_string_tool_choice_pass_through(self): + from litellm.proxy.response_api_endpoints.endpoints import _flatten_chat_tool_choice_for_responses + + for unchanged in ("auto", "required", None, {"type": "custom", "name": "x"}, {"type": "auto"}, 42): + assert _flatten_chat_tool_choice_for_responses(unchanged) == unchanged + + +class TestCursorInputArmFlattening: + @pytest.mark.asyncio + async def test_nested_chat_shapes_in_input_body_reach_aresponses_flattened(self): + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from openai.types.responses import ResponseOutputMessage, ResponseOutputText + + from litellm.types.llms.openai import ResponsesAPIResponse + + mock_response = ResponsesAPIResponse( + id="resp_flat123", + created_at=1234567890, + model="gpt-5.6", + object="response", + output=[ + ResponseOutputMessage( + id="msg_flat123", + type="message", + role="assistant", + status="completed", + content=[ResponseOutputText(type="output_text", text="ok", annotations=[])], + ) + ], + ) + + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(api_key="sk-1234") + try: + with patch("litellm.proxy.proxy_server.llm_router") as mock_router: + mock_router.aresponses = AsyncMock(return_value=mock_response) + client = TestClient(app) + response = client.post( + "/cursor/chat/completions", + json={ + "model": "gpt-5.6", + "input": [{"role": "user", "content": "use ApplyPatch"}], + "tools": [ + { + "type": "custom", + "custom": { + "name": "ApplyPatch", + "format": { + "type": "grammar", + "grammar": {"definition": "start: patch", "syntax": "lark"}, + }, + }, + }, + {"type": "function", "name": "read_file", "parameters": {"type": "object"}}, + ], + "tool_choice": {"type": "custom", "custom": {"name": "ApplyPatch"}}, + }, + headers={"Authorization": "Bearer sk-1234"}, + ) + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + assert response.status_code == 200 + call_kwargs = mock_router.aresponses.call_args.kwargs + assert call_kwargs["tools"] == [ + { + "type": "custom", + "name": "ApplyPatch", + "format": {"type": "grammar", "definition": "start: patch", "syntax": "lark"}, + }, + {"type": "function", "name": "read_file", "parameters": {"type": "object"}}, + ] + assert call_kwargs["tool_choice"] == {"type": "custom", "name": "ApplyPatch"} diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index 4611aafa3c1..b4f94177f7d 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -810,8 +810,12 @@ def test_responses_api_bridge_check_azure_gpt_5_4_tools_plus_reasoning_routes_to assert model_info.get("mode") == "responses" -def test_responses_api_bridge_check_azure_gpt_5_4_tools_without_reasoning_stays_chat(): - """Azure gpt-5.4 with tools only should not be force-routed to Responses API.""" +def test_responses_api_bridge_check_azure_gpt_5_4_tools_with_default_reasoning_routes_to_responses(): + """ + Azure gpt-5.4 with tools and UNSET reasoning_effort must bridge: OpenAI enables + reasoning by default for gpt-5.4+, and Chat Completions rejects function tools + whenever reasoning is on. + """ from litellm.main import responses_api_bridge_check with patch("litellm.main._get_model_info_helper") as mock_get_model_info: @@ -824,11 +828,15 @@ def test_responses_api_bridge_check_azure_gpt_5_4_tools_without_reasoning_stays_ ) assert model == "gpt-5.4" - assert model_info.get("mode") != "responses" + assert model_info.get("mode") == "responses" -def test_responses_api_bridge_check_gpt_5_4_tools_without_reasoning_stays_chat(): - """gpt-5.4 with tools only should not be force-routed to Responses API.""" +def test_responses_api_bridge_check_gpt_5_4_tools_with_default_reasoning_routes_to_responses(): + """ + gpt-5.4 with tools and UNSET reasoning_effort must bridge: OpenAI enables reasoning + by default for gpt-5.4+, and Chat Completions rejects function tools whenever + reasoning is on ("use /v1/responses or set reasoning_effort to 'none'"). + """ from litellm.main import responses_api_bridge_check with patch("litellm.main._get_model_info_helper") as mock_get_model_info: @@ -841,6 +849,60 @@ def test_responses_api_bridge_check_gpt_5_4_tools_without_reasoning_stays_chat() ) assert model == "gpt-5.4" + assert model_info.get("mode") == "responses" + + +def test_responses_api_bridge_check_gpt_5_4_tools_with_reasoning_none_stays_chat(): + """ + Explicit reasoning_effort "none" is OpenAI's documented escape hatch that keeps + function tools servable on Chat Completions; the bridge must not fire. + """ + from litellm.main import responses_api_bridge_check + + with patch("litellm.main._get_model_info_helper") as mock_get_model_info: + mock_get_model_info.return_value = {"max_tokens": 128000} + model_info, model = responses_api_bridge_check( + model="gpt-5.4", + custom_llm_provider="openai", + tools=[{"type": "function", "function": {"name": "get_capital"}}], + reasoning_effort="none", + ) + + assert model == "gpt-5.4" + assert model_info.get("mode") != "responses" + + +def test_responses_api_bridge_check_reasoning_none_with_summary_still_routes_to_responses(): + """A reasoning summary is Responses-only regardless of effort value.""" + from litellm.main import responses_api_bridge_check + + with patch("litellm.main._get_model_info_helper") as mock_get_model_info: + mock_get_model_info.return_value = {"max_tokens": 128000} + model_info, model = responses_api_bridge_check( + model="gpt-5.4", + custom_llm_provider="openai", + reasoning_effort="none", + reasoning_summary="detailed", + ) + + assert model == "gpt-5.4" + assert model_info.get("mode") == "responses" + + +def test_responses_api_bridge_check_older_gpt_5_tools_without_reasoning_stays_chat(): + """Pre-5.4 GPT-5 names keep the old boundary: tools alone never bridge.""" + from litellm.main import responses_api_bridge_check + + with patch("litellm.main._get_model_info_helper") as mock_get_model_info: + mock_get_model_info.return_value = {"max_tokens": 128000} + model_info, model = responses_api_bridge_check( + model="gpt-5.1", + custom_llm_provider="openai", + tools=[{"type": "function", "function": {"name": "get_capital"}}], + reasoning_effort=None, + ) + + assert model == "gpt-5.1" assert model_info.get("mode") != "responses"