mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
fix(litellm): bridge gpt-5.4+ chat requests with tools when reasoning defaults on
OpenAI enables reasoning by default for gpt-5.4+ (unset reasoning_effort means medium server-side) and Chat Completions rejects function tools whenever reasoning is on, so a tools request without an explicit reasoning_effort 400d instead of auto-bridging to the Responses API; the bridge heuristic now treats unset effort as reasoning-active and honors the documented escape hatch by keeping explicit "none" on chat completions. The cursor input arm also gains the mirror of the messages-arm normalization: chat-nested tool envelopes, grammar formats, and object tool_choice flatten to the Responses dialect before dispatch
This commit is contained in:
Tin
Tin Chi Lo
parent
ebe48d67de
commit
6d102ea559
4 changed files with 262 additions and 8 deletions
|
|
@ -1022,7 +1022,12 @@ def responses_api_bridge_check(
|
|||
# ``reasoningSummary`` in ``extra_body``) must be bridged; Chat Completions rejects
|
||||
# those keys.
|
||||
#
|
||||
# - gpt-5.4+: tools + reasoning_effort (original) or any reasoning-summary alias.
|
||||
# - gpt-5.4+: function tools with reasoning active must be bridged. OpenAI enables
|
||||
# reasoning by default for these models (unset reasoning_effort means medium
|
||||
# server-side), and Chat Completions rejects tools whenever reasoning is on
|
||||
# ("Function tools with reasoning_effort are not supported ... use /v1/responses
|
||||
# or set reasoning_effort to 'none'"), so only an explicit ``"none"`` keeps the
|
||||
# request chat-servable.
|
||||
# - Older GPT-5 names (e.g. ``gpt-5``, ``gpt-5.1``): bridge only when a reasoning
|
||||
# summary alias is present with ``reasoning_effort`` (tools alone stay on chat).
|
||||
if (
|
||||
|
|
@ -1030,8 +1035,10 @@ def responses_api_bridge_check(
|
|||
and model_info.get("mode") != "responses"
|
||||
and OpenAIGPT5Config.is_model_gpt_5_model(model)
|
||||
and not OpenAIGPT5Config.is_model_gpt_5_search_model(model)
|
||||
and reasoning_effort is not None
|
||||
and (reasoning_summary is not None or (OpenAIGPT5Config.is_model_gpt_5_4_plus_model(model) and tools))
|
||||
and (
|
||||
(reasoning_effort is not None and reasoning_summary is not None)
|
||||
or (OpenAIGPT5Config.is_model_gpt_5_4_plus_model(model) and tools and reasoning_effort != "none")
|
||||
)
|
||||
):
|
||||
model_info["mode"] = "responses"
|
||||
model = model.replace("responses/", "")
|
||||
|
|
|
|||
|
|
@ -66,6 +66,47 @@ def _nest_flat_chat_tool_choice(tool_choice: object) -> object:
|
|||
return tool_choice
|
||||
|
||||
|
||||
def _flatten_chat_tool_for_responses(tool: object) -> object:
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
convert_custom_tool_format_to_responses_shape,
|
||||
)
|
||||
|
||||
if not isinstance(tool, dict):
|
||||
return tool
|
||||
if tool.get("type") == "custom":
|
||||
if isinstance(tool.get("custom"), dict):
|
||||
payload = {k: tool["custom"][k] for k in _FLAT_CUSTOM_TOOL_KEYS if k in tool["custom"]}
|
||||
elif "name" in tool:
|
||||
payload = {k: tool[k] for k in _FLAT_CUSTOM_TOOL_KEYS if k in tool}
|
||||
else:
|
||||
return tool
|
||||
if isinstance(payload.get("format"), dict):
|
||||
payload = {**payload, "format": convert_custom_tool_format_to_responses_shape(payload["format"])}
|
||||
return {"type": "custom", **payload}
|
||||
if tool.get("type") == "function" and isinstance(tool.get("function"), dict):
|
||||
return {
|
||||
"type": "function",
|
||||
**{k: tool["function"][k] for k in _FLAT_FUNCTION_TOOL_KEYS if k in tool["function"]},
|
||||
}
|
||||
return tool
|
||||
|
||||
|
||||
def _flatten_chat_tools_for_responses(tools: list) -> list:
|
||||
return [_flatten_chat_tool_for_responses(tool) for tool in tools]
|
||||
|
||||
|
||||
def _flatten_chat_tool_choice_for_responses(tool_choice: object) -> object:
|
||||
if not isinstance(tool_choice, dict):
|
||||
return tool_choice
|
||||
choice_type = tool_choice.get("type")
|
||||
if choice_type not in ("custom", "function"):
|
||||
return tool_choice
|
||||
nested = tool_choice.get(choice_type)
|
||||
if isinstance(nested, dict) and isinstance(nested.get("name"), str):
|
||||
return {"type": choice_type, "name": nested["name"]}
|
||||
return tool_choice
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/responses",
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
|
|
@ -444,6 +485,14 @@ async def cursor_chat_completions(
|
|||
# cache's key snapshot so later readers get an empty body
|
||||
data = {key: value for key, value in data.items() if key != "stream_options"}
|
||||
|
||||
tools = data.get("tools")
|
||||
if isinstance(tools, list):
|
||||
data = {**data, "tools": _flatten_chat_tools_for_responses(tools)}
|
||||
tool_choice = data.get("tool_choice")
|
||||
flattened_tool_choice = _flatten_chat_tool_choice_for_responses(tool_choice)
|
||||
if flattened_tool_choice != tool_choice:
|
||||
data = {**data, "tool_choice": flattened_tool_choice}
|
||||
|
||||
processor = ProxyBaseLLMRequestProcessing(data=data)
|
||||
|
||||
def cursor_data_generator(response, user_api_key_dict, request_data, request=None):
|
||||
|
|
|
|||
|
|
@ -1151,3 +1151,139 @@ class TestNestFlatChatToolChoice:
|
|||
42,
|
||||
):
|
||||
assert _nest_flat_chat_tool_choice(unchanged) == unchanged
|
||||
|
||||
|
||||
class TestFlattenChatToolsForResponsesInputArm:
|
||||
"""
|
||||
Mirror of TestNestFlatChatToolShapeMatrix for the input arm: chat-nested shapes in a
|
||||
Responses-shaped body must flatten to the Responses dialect, per level, idempotently.
|
||||
"""
|
||||
|
||||
FLAT_GRAMMAR = {"type": "grammar", "definition": "start: patch", "syntax": "lark"}
|
||||
NESTED_GRAMMAR = {"type": "grammar", "grammar": {"definition": "start: patch", "syntax": "lark"}}
|
||||
|
||||
@pytest.mark.parametrize("envelope", ["flat", "nested"])
|
||||
@pytest.mark.parametrize("format_shape", ["absent", "text", "flat_grammar", "nested_grammar"])
|
||||
def test_every_envelope_and_format_combination_lands_flat(self, envelope, format_shape):
|
||||
from litellm.proxy.response_api_endpoints.endpoints import _flatten_chat_tools_for_responses
|
||||
|
||||
format_value = {
|
||||
"absent": None,
|
||||
"text": {"type": "text"},
|
||||
"flat_grammar": self.FLAT_GRAMMAR,
|
||||
"nested_grammar": self.NESTED_GRAMMAR,
|
||||
}[format_shape]
|
||||
payload = {"name": "ApplyPatch", "description": "V4A patch"}
|
||||
if format_value is not None:
|
||||
payload["format"] = format_value
|
||||
tool = {"type": "custom", "custom": payload} if envelope == "nested" else {"type": "custom", **payload}
|
||||
|
||||
canonical = {"type": "custom", "name": "ApplyPatch", "description": "V4A patch"}
|
||||
if format_shape in ("flat_grammar", "nested_grammar"):
|
||||
canonical["format"] = self.FLAT_GRAMMAR
|
||||
elif format_shape == "text":
|
||||
canonical["format"] = {"type": "text"}
|
||||
|
||||
assert _flatten_chat_tools_for_responses([tool]) == [canonical]
|
||||
|
||||
def test_nested_function_tool_is_flattened_and_flat_passes_through(self):
|
||||
from litellm.proxy.response_api_endpoints.endpoints import _flatten_chat_tools_for_responses
|
||||
|
||||
nested = {"type": "function", "function": {"name": "read_file", "parameters": {"type": "object"}}}
|
||||
flat = {"type": "function", "name": "read_file", "parameters": {"type": "object"}}
|
||||
assert _flatten_chat_tools_for_responses([nested]) == [flat]
|
||||
assert _flatten_chat_tools_for_responses([flat]) == [flat]
|
||||
|
||||
def test_unrecognized_entries_pass_through(self):
|
||||
from litellm.proxy.response_api_endpoints.endpoints import _flatten_chat_tools_for_responses
|
||||
|
||||
entries = [{"type": "web_search"}, {"type": "custom"}, "junk", None, {}]
|
||||
assert _flatten_chat_tools_for_responses(entries) == entries
|
||||
|
||||
|
||||
class TestFlattenChatToolChoiceForResponsesInputArm:
|
||||
def test_nested_custom_and_function_tool_choice_flatten(self):
|
||||
from litellm.proxy.response_api_endpoints.endpoints import _flatten_chat_tool_choice_for_responses
|
||||
|
||||
assert _flatten_chat_tool_choice_for_responses({"type": "custom", "custom": {"name": "ApplyPatch"}}) == {
|
||||
"type": "custom",
|
||||
"name": "ApplyPatch",
|
||||
}
|
||||
assert _flatten_chat_tool_choice_for_responses({"type": "function", "function": {"name": "f"}}) == {
|
||||
"type": "function",
|
||||
"name": "f",
|
||||
}
|
||||
|
||||
def test_flat_and_string_tool_choice_pass_through(self):
|
||||
from litellm.proxy.response_api_endpoints.endpoints import _flatten_chat_tool_choice_for_responses
|
||||
|
||||
for unchanged in ("auto", "required", None, {"type": "custom", "name": "x"}, {"type": "auto"}, 42):
|
||||
assert _flatten_chat_tool_choice_for_responses(unchanged) == unchanged
|
||||
|
||||
|
||||
class TestCursorInputArmFlattening:
|
||||
@pytest.mark.asyncio
|
||||
async def test_nested_chat_shapes_in_input_body_reach_aresponses_flattened(self):
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from openai.types.responses import ResponseOutputMessage, ResponseOutputText
|
||||
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
|
||||
mock_response = ResponsesAPIResponse(
|
||||
id="resp_flat123",
|
||||
created_at=1234567890,
|
||||
model="gpt-5.6",
|
||||
object="response",
|
||||
output=[
|
||||
ResponseOutputMessage(
|
||||
id="msg_flat123",
|
||||
type="message",
|
||||
role="assistant",
|
||||
status="completed",
|
||||
content=[ResponseOutputText(type="output_text", text="ok", annotations=[])],
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(api_key="sk-1234")
|
||||
try:
|
||||
with patch("litellm.proxy.proxy_server.llm_router") as mock_router:
|
||||
mock_router.aresponses = AsyncMock(return_value=mock_response)
|
||||
client = TestClient(app)
|
||||
response = client.post(
|
||||
"/cursor/chat/completions",
|
||||
json={
|
||||
"model": "gpt-5.6",
|
||||
"input": [{"role": "user", "content": "use ApplyPatch"}],
|
||||
"tools": [
|
||||
{
|
||||
"type": "custom",
|
||||
"custom": {
|
||||
"name": "ApplyPatch",
|
||||
"format": {
|
||||
"type": "grammar",
|
||||
"grammar": {"definition": "start: patch", "syntax": "lark"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{"type": "function", "name": "read_file", "parameters": {"type": "object"}},
|
||||
],
|
||||
"tool_choice": {"type": "custom", "custom": {"name": "ApplyPatch"}},
|
||||
},
|
||||
headers={"Authorization": "Bearer sk-1234"},
|
||||
)
|
||||
finally:
|
||||
app.dependency_overrides.pop(user_api_key_auth, None)
|
||||
|
||||
assert response.status_code == 200
|
||||
call_kwargs = mock_router.aresponses.call_args.kwargs
|
||||
assert call_kwargs["tools"] == [
|
||||
{
|
||||
"type": "custom",
|
||||
"name": "ApplyPatch",
|
||||
"format": {"type": "grammar", "definition": "start: patch", "syntax": "lark"},
|
||||
},
|
||||
{"type": "function", "name": "read_file", "parameters": {"type": "object"}},
|
||||
]
|
||||
assert call_kwargs["tool_choice"] == {"type": "custom", "name": "ApplyPatch"}
|
||||
|
|
|
|||
|
|
@ -810,8 +810,12 @@ def test_responses_api_bridge_check_azure_gpt_5_4_tools_plus_reasoning_routes_to
|
|||
assert model_info.get("mode") == "responses"
|
||||
|
||||
|
||||
def test_responses_api_bridge_check_azure_gpt_5_4_tools_without_reasoning_stays_chat():
|
||||
"""Azure gpt-5.4 with tools only should not be force-routed to Responses API."""
|
||||
def test_responses_api_bridge_check_azure_gpt_5_4_tools_with_default_reasoning_routes_to_responses():
|
||||
"""
|
||||
Azure gpt-5.4 with tools and UNSET reasoning_effort must bridge: OpenAI enables
|
||||
reasoning by default for gpt-5.4+, and Chat Completions rejects function tools
|
||||
whenever reasoning is on.
|
||||
"""
|
||||
from litellm.main import responses_api_bridge_check
|
||||
|
||||
with patch("litellm.main._get_model_info_helper") as mock_get_model_info:
|
||||
|
|
@ -824,11 +828,15 @@ def test_responses_api_bridge_check_azure_gpt_5_4_tools_without_reasoning_stays_
|
|||
)
|
||||
|
||||
assert model == "gpt-5.4"
|
||||
assert model_info.get("mode") != "responses"
|
||||
assert model_info.get("mode") == "responses"
|
||||
|
||||
|
||||
def test_responses_api_bridge_check_gpt_5_4_tools_without_reasoning_stays_chat():
|
||||
"""gpt-5.4 with tools only should not be force-routed to Responses API."""
|
||||
def test_responses_api_bridge_check_gpt_5_4_tools_with_default_reasoning_routes_to_responses():
|
||||
"""
|
||||
gpt-5.4 with tools and UNSET reasoning_effort must bridge: OpenAI enables reasoning
|
||||
by default for gpt-5.4+, and Chat Completions rejects function tools whenever
|
||||
reasoning is on ("use /v1/responses or set reasoning_effort to 'none'").
|
||||
"""
|
||||
from litellm.main import responses_api_bridge_check
|
||||
|
||||
with patch("litellm.main._get_model_info_helper") as mock_get_model_info:
|
||||
|
|
@ -841,6 +849,60 @@ def test_responses_api_bridge_check_gpt_5_4_tools_without_reasoning_stays_chat()
|
|||
)
|
||||
|
||||
assert model == "gpt-5.4"
|
||||
assert model_info.get("mode") == "responses"
|
||||
|
||||
|
||||
def test_responses_api_bridge_check_gpt_5_4_tools_with_reasoning_none_stays_chat():
|
||||
"""
|
||||
Explicit reasoning_effort "none" is OpenAI's documented escape hatch that keeps
|
||||
function tools servable on Chat Completions; the bridge must not fire.
|
||||
"""
|
||||
from litellm.main import responses_api_bridge_check
|
||||
|
||||
with patch("litellm.main._get_model_info_helper") as mock_get_model_info:
|
||||
mock_get_model_info.return_value = {"max_tokens": 128000}
|
||||
model_info, model = responses_api_bridge_check(
|
||||
model="gpt-5.4",
|
||||
custom_llm_provider="openai",
|
||||
tools=[{"type": "function", "function": {"name": "get_capital"}}],
|
||||
reasoning_effort="none",
|
||||
)
|
||||
|
||||
assert model == "gpt-5.4"
|
||||
assert model_info.get("mode") != "responses"
|
||||
|
||||
|
||||
def test_responses_api_bridge_check_reasoning_none_with_summary_still_routes_to_responses():
|
||||
"""A reasoning summary is Responses-only regardless of effort value."""
|
||||
from litellm.main import responses_api_bridge_check
|
||||
|
||||
with patch("litellm.main._get_model_info_helper") as mock_get_model_info:
|
||||
mock_get_model_info.return_value = {"max_tokens": 128000}
|
||||
model_info, model = responses_api_bridge_check(
|
||||
model="gpt-5.4",
|
||||
custom_llm_provider="openai",
|
||||
reasoning_effort="none",
|
||||
reasoning_summary="detailed",
|
||||
)
|
||||
|
||||
assert model == "gpt-5.4"
|
||||
assert model_info.get("mode") == "responses"
|
||||
|
||||
|
||||
def test_responses_api_bridge_check_older_gpt_5_tools_without_reasoning_stays_chat():
|
||||
"""Pre-5.4 GPT-5 names keep the old boundary: tools alone never bridge."""
|
||||
from litellm.main import responses_api_bridge_check
|
||||
|
||||
with patch("litellm.main._get_model_info_helper") as mock_get_model_info:
|
||||
mock_get_model_info.return_value = {"max_tokens": 128000}
|
||||
model_info, model = responses_api_bridge_check(
|
||||
model="gpt-5.1",
|
||||
custom_llm_provider="openai",
|
||||
tools=[{"type": "function", "function": {"name": "get_capital"}}],
|
||||
reasoning_effort=None,
|
||||
)
|
||||
|
||||
assert model == "gpt-5.1"
|
||||
assert model_info.get("mode") != "responses"
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue