From c70dd4ece6cd7819ebafebe493ca7eefc6230b45 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:46:28 -0700 Subject: [PATCH] fix(responses bridge): keep defer_loading on function tools sent to the Responses API (#46439) * fix(responses bridge): keep defer_loading on function tools sent to the Responses API Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses bridge): give the defer_loading casts a cast-ok reason Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(responses bridge): build defer_loading outside the tool loop so Final stays valid Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses bridge): honor drop_params for defer_loading and route tool_search to Responses Under drop_params, defer_loading is dropped when no tool_search tool is sent, so the 400 it would cause upstream goes away as before. A non-bool value answers 400 without drop_params and is dropped with it. A null tool-level value falls back to the nested function value. A request carrying a tool_search tool now bridges to Responses on OpenAI and Azure endpoints even with reasoning_effort none, since chat completions rejects the tool_search type. * fix(responses bridge): forward defer_loading only beside a tool_search tool defer_loading only means something with tool_search, and backends without it reject the key (xAI answers 403). A true value with no tool_search tool now answers 400 naming drop_params and is dropped under drop_params; a false value there is a no-op and is dropped. * fix(responses bridge): drop defer_loading beside a client-executed tool_search Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../transformation.py | 71 +++++++- litellm/responses/bridge_check.py | 10 +- .../test_responses_bridge_tool_search_wire.py | 68 ++++++++ ...responses_transformation_transformation.py | 162 ++++++++++++++++++ tests/unit/test_main.py | 31 ++++ 5 files changed, 335 insertions(+), 7 deletions(-) create mode 100644 tests/integration/providers/test_responses_bridge_tool_search_wire.py diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 638acfe0723..da68b74cb6e 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -517,6 +517,50 @@ def _reasoning_item_to_response_input( } +def _requested_defer_loading(tool: Mapping[str, object], function_tool: Mapping[str, object]) -> object: + tool_level: Final = tool.get("defer_loading") + return tool_level if tool_level is not None else function_tool.get("defer_loading") + + +def _defer_loading_field( + tool: Mapping[str, object], + function_tool: Mapping[str, object], + *, + tool_search: Literal["hosted", "client", "absent"], + drop_params: bool, +) -> Mapping[str, bool]: + defer_loading: Final = _requested_defer_loading(tool, function_tool) + if defer_loading is None or (defer_loading is False and tool_search == "absent"): + return {} + if tool_search == "client": + verbose_logger.debug( + "Chat provider: dropping defer_loading=%r beside a client-executed tool_search", defer_loading + ) + return {} + if isinstance(defer_loading, bool) and tool_search == "hosted": + return {"defer_loading": defer_loading} + if drop_params: + verbose_logger.debug("Chat provider: dropping defer_loading=%r under drop_params", defer_loading) + return {} + problem: Final = ( + 'defer_loading needs a {"type": "tool_search"} tool in the same request' + if isinstance(defer_loading, bool) + else f"defer_loading must be a boolean, got {defer_loading!r}" + ) + raise litellm.UnsupportedParamsError( + message=f"{problem}. To drop it, set `litellm.drop_params=True` or pass `drop_params=True`." + ) + + +def _without_dropped_defer_loading( + tool: Mapping[str, object], *, tool_search: Literal["hosted", "client", "absent"], drop_params: bool +) -> Mapping[str, object]: + if tool.get("type") != "function" or "defer_loading" not in tool: + return tool + kept: Final = _defer_loading_field(tool, {}, tool_search=tool_search, drop_params=drop_params) + return {**{key: value for key, value in tool.items() if key != "defer_loading"}, **kept} + + class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): """ Handler for transforming /chat/completions api requests to litellm.responses requests @@ -747,6 +791,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): self, optional_params: dict, responses_api_request: "ResponsesAPIOptionalRequestParams", + drop_params: bool = False, ) -> None: """Map optional_params into responses_api_request (mutates in place).""" for key, value in optional_params.items(): @@ -756,7 +801,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): responses_api_request["max_output_tokens"] = value elif key == "tools" and value is not None: responses_api_request["tools"] = self._convert_tools_to_responses_format( - cast(list[dict[str, object]], value) + cast(list[dict[str, object]], value), drop_params=drop_params ) elif key == "response_format": text_format = self._transform_response_format_to_text_format(value) @@ -889,7 +934,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if instructions: responses_api_request["instructions"] = instructions - self._map_optional_params_to_responses_api_request(optional_params, responses_api_request) + self._map_optional_params_to_responses_api_request( + optional_params, + responses_api_request, + drop_params=bool(litellm_params.get("drop_params") or litellm.drop_params), + ) stream: Final = optional_params.get("stream") or litellm_params.get("stream", False) verbose_logger.debug("Chat provider: Stream parameter: %s", stream) @@ -1417,14 +1466,21 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): return result def _convert_tools_to_responses_format( - self, tools: list[dict[str, object]] + self, tools: list[dict[str, object]], drop_params: bool = False ) -> list["ALL_RESPONSES_API_TOOL_PARAMS"]: """Convert chat completion tools to responses API tools format""" + tool_search: Final[Literal["hosted", "client", "absent"]] = ( + "client" + if any(tool.get("type") == "tool_search" and tool.get("execution") == "client" for tool in tools) + else "hosted" + if any(tool.get("type") == "tool_search" for tool in tools) + else "absent" + ) responses_tools: Final[list[ALL_RESPONSES_API_TOOL_PARAMS]] = [] for tool in tools: # convert function tool from chat completion to responses API format - if tool.get("type") == "function" and isinstance(tool.get("function"), dict): - function_tool = cast(ChatCompletionToolParamFunctionChunk, tool.get("function")) + if tool.get("type") == "function" and isinstance(function_payload := tool.get("function"), dict): + function_tool = cast(ChatCompletionToolParamFunctionChunk, function_payload) # cast-ok: client tool responses_tools.append( FunctionToolParam( name=function_tool["name"], @@ -1432,6 +1488,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): strict=function_tool.get("strict"), type="function", description=function_tool.get("description"), + **_defer_loading_field(tool, function_tool, tool_search=tool_search, drop_params=drop_params), ) ) elif tool.get("type") == "custom" and isinstance(custom_payload := tool.get("custom"), dict): @@ -1446,7 +1503,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): flat_custom["format"] = convert_custom_tool_format_to_responses_shape(custom_payload["format"]) responses_tools.append(flat_custom) else: - responses_tools.append(tool) + responses_tools.append( + _without_dropped_defer_loading(tool, tool_search=tool_search, drop_params=drop_params) + ) return cast(list["ALL_RESPONSES_API_TOOL_PARAMS"], responses_tools) diff --git a/litellm/responses/bridge_check.py b/litellm/responses/bridge_check.py index 3b1e958ff86..c2b4c105199 100644 --- a/litellm/responses/bridge_check.py +++ b/litellm/responses/bridge_check.py @@ -108,6 +108,10 @@ def responses_api_bridge_check( ) for tool in (tools or ()) ) + carries_tool_search: Final = any( + (tool.get("type") if isinstance(tool, dict) else getattr(tool, "type", None)) == "tool_search" + for tool in (tools or ()) + ) if isinstance(reasoning_effort, dict): reasoning_active = reasoning_effort.get("effort") != "none" or reasoning_effort.get("summary") is not None else: @@ -142,7 +146,11 @@ def responses_api_bridge_check( and model_info.get("mode") != "responses" and OpenAIGPT5Config.is_model_gpt_5_model(model) and not OpenAIGPT5Config.is_model_gpt_5_search_model(model) - and ((reasoning_effort is not None and reasoning_summary is not None) or chat_rejects_function_tools) + and ( + (reasoning_effort is not None and reasoning_summary is not None) + or chat_rejects_function_tools + or (carries_tool_search and on_constraint_enforcing_endpoint) + ) ): model_info["mode"] = "responses" model = model.replace("responses/", "") diff --git a/tests/integration/providers/test_responses_bridge_tool_search_wire.py b/tests/integration/providers/test_responses_bridge_tool_search_wire.py new file mode 100644 index 00000000000..fd7a5218049 --- /dev/null +++ b/tests/integration/providers/test_responses_bridge_tool_search_wire.py @@ -0,0 +1,68 @@ +import uuid +from typing import Final + +import pytest +from integration._support import prompt_cache_breakpoint as pcb +from integration._support import responses_vendor as rv +from integration._support.client import Gateway +from integration._support.wire import wire_server +from pydantic import JsonValue + +pytestmark: Final = pytest.mark.timeout(120) + +_PARAMETERS: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], +} +_TOOL_SEARCH: Final[dict[str, JsonValue]] = {"type": "tool_search"} + + +def _function(name: str, **extra: JsonValue) -> dict[str, JsonValue]: + return {"name": name, "description": f"Look up {name}", "parameters": _PARAMETERS, **extra} + + +def _wire_function(name: str, **extra: JsonValue) -> dict[str, JsonValue]: + return {"type": "function", "strict": None, **_function(name), **extra} + + +@pytest.mark.parametrize( + "deferred_tool", + [ + {"type": "function", "function": _function("get_weather"), "defer_loading": True}, + {"type": "function", "function": _function("get_weather", defer_loading=True)}, + ], + ids=["tool-level", "function-level"], +) +def test_chat_bridge_forwards_defer_loading_so_tool_search_has_a_deferred_tool( + gateway: Gateway, deferred_tool: dict[str, JsonValue] +) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(pcb.respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=pcb.MODEL, api_base=f"{wire.url}/v1") + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": pcb.prompt(marker)}], + "tools": [ + _TOOL_SEARCH, + deferred_tool, + {"type": "function", "function": _function("get_time")}, + {"type": "function", "function": _function("get_news", defer_loading=False)}, + ], + **pcb.NO_CACHE, + }, + ) + assert response.status_code == 200, response.text + message: Final = rv.JSON_OBJECT.validate_python( + rv.ITEMS.validate_python(rv.JSON_OBJECT.validate_json(response.text)["choices"])[0]["message"] + ) + assert message["content"] == rv.answer(marker), response.text + assert pcb.body_of(pcb.posted(wire, marker))["tools"] == [ + _TOOL_SEARCH, + _wire_function("get_weather", defer_loading=True), + _wire_function("get_time"), + _wire_function("get_news", defer_loading=False), + ] diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 91bfb8d78d6..056ee1bd717 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -832,6 +832,168 @@ def test_convert_tools_to_responses_format_passes_flat_function_tool_through(): assert converted == [flat_tool] +@pytest.mark.parametrize( + ("tool", "expected_defer_loading"), + [ + ({"type": "function", "function": {"name": "lookup"}, "defer_loading": True}, True), + ({"type": "function", "function": {"name": "lookup", "defer_loading": True}}, True), + ({"type": "function", "function": {"name": "lookup", "defer_loading": True}, "defer_loading": False}, False), + ({"type": "function", "function": {"name": "lookup", "defer_loading": True}, "defer_loading": None}, True), + ], + ids=["tool-level", "function-level", "tool-level-wins", "null-tool-level-falls-back"], +) +def test_convert_tools_to_responses_format_keeps_defer_loading(tool, expected_defer_loading): + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + + converted = LiteLLMResponsesTransformationHandler()._convert_tools_to_responses_format( + [{"type": "tool_search"}, tool] + ) + + assert converted == [ + {"type": "tool_search"}, + { + "type": "function", + "name": "lookup", + "parameters": None, + "strict": None, + "description": None, + "defer_loading": expected_defer_loading, + }, + ] + + +_TOOL_SEARCH: Final = {"type": "tool_search"} + + +@pytest.mark.parametrize( + "deferred_tool", + [ + {"type": "function", "function": {"name": "lookup"}, "defer_loading": True}, + {"type": "function", "function": {"name": "lookup", "defer_loading": True}}, + {"type": "function", "name": "lookup", "defer_loading": True}, + ], + ids=["tool-level", "function-level", "flat"], +) +@pytest.mark.parametrize( + ("drop_params", "with_tool_search", "expected_defer_loading"), + [(True, False, None), (True, True, True), (False, True, True)], + ids=["drop-params-no-search", "drop-params-with-search", "no-drop-params-with-search"], +) +def test_transform_request_forwards_defer_loading_only_beside_tool_search( + deferred_tool, drop_params, with_tool_search, expected_defer_loading +): + tools: Final = [_TOOL_SEARCH, deferred_tool] if with_tool_search else [deferred_tool] + + result = LiteLLMResponsesTransformationHandler().transform_request( + model="gpt-5.5", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + optional_params={"tools": tools}, + litellm_params={"drop_params": drop_params}, + headers={}, + litellm_logging_obj=Mock(), + ) + + lookup_tool: Final = next(tool for tool in result["tools"] if tool.get("name") == "lookup") + assert lookup_tool.get("defer_loading") == expected_defer_loading + + +@pytest.mark.parametrize( + "deferred_tool", + [ + {"type": "function", "function": {"name": "lookup"}, "defer_loading": True}, + {"type": "function", "function": {"name": "lookup", "defer_loading": True}}, + ], + ids=["tool-level", "function-level"], +) +@pytest.mark.parametrize("drop_params", [False, True], ids=["no-drop-params", "drop-params"]) +@pytest.mark.parametrize( + "tool_search_tool", + [{"type": "tool_search", "execution": "client"}, {"type": "tool_search"}], + ids=["client-only", "hosted-and-client"], +) +def test_transform_request_drops_defer_loading_beside_client_tool_search( + deferred_tool, drop_params, tool_search_tool +): + tool_search_tools: Final = ( + [tool_search_tool] + if tool_search_tool.get("execution") == "client" + else [tool_search_tool, {"type": "tool_search", "execution": "client"}] + ) + + result = LiteLLMResponsesTransformationHandler().transform_request( + model="gpt-5.5", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + optional_params={"tools": [*tool_search_tools, deferred_tool]}, + litellm_params={"drop_params": drop_params}, + headers={}, + litellm_logging_obj=Mock(), + ) + + lookup_tool: Final = next(tool for tool in result["tools"] if tool.get("name") == "lookup") + assert "defer_loading" not in lookup_tool + + +@pytest.mark.parametrize( + "deferred_tool", + [ + {"type": "function", "function": {"name": "lookup"}, "defer_loading": True}, + {"type": "function", "function": {"name": "lookup", "defer_loading": True}}, + {"type": "function", "name": "lookup", "defer_loading": True}, + ], + ids=["tool-level", "function-level", "flat"], +) +def test_convert_tools_to_responses_format_rejects_deferred_tool_without_tool_search(deferred_tool): + with pytest.raises(litellm.UnsupportedParamsError, match=r"tool_search.*drop_params"): + LiteLLMResponsesTransformationHandler()._convert_tools_to_responses_format([deferred_tool]) + + +@pytest.mark.parametrize( + "tool", + [ + {"type": "function", "function": {"name": "lookup"}, "defer_loading": False}, + {"type": "function", "name": "lookup", "defer_loading": False}, + ], + ids=["nested", "flat"], +) +def test_convert_tools_to_responses_format_omits_false_defer_loading_without_tool_search(tool): + converted: Final = LiteLLMResponsesTransformationHandler()._convert_tools_to_responses_format([tool]) + + assert "defer_loading" not in converted[0] + + +@pytest.mark.parametrize( + "tool", + [ + {"type": "function", "function": {"name": "lookup"}, "defer_loading": "true"}, + {"type": "function", "function": {"name": "lookup", "defer_loading": "yes"}}, + {"type": "function", "name": "lookup", "defer_loading": "true"}, + ], + ids=["tool-level", "function-level", "flat"], +) +def test_convert_tools_to_responses_format_rejects_non_bool_defer_loading_without_drop_params(tool): + with pytest.raises(litellm.UnsupportedParamsError, match="defer_loading must be a boolean"): + LiteLLMResponsesTransformationHandler()._convert_tools_to_responses_format([_TOOL_SEARCH, tool]) + + +@pytest.mark.parametrize( + "tool", + [ + {"type": "function", "function": {"name": "lookup"}, "defer_loading": "true"}, + {"type": "function", "name": "lookup", "defer_loading": "true"}, + ], + ids=["nested", "flat"], +) +def test_convert_tools_to_responses_format_drops_non_bool_defer_loading_under_drop_params(tool): + converted: Final = LiteLLMResponsesTransformationHandler()._convert_tools_to_responses_format( + [_TOOL_SEARCH, tool], drop_params=True + ) + + lookup_tool: Final = next(item for item in converted if item.get("name") == "lookup") + assert "defer_loading" not in lookup_tool + + def test_extract_extra_body_params_reasoning_effort_override(): """Test that reasoning_effort from extra_body overrides top-level reasoning_effort""" from litellm.completion_extras.litellm_responses_transformation.transformation import ( diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 7dc26fca1f1..de06518e3e1 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -1244,6 +1244,37 @@ def test_responses_api_bridge_check_blank_api_base_is_default_openai(blank_api_b assert model_info.get("mode") == "responses" +@pytest.mark.parametrize( + ("custom_llm_provider", "reasoning_effort", "api_base", "expected_mode"), + [ + ("openai", "none", None, "responses"), + ("openai", {"effort": "none"}, None, "responses"), + ("azure", "none", "https://example.openai.azure.com", "responses"), + ("openai", "none", "http://vllm.internal:8000/v1", None), + ], + ids=["openai-effort-none", "openai-dict-effort-none", "azure-effort-none", "chat-only-custom-base"], +) +def test_responses_api_bridge_check_tool_search_routes_to_responses_on_openai_endpoints( + custom_llm_provider, reasoning_effort, api_base, expected_mode +): + from litellm.main import responses_api_bridge_check + + with patch("litellm.responses.bridge_check.get_model_info_helper") as mock_get_model_info: + mock_get_model_info.return_value = {"max_tokens": 128000} + model_info, _ = responses_api_bridge_check( + model="gpt-5.5", + custom_llm_provider=custom_llm_provider, + tools=[ + {"type": "tool_search"}, + {"type": "function", "function": {"name": "get_weather"}, "defer_loading": True}, + ], + reasoning_effort=reasoning_effort, + api_base=api_base, + ) + + assert model_info.get("mode") == expected_mode + + def test_responses_api_bridge_check_custom_api_base_with_unset_effort_stays_chat(): """ Chat-only OpenAI-compatible backends registered under the openai provider with a