mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-15 04:19:08 +00:00
fix(responses bridge): keep defer_loading on function tools sent to the Responses API (#46439)
* fix(responses bridge): keep defer_loading on function tools sent to the Responses API Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses bridge): give the defer_loading casts a cast-ok reason Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(responses bridge): build defer_loading outside the tool loop so Final stays valid Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses bridge): honor drop_params for defer_loading and route tool_search to Responses Under drop_params, defer_loading is dropped when no tool_search tool is sent, so the 400 it would cause upstream goes away as before. A non-bool value answers 400 without drop_params and is dropped with it. A null tool-level value falls back to the nested function value. A request carrying a tool_search tool now bridges to Responses on OpenAI and Azure endpoints even with reasoning_effort none, since chat completions rejects the tool_search type. * fix(responses bridge): forward defer_loading only beside a tool_search tool defer_loading only means something with tool_search, and backends without it reject the key (xAI answers 403). A true value with no tool_search tool now answers 400 naming drop_params and is dropped under drop_params; a false value there is a no-op and is dropped. * fix(responses bridge): drop defer_loading beside a client-executed tool_search Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Mateo Wang
GitHub
parent
9423966f5d
commit
c70dd4ece6
5 changed files with 335 additions and 7 deletions
|
|
@ -517,6 +517,50 @@ def _reasoning_item_to_response_input(
|
|||
}
|
||||
|
||||
|
||||
def _requested_defer_loading(tool: Mapping[str, object], function_tool: Mapping[str, object]) -> object:
|
||||
tool_level: Final = tool.get("defer_loading")
|
||||
return tool_level if tool_level is not None else function_tool.get("defer_loading")
|
||||
|
||||
|
||||
def _defer_loading_field(
|
||||
tool: Mapping[str, object],
|
||||
function_tool: Mapping[str, object],
|
||||
*,
|
||||
tool_search: Literal["hosted", "client", "absent"],
|
||||
drop_params: bool,
|
||||
) -> Mapping[str, bool]:
|
||||
defer_loading: Final = _requested_defer_loading(tool, function_tool)
|
||||
if defer_loading is None or (defer_loading is False and tool_search == "absent"):
|
||||
return {}
|
||||
if tool_search == "client":
|
||||
verbose_logger.debug(
|
||||
"Chat provider: dropping defer_loading=%r beside a client-executed tool_search", defer_loading
|
||||
)
|
||||
return {}
|
||||
if isinstance(defer_loading, bool) and tool_search == "hosted":
|
||||
return {"defer_loading": defer_loading}
|
||||
if drop_params:
|
||||
verbose_logger.debug("Chat provider: dropping defer_loading=%r under drop_params", defer_loading)
|
||||
return {}
|
||||
problem: Final = (
|
||||
'defer_loading needs a {"type": "tool_search"} tool in the same request'
|
||||
if isinstance(defer_loading, bool)
|
||||
else f"defer_loading must be a boolean, got {defer_loading!r}"
|
||||
)
|
||||
raise litellm.UnsupportedParamsError(
|
||||
message=f"{problem}. To drop it, set `litellm.drop_params=True` or pass `drop_params=True`."
|
||||
)
|
||||
|
||||
|
||||
def _without_dropped_defer_loading(
|
||||
tool: Mapping[str, object], *, tool_search: Literal["hosted", "client", "absent"], drop_params: bool
|
||||
) -> Mapping[str, object]:
|
||||
if tool.get("type") != "function" or "defer_loading" not in tool:
|
||||
return tool
|
||||
kept: Final = _defer_loading_field(tool, {}, tool_search=tool_search, drop_params=drop_params)
|
||||
return {**{key: value for key, value in tool.items() if key != "defer_loading"}, **kept}
|
||||
|
||||
|
||||
class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
||||
"""
|
||||
Handler for transforming /chat/completions api requests to litellm.responses requests
|
||||
|
|
@ -747,6 +791,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
self,
|
||||
optional_params: dict,
|
||||
responses_api_request: "ResponsesAPIOptionalRequestParams",
|
||||
drop_params: bool = False,
|
||||
) -> None:
|
||||
"""Map optional_params into responses_api_request (mutates in place)."""
|
||||
for key, value in optional_params.items():
|
||||
|
|
@ -756,7 +801,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
responses_api_request["max_output_tokens"] = value
|
||||
elif key == "tools" and value is not None:
|
||||
responses_api_request["tools"] = self._convert_tools_to_responses_format(
|
||||
cast(list[dict[str, object]], value)
|
||||
cast(list[dict[str, object]], value), drop_params=drop_params
|
||||
)
|
||||
elif key == "response_format":
|
||||
text_format = self._transform_response_format_to_text_format(value)
|
||||
|
|
@ -889,7 +934,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
if instructions:
|
||||
responses_api_request["instructions"] = instructions
|
||||
|
||||
self._map_optional_params_to_responses_api_request(optional_params, responses_api_request)
|
||||
self._map_optional_params_to_responses_api_request(
|
||||
optional_params,
|
||||
responses_api_request,
|
||||
drop_params=bool(litellm_params.get("drop_params") or litellm.drop_params),
|
||||
)
|
||||
|
||||
stream: Final = optional_params.get("stream") or litellm_params.get("stream", False)
|
||||
verbose_logger.debug("Chat provider: Stream parameter: %s", stream)
|
||||
|
|
@ -1417,14 +1466,21 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
return result
|
||||
|
||||
def _convert_tools_to_responses_format(
|
||||
self, tools: list[dict[str, object]]
|
||||
self, tools: list[dict[str, object]], drop_params: bool = False
|
||||
) -> list["ALL_RESPONSES_API_TOOL_PARAMS"]:
|
||||
"""Convert chat completion tools to responses API tools format"""
|
||||
tool_search: Final[Literal["hosted", "client", "absent"]] = (
|
||||
"client"
|
||||
if any(tool.get("type") == "tool_search" and tool.get("execution") == "client" for tool in tools)
|
||||
else "hosted"
|
||||
if any(tool.get("type") == "tool_search" for tool in tools)
|
||||
else "absent"
|
||||
)
|
||||
responses_tools: Final[list[ALL_RESPONSES_API_TOOL_PARAMS]] = []
|
||||
for tool in tools:
|
||||
# convert function tool from chat completion to responses API format
|
||||
if tool.get("type") == "function" and isinstance(tool.get("function"), dict):
|
||||
function_tool = cast(ChatCompletionToolParamFunctionChunk, tool.get("function"))
|
||||
if tool.get("type") == "function" and isinstance(function_payload := tool.get("function"), dict):
|
||||
function_tool = cast(ChatCompletionToolParamFunctionChunk, function_payload) # cast-ok: client tool
|
||||
responses_tools.append(
|
||||
FunctionToolParam(
|
||||
name=function_tool["name"],
|
||||
|
|
@ -1432,6 +1488,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
strict=function_tool.get("strict"),
|
||||
type="function",
|
||||
description=function_tool.get("description"),
|
||||
**_defer_loading_field(tool, function_tool, tool_search=tool_search, drop_params=drop_params),
|
||||
)
|
||||
)
|
||||
elif tool.get("type") == "custom" and isinstance(custom_payload := tool.get("custom"), dict):
|
||||
|
|
@ -1446,7 +1503,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
flat_custom["format"] = convert_custom_tool_format_to_responses_shape(custom_payload["format"])
|
||||
responses_tools.append(flat_custom)
|
||||
else:
|
||||
responses_tools.append(tool)
|
||||
responses_tools.append(
|
||||
_without_dropped_defer_loading(tool, tool_search=tool_search, drop_params=drop_params)
|
||||
)
|
||||
|
||||
return cast(list["ALL_RESPONSES_API_TOOL_PARAMS"], responses_tools)
|
||||
|
||||
|
|
|
|||
|
|
@ -108,6 +108,10 @@ def responses_api_bridge_check(
|
|||
)
|
||||
for tool in (tools or ())
|
||||
)
|
||||
carries_tool_search: Final = any(
|
||||
(tool.get("type") if isinstance(tool, dict) else getattr(tool, "type", None)) == "tool_search"
|
||||
for tool in (tools or ())
|
||||
)
|
||||
if isinstance(reasoning_effort, dict):
|
||||
reasoning_active = reasoning_effort.get("effort") != "none" or reasoning_effort.get("summary") is not None
|
||||
else:
|
||||
|
|
@ -142,7 +146,11 @@ def responses_api_bridge_check(
|
|||
and model_info.get("mode") != "responses"
|
||||
and OpenAIGPT5Config.is_model_gpt_5_model(model)
|
||||
and not OpenAIGPT5Config.is_model_gpt_5_search_model(model)
|
||||
and ((reasoning_effort is not None and reasoning_summary is not None) or chat_rejects_function_tools)
|
||||
and (
|
||||
(reasoning_effort is not None and reasoning_summary is not None)
|
||||
or chat_rejects_function_tools
|
||||
or (carries_tool_search and on_constraint_enforcing_endpoint)
|
||||
)
|
||||
):
|
||||
model_info["mode"] = "responses"
|
||||
model = model.replace("responses/", "")
|
||||
|
|
|
|||
|
|
@ -0,0 +1,68 @@
|
|||
import uuid
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support import prompt_cache_breakpoint as pcb
|
||||
from integration._support import responses_vendor as rv
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.wire import wire_server
|
||||
from pydantic import JsonValue
|
||||
|
||||
pytestmark: Final = pytest.mark.timeout(120)
|
||||
|
||||
_PARAMETERS: Final[dict[str, JsonValue]] = {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
}
|
||||
_TOOL_SEARCH: Final[dict[str, JsonValue]] = {"type": "tool_search"}
|
||||
|
||||
|
||||
def _function(name: str, **extra: JsonValue) -> dict[str, JsonValue]:
|
||||
return {"name": name, "description": f"Look up {name}", "parameters": _PARAMETERS, **extra}
|
||||
|
||||
|
||||
def _wire_function(name: str, **extra: JsonValue) -> dict[str, JsonValue]:
|
||||
return {"type": "function", "strict": None, **_function(name), **extra}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"deferred_tool",
|
||||
[
|
||||
{"type": "function", "function": _function("get_weather"), "defer_loading": True},
|
||||
{"type": "function", "function": _function("get_weather", defer_loading=True)},
|
||||
],
|
||||
ids=["tool-level", "function-level"],
|
||||
)
|
||||
def test_chat_bridge_forwards_defer_loading_so_tool_search_has_a_deferred_tool(
|
||||
gateway: Gateway, deferred_tool: dict[str, JsonValue]
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with wire_server(pcb.respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=pcb.MODEL, api_base=f"{wire.url}/v1")
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": pcb.prompt(marker)}],
|
||||
"tools": [
|
||||
_TOOL_SEARCH,
|
||||
deferred_tool,
|
||||
{"type": "function", "function": _function("get_time")},
|
||||
{"type": "function", "function": _function("get_news", defer_loading=False)},
|
||||
],
|
||||
**pcb.NO_CACHE,
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
message: Final = rv.JSON_OBJECT.validate_python(
|
||||
rv.ITEMS.validate_python(rv.JSON_OBJECT.validate_json(response.text)["choices"])[0]["message"]
|
||||
)
|
||||
assert message["content"] == rv.answer(marker), response.text
|
||||
assert pcb.body_of(pcb.posted(wire, marker))["tools"] == [
|
||||
_TOOL_SEARCH,
|
||||
_wire_function("get_weather", defer_loading=True),
|
||||
_wire_function("get_time"),
|
||||
_wire_function("get_news", defer_loading=False),
|
||||
]
|
||||
|
|
@ -832,6 +832,168 @@ def test_convert_tools_to_responses_format_passes_flat_function_tool_through():
|
|||
assert converted == [flat_tool]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tool", "expected_defer_loading"),
|
||||
[
|
||||
({"type": "function", "function": {"name": "lookup"}, "defer_loading": True}, True),
|
||||
({"type": "function", "function": {"name": "lookup", "defer_loading": True}}, True),
|
||||
({"type": "function", "function": {"name": "lookup", "defer_loading": True}, "defer_loading": False}, False),
|
||||
({"type": "function", "function": {"name": "lookup", "defer_loading": True}, "defer_loading": None}, True),
|
||||
],
|
||||
ids=["tool-level", "function-level", "tool-level-wins", "null-tool-level-falls-back"],
|
||||
)
|
||||
def test_convert_tools_to_responses_format_keeps_defer_loading(tool, expected_defer_loading):
|
||||
from litellm.completion_extras.litellm_responses_transformation.transformation import (
|
||||
LiteLLMResponsesTransformationHandler,
|
||||
)
|
||||
|
||||
converted = LiteLLMResponsesTransformationHandler()._convert_tools_to_responses_format(
|
||||
[{"type": "tool_search"}, tool]
|
||||
)
|
||||
|
||||
assert converted == [
|
||||
{"type": "tool_search"},
|
||||
{
|
||||
"type": "function",
|
||||
"name": "lookup",
|
||||
"parameters": None,
|
||||
"strict": None,
|
||||
"description": None,
|
||||
"defer_loading": expected_defer_loading,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
_TOOL_SEARCH: Final = {"type": "tool_search"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"deferred_tool",
|
||||
[
|
||||
{"type": "function", "function": {"name": "lookup"}, "defer_loading": True},
|
||||
{"type": "function", "function": {"name": "lookup", "defer_loading": True}},
|
||||
{"type": "function", "name": "lookup", "defer_loading": True},
|
||||
],
|
||||
ids=["tool-level", "function-level", "flat"],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("drop_params", "with_tool_search", "expected_defer_loading"),
|
||||
[(True, False, None), (True, True, True), (False, True, True)],
|
||||
ids=["drop-params-no-search", "drop-params-with-search", "no-drop-params-with-search"],
|
||||
)
|
||||
def test_transform_request_forwards_defer_loading_only_beside_tool_search(
|
||||
deferred_tool, drop_params, with_tool_search, expected_defer_loading
|
||||
):
|
||||
tools: Final = [_TOOL_SEARCH, deferred_tool] if with_tool_search else [deferred_tool]
|
||||
|
||||
result = LiteLLMResponsesTransformationHandler().transform_request(
|
||||
model="gpt-5.5",
|
||||
messages=[{"role": "user", "content": "What is the weather in Paris?"}],
|
||||
optional_params={"tools": tools},
|
||||
litellm_params={"drop_params": drop_params},
|
||||
headers={},
|
||||
litellm_logging_obj=Mock(),
|
||||
)
|
||||
|
||||
lookup_tool: Final = next(tool for tool in result["tools"] if tool.get("name") == "lookup")
|
||||
assert lookup_tool.get("defer_loading") == expected_defer_loading
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"deferred_tool",
|
||||
[
|
||||
{"type": "function", "function": {"name": "lookup"}, "defer_loading": True},
|
||||
{"type": "function", "function": {"name": "lookup", "defer_loading": True}},
|
||||
],
|
||||
ids=["tool-level", "function-level"],
|
||||
)
|
||||
@pytest.mark.parametrize("drop_params", [False, True], ids=["no-drop-params", "drop-params"])
|
||||
@pytest.mark.parametrize(
|
||||
"tool_search_tool",
|
||||
[{"type": "tool_search", "execution": "client"}, {"type": "tool_search"}],
|
||||
ids=["client-only", "hosted-and-client"],
|
||||
)
|
||||
def test_transform_request_drops_defer_loading_beside_client_tool_search(
|
||||
deferred_tool, drop_params, tool_search_tool
|
||||
):
|
||||
tool_search_tools: Final = (
|
||||
[tool_search_tool]
|
||||
if tool_search_tool.get("execution") == "client"
|
||||
else [tool_search_tool, {"type": "tool_search", "execution": "client"}]
|
||||
)
|
||||
|
||||
result = LiteLLMResponsesTransformationHandler().transform_request(
|
||||
model="gpt-5.5",
|
||||
messages=[{"role": "user", "content": "What is the weather in Paris?"}],
|
||||
optional_params={"tools": [*tool_search_tools, deferred_tool]},
|
||||
litellm_params={"drop_params": drop_params},
|
||||
headers={},
|
||||
litellm_logging_obj=Mock(),
|
||||
)
|
||||
|
||||
lookup_tool: Final = next(tool for tool in result["tools"] if tool.get("name") == "lookup")
|
||||
assert "defer_loading" not in lookup_tool
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"deferred_tool",
|
||||
[
|
||||
{"type": "function", "function": {"name": "lookup"}, "defer_loading": True},
|
||||
{"type": "function", "function": {"name": "lookup", "defer_loading": True}},
|
||||
{"type": "function", "name": "lookup", "defer_loading": True},
|
||||
],
|
||||
ids=["tool-level", "function-level", "flat"],
|
||||
)
|
||||
def test_convert_tools_to_responses_format_rejects_deferred_tool_without_tool_search(deferred_tool):
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=r"tool_search.*drop_params"):
|
||||
LiteLLMResponsesTransformationHandler()._convert_tools_to_responses_format([deferred_tool])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tool",
|
||||
[
|
||||
{"type": "function", "function": {"name": "lookup"}, "defer_loading": False},
|
||||
{"type": "function", "name": "lookup", "defer_loading": False},
|
||||
],
|
||||
ids=["nested", "flat"],
|
||||
)
|
||||
def test_convert_tools_to_responses_format_omits_false_defer_loading_without_tool_search(tool):
|
||||
converted: Final = LiteLLMResponsesTransformationHandler()._convert_tools_to_responses_format([tool])
|
||||
|
||||
assert "defer_loading" not in converted[0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tool",
|
||||
[
|
||||
{"type": "function", "function": {"name": "lookup"}, "defer_loading": "true"},
|
||||
{"type": "function", "function": {"name": "lookup", "defer_loading": "yes"}},
|
||||
{"type": "function", "name": "lookup", "defer_loading": "true"},
|
||||
],
|
||||
ids=["tool-level", "function-level", "flat"],
|
||||
)
|
||||
def test_convert_tools_to_responses_format_rejects_non_bool_defer_loading_without_drop_params(tool):
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match="defer_loading must be a boolean"):
|
||||
LiteLLMResponsesTransformationHandler()._convert_tools_to_responses_format([_TOOL_SEARCH, tool])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tool",
|
||||
[
|
||||
{"type": "function", "function": {"name": "lookup"}, "defer_loading": "true"},
|
||||
{"type": "function", "name": "lookup", "defer_loading": "true"},
|
||||
],
|
||||
ids=["nested", "flat"],
|
||||
)
|
||||
def test_convert_tools_to_responses_format_drops_non_bool_defer_loading_under_drop_params(tool):
|
||||
converted: Final = LiteLLMResponsesTransformationHandler()._convert_tools_to_responses_format(
|
||||
[_TOOL_SEARCH, tool], drop_params=True
|
||||
)
|
||||
|
||||
lookup_tool: Final = next(item for item in converted if item.get("name") == "lookup")
|
||||
assert "defer_loading" not in lookup_tool
|
||||
|
||||
|
||||
def test_extract_extra_body_params_reasoning_effort_override():
|
||||
"""Test that reasoning_effort from extra_body overrides top-level reasoning_effort"""
|
||||
from litellm.completion_extras.litellm_responses_transformation.transformation import (
|
||||
|
|
|
|||
|
|
@ -1244,6 +1244,37 @@ def test_responses_api_bridge_check_blank_api_base_is_default_openai(blank_api_b
|
|||
assert model_info.get("mode") == "responses"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("custom_llm_provider", "reasoning_effort", "api_base", "expected_mode"),
|
||||
[
|
||||
("openai", "none", None, "responses"),
|
||||
("openai", {"effort": "none"}, None, "responses"),
|
||||
("azure", "none", "https://example.openai.azure.com", "responses"),
|
||||
("openai", "none", "http://vllm.internal:8000/v1", None),
|
||||
],
|
||||
ids=["openai-effort-none", "openai-dict-effort-none", "azure-effort-none", "chat-only-custom-base"],
|
||||
)
|
||||
def test_responses_api_bridge_check_tool_search_routes_to_responses_on_openai_endpoints(
|
||||
custom_llm_provider, reasoning_effort, api_base, expected_mode
|
||||
):
|
||||
from litellm.main import responses_api_bridge_check
|
||||
|
||||
with patch("litellm.responses.bridge_check.get_model_info_helper") as mock_get_model_info:
|
||||
mock_get_model_info.return_value = {"max_tokens": 128000}
|
||||
model_info, _ = responses_api_bridge_check(
|
||||
model="gpt-5.5",
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
tools=[
|
||||
{"type": "tool_search"},
|
||||
{"type": "function", "function": {"name": "get_weather"}, "defer_loading": True},
|
||||
],
|
||||
reasoning_effort=reasoning_effort,
|
||||
api_base=api_base,
|
||||
)
|
||||
|
||||
assert model_info.get("mode") == expected_mode
|
||||
|
||||
|
||||
def test_responses_api_bridge_check_custom_api_base_with_unset_effort_stays_chat():
|
||||
"""
|
||||
Chat-only OpenAI-compatible backends registered under the openai provider with a
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue