mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(anthropic): keep tool search deferral through forced tools, drop lists, guardrails, and custom
A tool_choice forcing a deferred function now sends that function loaded, so OpenAI can find it. additional_drop_params naming tool_choice or tools[*].defer_loading is read before tool search is kept. A wrapped custom.defer_loading counts as deferral, a guardrail write-back keeps defer_loading, and an unversioned typed tool keeps the client's own schema.
This commit is contained in:
parent
0150e87bf7
commit
45d9d5c95c
6 changed files with 203 additions and 23 deletions
|
|
@ -258,6 +258,27 @@ def _with_original_name(tool: ChatCompletionToolParam, original_names: Mapping[s
|
|||
return {**tool, "function": {**function, "name": original}}
|
||||
|
||||
|
||||
def _request_deferrals(request: Mapping[str, object]) -> Mapping[str, object]:
|
||||
tools: Final = tuple(
|
||||
fields for tool in list_field(request, "tools") if (fields := as_str_mapping(tool)) is not None
|
||||
)
|
||||
return MappingProxyType(
|
||||
{
|
||||
name: tool["defer_loading"]
|
||||
for tool in tools
|
||||
if isinstance(name := tool.get("name"), str) and "defer_loading" in tool
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _with_request_deferral(tool: AllAnthropicToolsValues, deferrals: Mapping[str, object]) -> Mapping[str, object]:
|
||||
fields: Final = as_str_mapping(tool) or {}
|
||||
name: Final = fields.get("name")
|
||||
if "defer_loading" in fields or not isinstance(name, str) or name not in deferrals:
|
||||
return fields
|
||||
return {**fields, "defer_loading": deferrals[name]}
|
||||
|
||||
|
||||
def _write_back_system_block(system: object, block_idx: int, response: str) -> None:
|
||||
if not isinstance(system, list):
|
||||
return
|
||||
|
|
@ -688,9 +709,8 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
model: Final = data.get("model")
|
||||
if model:
|
||||
inputs["model"] = model
|
||||
original_tool_names: Final = _original_tool_names(
|
||||
cast(Mapping[str, object], data) # cast-ok: Anthropic request body
|
||||
)
|
||||
request_body: Final = cast(Mapping[str, object], data) # cast-ok: Anthropic request body
|
||||
original_tool_names: Final = _original_tool_names(request_body)
|
||||
guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail(
|
||||
inputs=inputs,
|
||||
request_data=data,
|
||||
|
|
@ -703,13 +723,14 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
if guardrailed_tools is not None:
|
||||
# Convert tools back from OpenAI format to Anthropic format
|
||||
anthropic_config: Final = AnthropicConfig()
|
||||
anthropic_tools: Final[list[AllAnthropicToolsValues]] = []
|
||||
deferrals: Final = _request_deferrals(request_body)
|
||||
anthropic_tools: Final[list[Mapping[str, object]]] = []
|
||||
for tool in guardrailed_tools:
|
||||
converted_tool, mcp_server = anthropic_config.map_tool_helper(
|
||||
_with_original_name(tool, original_tool_names.declared)
|
||||
)
|
||||
if converted_tool is not None:
|
||||
anthropic_tools.append(converted_tool)
|
||||
anthropic_tools.append(_with_request_deferral(converted_tool, deferrals))
|
||||
# Note: MCP servers are handled separately in the main transformation
|
||||
data["tools"] = (
|
||||
merge_returned_tools_into_request_tools(
|
||||
|
|
|
|||
|
|
@ -183,6 +183,9 @@ def _provider_returns_encrypted_reasoning(model: str, custom_llm_provider: objec
|
|||
return responses_config is not None and "include" in responses_config.get_supported_openai_params(provider_model)
|
||||
|
||||
|
||||
_DEFERRAL_DROP_PATH: Final = "tools[*].defer_loading"
|
||||
|
||||
|
||||
def _maps_tool_search(model: str, forwarded_kwargs: Mapping[str, object]) -> bool:
|
||||
provider: Final = _resolved_provider(model, forwarded_kwargs.get("custom_llm_provider"))
|
||||
return provider == LlmProviders.OPENAI.value and openai_runs_tool_search(
|
||||
|
|
@ -315,8 +318,9 @@ def _build_responses_kwargs(
|
|||
request_data["temperature"] = temperature
|
||||
if thinking:
|
||||
request_data["thinking"] = thinking
|
||||
drops_tools: Final = "tools" in drop_policy(forwarded_kwargs).additional_drop_params
|
||||
if tool_choice and not drops_tools:
|
||||
dropped: Final = drop_policy(forwarded_kwargs).additional_drop_params
|
||||
drops_tools: Final = "tools" in dropped
|
||||
if tool_choice and not drops_tools and "tool_choice" not in dropped:
|
||||
request_data["tool_choice"] = tool_choice
|
||||
if tools and not drops_tools:
|
||||
request_data["tools"] = tools
|
||||
|
|
@ -333,7 +337,7 @@ def _build_responses_kwargs(
|
|||
),
|
||||
auto_summary_level=auto_reasoning_summary(model, forwarded_kwargs.get("custom_llm_provider")),
|
||||
drop_params=drops_unsupported_params(forwarded_kwargs.get("drop_params")),
|
||||
maps_tool_search=_maps_tool_search(model, forwarded_kwargs),
|
||||
maps_tool_search=_maps_tool_search(model, forwarded_kwargs) and _DEFERRAL_DROP_PATH not in dropped,
|
||||
)
|
||||
|
||||
# Normalize reasoning effort based on model capabilities
|
||||
|
|
|
|||
|
|
@ -236,16 +236,38 @@ def is_tool_search(tool: Mapping[str, object]) -> bool:
|
|||
return isinstance(tool_type, str) and tool_type in ANTHROPIC_TOOL_SEARCH_TOOL_TYPES
|
||||
|
||||
|
||||
def _defers_a_function(tool: Mapping[str, object]) -> bool:
|
||||
translated: Final = _translated_tool(tool, False) if tool.get("defer_loading") is True else None
|
||||
def forced_tool_name(tool_choice: object) -> str | None:
|
||||
match tool_choice:
|
||||
case {"type": "tool", "name": str() as name}:
|
||||
return name
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def _defer_loading(tool: Mapping[str, object], forced: str | None) -> object:
|
||||
if forced is not None and tool.get("name") == forced:
|
||||
return None
|
||||
if "defer_loading" in tool:
|
||||
return tool["defer_loading"]
|
||||
match tool.get("custom"):
|
||||
case {"defer_loading": bool() as wrapped}:
|
||||
return wrapped
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def _defers_a_function(tool: Mapping[str, object], forced: str | None) -> bool:
|
||||
translated: Final = _translated_tool(tool, False) if _defer_loading(tool, forced) is True else None
|
||||
return isinstance(translated, ResponsesTool) and translated.payload.get("type") == "function"
|
||||
|
||||
|
||||
def keeps_tool_search(tools: Sequence[Mapping[str, object]], *, maps_tool_search: bool) -> bool:
|
||||
def keeps_tool_search(
|
||||
tools: Sequence[Mapping[str, object]], *, maps_tool_search: bool, forced: str | None = None
|
||||
) -> bool:
|
||||
return (
|
||||
maps_tool_search
|
||||
and any(is_tool_search(tool) for tool in tools)
|
||||
and any(_defers_a_function(tool) for tool in tools)
|
||||
and any(_defers_a_function(tool, forced) for tool in tools)
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -257,9 +279,11 @@ def _with_deferral(translated: TranslatedTool, defer_loading: object) -> Transla
|
|||
return translated
|
||||
|
||||
|
||||
def responses_tool(tool: Mapping[str, object], *, runs_tool_search: bool = False) -> TranslatedTool:
|
||||
def responses_tool(
|
||||
tool: Mapping[str, object], *, runs_tool_search: bool = False, forced: str | None = None
|
||||
) -> TranslatedTool:
|
||||
translated: Final = _translated_tool(tool, runs_tool_search)
|
||||
return _with_deferral(translated, tool.get("defer_loading")) if runs_tool_search else translated
|
||||
return _with_deferral(translated, _defer_loading(tool, forced)) if runs_tool_search else translated
|
||||
|
||||
|
||||
def _translated_tool(tool: Mapping[str, object], runs_tool_search: bool) -> TranslatedTool:
|
||||
|
|
@ -272,6 +296,8 @@ def _translated_tool(tool: Mapping[str, object], runs_tool_search: bool) -> Tran
|
|||
return _hosted(name, {"type": "web_search_preview"})
|
||||
if runs_tool_search and is_tool_search(tool):
|
||||
return _tool_search(name, tool)
|
||||
if not version and family not in _UNVERSIONED_SERVER_TOOL_TYPES:
|
||||
return _custom_function(name, tool)
|
||||
match family:
|
||||
case "bash":
|
||||
return _bash_tool(name)
|
||||
|
|
@ -281,10 +307,8 @@ def _translated_tool(tool: Mapping[str, object], runs_tool_search: bool) -> Tran
|
|||
return _memory_tool(name)
|
||||
case "computer":
|
||||
return _computer_tool(name, version, tool)
|
||||
case _ if version or family in _UNVERSIONED_SERVER_TOOL_TYPES:
|
||||
return UntranslatableTool(name=name, type=type_name)
|
||||
case _:
|
||||
return _custom_function(name, tool)
|
||||
return UntranslatableTool(name=name, type=type_name)
|
||||
|
||||
|
||||
_DROP_PARAMS_HOW_TO: Final = (
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ from litellm.llms.anthropic.messages.responses_adapters.tool_translation import
|
|||
ResponsesTool,
|
||||
TranslatedTool,
|
||||
UntranslatableTool,
|
||||
forced_tool_name,
|
||||
keeps_tool_search,
|
||||
log_dropped_tools,
|
||||
responses_tool,
|
||||
|
|
@ -471,9 +472,12 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
*,
|
||||
drop_params: bool = True,
|
||||
maps_tool_search: bool = False,
|
||||
forced: str | None = None,
|
||||
) -> tuple[TranslatedTool, ...]:
|
||||
runs_tool_search: Final = keeps_tool_search(tools, maps_tool_search=maps_tool_search)
|
||||
translated: Final = tuple(responses_tool(tool, runs_tool_search=runs_tool_search) for tool in tools)
|
||||
runs_tool_search: Final = keeps_tool_search(tools, maps_tool_search=maps_tool_search, forced=forced)
|
||||
translated: Final = tuple(
|
||||
responses_tool(tool, runs_tool_search=runs_tool_search, forced=forced) for tool in tools
|
||||
)
|
||||
untranslatable: Final = tuple(tool for tool in translated if isinstance(tool, UntranslatableTool))
|
||||
if untranslatable and not drop_params:
|
||||
raise untranslatable_tools_error(untranslatable)
|
||||
|
|
@ -671,20 +675,19 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
if "top_p" in anthropic_request:
|
||||
responses_kwargs["top_p"] = anthropic_request["top_p"]
|
||||
|
||||
tool_choice: Final = cast(AnthropicMessagesToolChoice | None, anthropic_request.get("tool_choice"))
|
||||
translated_tools: Final = self.translated_tools(
|
||||
cast(list[AllAnthropicToolsValues], anthropic_request.get("tools") or []),
|
||||
drop_params=drop_params,
|
||||
maps_tool_search=maps_tool_search,
|
||||
forced=forced_tool_name(tool_choice),
|
||||
)
|
||||
responses_tools: Final = [dict(tool.payload) for tool in translated_tools if isinstance(tool, ResponsesTool)]
|
||||
if responses_tools:
|
||||
responses_kwargs["tools"] = responses_tools
|
||||
|
||||
tool_choice: Final = anthropic_request.get("tool_choice")
|
||||
responses_tool_choice: Final = (
|
||||
self.translate_tool_choice_to_responses_api(
|
||||
cast(AnthropicMessagesToolChoice, tool_choice), translated_tools, drop_params=drop_params
|
||||
)
|
||||
self.translate_tool_choice_to_responses_api(tool_choice, translated_tools, drop_params=drop_params)
|
||||
if tool_choice and (responses_tools or not translated_tools)
|
||||
else None
|
||||
)
|
||||
|
|
|
|||
|
|
@ -649,6 +649,30 @@ class TestAnthropicMessagesHandlerInputProcessing:
|
|||
assert {"googleMaps": {"enable_widget": True}} in data["tools"]
|
||||
assert [tool["name"] for tool in data["tools"] if "name" in tool] == ["get_weather"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_deferred_tool_keeps_its_defer_loading_through_a_guardrail_that_returns_the_tools(self):
|
||||
handler = AnthropicMessagesHandler()
|
||||
guardrail = MockPassThroughGuardrail(guardrail_name="test")
|
||||
data = {
|
||||
"model": "gpt-5.6-luna",
|
||||
"messages": [{"role": "user", "content": "what is AAPL at?"}],
|
||||
"tools": [
|
||||
{"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"},
|
||||
{"name": "get_stock", "input_schema": {"type": "object", "properties": {}}, "defer_loading": True},
|
||||
{"name": "get_weather", "input_schema": {"type": "object", "properties": {}}, "defer_loading": False},
|
||||
{"name": "get_time", "input_schema": {"type": "object", "properties": {}}},
|
||||
],
|
||||
}
|
||||
|
||||
await handler.process_input_messages(data=data, guardrail_to_apply=guardrail)
|
||||
|
||||
assert [(tool["name"], tool.get("defer_loading")) for tool in data["tools"]] == [
|
||||
("tool_search_tool_regex", None),
|
||||
("get_stock", True),
|
||||
("get_weather", False),
|
||||
("get_time", None),
|
||||
]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_midturn_system_correction_is_guardrailed_when_top_level_system_is_skipped(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -962,6 +962,110 @@ async def test_additional_drop_params_tools_sends_no_tool_choice_forcing_a_dropp
|
|||
assert "tool_choice" not in sent
|
||||
|
||||
|
||||
DEFERRED_STOCK: Final = {
|
||||
"name": "get_stock",
|
||||
"description": "Price for a ticker",
|
||||
"input_schema": {"type": "object", "properties": {"ticker": {"type": "string"}}, "required": ["ticker"]},
|
||||
"defer_loading": True,
|
||||
}
|
||||
FORCE_STOCK: Final = {"type": "tool", "name": "get_stock"}
|
||||
|
||||
|
||||
def _tool_search_wire(upstream: respx.Route) -> list[tuple[object, ...]]:
|
||||
return [(tool["type"], tool.get("name"), tool.get("defer_loading")) for tool in _tools_sent(upstream)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_tool_choice_forcing_a_deferred_function_sends_that_function_loaded_beside_tool_search(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
upstream: Final = await _openai_request_for(
|
||||
respx_mock,
|
||||
monkeypatch,
|
||||
[TOOL_SEARCH, DEFERRED_STOCK, DEFERRED_WEATHER],
|
||||
drop_params=False,
|
||||
tool_choice=dict(FORCE_STOCK),
|
||||
)
|
||||
|
||||
assert _tool_search_wire(upstream) == [
|
||||
("tool_search", None, None),
|
||||
("function", "get_stock", None),
|
||||
("function", "get_weather", True),
|
||||
]
|
||||
assert json.loads(upstream.calls.last.request.content)["tool_choice"] == {"type": "function", "name": "get_stock"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_drop_params_drops_tool_search_when_the_forced_function_is_the_only_deferred_one(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
upstream: Final = await _openai_request_for(
|
||||
respx_mock, monkeypatch, [TOOL_SEARCH, DEFERRED_STOCK], drop_params=True, tool_choice=dict(FORCE_STOCK)
|
||||
)
|
||||
|
||||
assert _tool_search_wire(upstream) == [("function", "get_stock", None)]
|
||||
assert json.loads(upstream.calls.last.request.content)["tool_choice"] == {"type": "function", "name": "get_stock"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_additional_drop_params_tool_choice_sends_tool_search_without_the_choice_that_forces_it(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
upstream: Final = await _openai_request_for(
|
||||
respx_mock,
|
||||
monkeypatch,
|
||||
[TOOL_SEARCH, DEFERRED_WEATHER],
|
||||
drop_params=False,
|
||||
additional_drop_params=["tool_choice"],
|
||||
tool_choice={"type": "tool", "name": "tool_search_tool_bm25"},
|
||||
)
|
||||
|
||||
assert "tool_choice" not in json.loads(upstream.calls.last.request.content)
|
||||
assert _tool_search_wire(upstream) == [("tool_search", None, None), ("function", "get_weather", True)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_additional_drop_params_of_every_defer_loading_answers_400_for_tool_search_without_drop_params(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
with pytest.raises(litellm.UnsupportedParamsError) as error:
|
||||
await _openai_request_for(
|
||||
respx_mock,
|
||||
monkeypatch,
|
||||
[TOOL_SEARCH, DEFERRED_WEATHER],
|
||||
drop_params=False,
|
||||
additional_drop_params=["tools[*].defer_loading"],
|
||||
)
|
||||
|
||||
assert "tool_search_tool_bm25_20251119" in error.value.message
|
||||
assert not respx_mock.calls
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_deferral_inside_the_custom_object_keeps_tool_search_with_the_tool_deferred(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
wrapped: Final = {**UNDEFERRED_WEATHER, "custom": {"defer_loading": True}}
|
||||
|
||||
upstream: Final = await _openai_request_for(respx_mock, monkeypatch, [TOOL_SEARCH, wrapped], drop_params=False)
|
||||
|
||||
assert _tool_search_wire(upstream) == [("tool_search", None, None), ("function", "get_weather", True)]
|
||||
assert all("custom" not in tool for tool in _tools_sent(upstream))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_unversioned_typed_tool_name_reaches_openai_with_the_clients_own_schema(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
schema: Final = {"type": "object", "properties": {"script": {"type": "string"}}, "required": ["script"]}
|
||||
|
||||
upstream: Final = await _openai_request_for(
|
||||
respx_mock, monkeypatch, [{"type": "bash", "name": "bash", "input_schema": schema}], drop_params=False
|
||||
)
|
||||
|
||||
assert [(tool["name"], tool["parameters"]) for tool in _tools_sent(upstream)] == [("bash", schema)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_tool_openai_cannot_run_answers_400_before_any_openai_call_without_drop_params(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue