fix(anthropic): keep tool search deferral through forced tools, drop lists, guardrails, and custom

A tool_choice forcing a deferred function now sends that function loaded, so OpenAI can find it.
additional_drop_params naming tool_choice or tools[*].defer_loading is read before tool search is kept.
A wrapped custom.defer_loading counts as deferral, a guardrail write-back keeps defer_loading, and an
unversioned typed tool keeps the client's own schema.
This commit is contained in:
mateo-berri 2026-10-30 08:53:56 -07:00
parent 0150e87bf7
commit 45d9d5c95c
6 changed files with 203 additions and 23 deletions

View file

@ -258,6 +258,27 @@ def _with_original_name(tool: ChatCompletionToolParam, original_names: Mapping[s
return {**tool, "function": {**function, "name": original}}
def _request_deferrals(request: Mapping[str, object]) -> Mapping[str, object]:
tools: Final = tuple(
fields for tool in list_field(request, "tools") if (fields := as_str_mapping(tool)) is not None
)
return MappingProxyType(
{
name: tool["defer_loading"]
for tool in tools
if isinstance(name := tool.get("name"), str) and "defer_loading" in tool
}
)
def _with_request_deferral(tool: AllAnthropicToolsValues, deferrals: Mapping[str, object]) -> Mapping[str, object]:
fields: Final = as_str_mapping(tool) or {}
name: Final = fields.get("name")
if "defer_loading" in fields or not isinstance(name, str) or name not in deferrals:
return fields
return {**fields, "defer_loading": deferrals[name]}
def _write_back_system_block(system: object, block_idx: int, response: str) -> None:
if not isinstance(system, list):
return
@ -688,9 +709,8 @@ class AnthropicMessagesHandler(BaseTranslation):
model: Final = data.get("model")
if model:
inputs["model"] = model
original_tool_names: Final = _original_tool_names(
cast(Mapping[str, object], data) # cast-ok: Anthropic request body
)
request_body: Final = cast(Mapping[str, object], data) # cast-ok: Anthropic request body
original_tool_names: Final = _original_tool_names(request_body)
guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail(
inputs=inputs,
request_data=data,
@ -703,13 +723,14 @@ class AnthropicMessagesHandler(BaseTranslation):
if guardrailed_tools is not None:
# Convert tools back from OpenAI format to Anthropic format
anthropic_config: Final = AnthropicConfig()
anthropic_tools: Final[list[AllAnthropicToolsValues]] = []
deferrals: Final = _request_deferrals(request_body)
anthropic_tools: Final[list[Mapping[str, object]]] = []
for tool in guardrailed_tools:
converted_tool, mcp_server = anthropic_config.map_tool_helper(
_with_original_name(tool, original_tool_names.declared)
)
if converted_tool is not None:
anthropic_tools.append(converted_tool)
anthropic_tools.append(_with_request_deferral(converted_tool, deferrals))
# Note: MCP servers are handled separately in the main transformation
data["tools"] = (
merge_returned_tools_into_request_tools(

View file

@ -183,6 +183,9 @@ def _provider_returns_encrypted_reasoning(model: str, custom_llm_provider: objec
return responses_config is not None and "include" in responses_config.get_supported_openai_params(provider_model)
_DEFERRAL_DROP_PATH: Final = "tools[*].defer_loading"
def _maps_tool_search(model: str, forwarded_kwargs: Mapping[str, object]) -> bool:
provider: Final = _resolved_provider(model, forwarded_kwargs.get("custom_llm_provider"))
return provider == LlmProviders.OPENAI.value and openai_runs_tool_search(
@ -315,8 +318,9 @@ def _build_responses_kwargs(
request_data["temperature"] = temperature
if thinking:
request_data["thinking"] = thinking
drops_tools: Final = "tools" in drop_policy(forwarded_kwargs).additional_drop_params
if tool_choice and not drops_tools:
dropped: Final = drop_policy(forwarded_kwargs).additional_drop_params
drops_tools: Final = "tools" in dropped
if tool_choice and not drops_tools and "tool_choice" not in dropped:
request_data["tool_choice"] = tool_choice
if tools and not drops_tools:
request_data["tools"] = tools
@ -333,7 +337,7 @@ def _build_responses_kwargs(
),
auto_summary_level=auto_reasoning_summary(model, forwarded_kwargs.get("custom_llm_provider")),
drop_params=drops_unsupported_params(forwarded_kwargs.get("drop_params")),
maps_tool_search=_maps_tool_search(model, forwarded_kwargs),
maps_tool_search=_maps_tool_search(model, forwarded_kwargs) and _DEFERRAL_DROP_PATH not in dropped,
)
# Normalize reasoning effort based on model capabilities

View file

@ -236,16 +236,38 @@ def is_tool_search(tool: Mapping[str, object]) -> bool:
return isinstance(tool_type, str) and tool_type in ANTHROPIC_TOOL_SEARCH_TOOL_TYPES
def _defers_a_function(tool: Mapping[str, object]) -> bool:
translated: Final = _translated_tool(tool, False) if tool.get("defer_loading") is True else None
def forced_tool_name(tool_choice: object) -> str | None:
match tool_choice:
case {"type": "tool", "name": str() as name}:
return name
case _:
return None
def _defer_loading(tool: Mapping[str, object], forced: str | None) -> object:
if forced is not None and tool.get("name") == forced:
return None
if "defer_loading" in tool:
return tool["defer_loading"]
match tool.get("custom"):
case {"defer_loading": bool() as wrapped}:
return wrapped
case _:
return None
def _defers_a_function(tool: Mapping[str, object], forced: str | None) -> bool:
translated: Final = _translated_tool(tool, False) if _defer_loading(tool, forced) is True else None
return isinstance(translated, ResponsesTool) and translated.payload.get("type") == "function"
def keeps_tool_search(tools: Sequence[Mapping[str, object]], *, maps_tool_search: bool) -> bool:
def keeps_tool_search(
tools: Sequence[Mapping[str, object]], *, maps_tool_search: bool, forced: str | None = None
) -> bool:
return (
maps_tool_search
and any(is_tool_search(tool) for tool in tools)
and any(_defers_a_function(tool) for tool in tools)
and any(_defers_a_function(tool, forced) for tool in tools)
)
@ -257,9 +279,11 @@ def _with_deferral(translated: TranslatedTool, defer_loading: object) -> Transla
return translated
def responses_tool(tool: Mapping[str, object], *, runs_tool_search: bool = False) -> TranslatedTool:
def responses_tool(
tool: Mapping[str, object], *, runs_tool_search: bool = False, forced: str | None = None
) -> TranslatedTool:
translated: Final = _translated_tool(tool, runs_tool_search)
return _with_deferral(translated, tool.get("defer_loading")) if runs_tool_search else translated
return _with_deferral(translated, _defer_loading(tool, forced)) if runs_tool_search else translated
def _translated_tool(tool: Mapping[str, object], runs_tool_search: bool) -> TranslatedTool:
@ -272,6 +296,8 @@ def _translated_tool(tool: Mapping[str, object], runs_tool_search: bool) -> Tran
return _hosted(name, {"type": "web_search_preview"})
if runs_tool_search and is_tool_search(tool):
return _tool_search(name, tool)
if not version and family not in _UNVERSIONED_SERVER_TOOL_TYPES:
return _custom_function(name, tool)
match family:
case "bash":
return _bash_tool(name)
@ -281,10 +307,8 @@ def _translated_tool(tool: Mapping[str, object], runs_tool_search: bool) -> Tran
return _memory_tool(name)
case "computer":
return _computer_tool(name, version, tool)
case _ if version or family in _UNVERSIONED_SERVER_TOOL_TYPES:
return UntranslatableTool(name=name, type=type_name)
case _:
return _custom_function(name, tool)
return UntranslatableTool(name=name, type=type_name)
_DROP_PARAMS_HOW_TO: Final = (

View file

@ -30,6 +30,7 @@ from litellm.llms.anthropic.messages.responses_adapters.tool_translation import
ResponsesTool,
TranslatedTool,
UntranslatableTool,
forced_tool_name,
keeps_tool_search,
log_dropped_tools,
responses_tool,
@ -471,9 +472,12 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
*,
drop_params: bool = True,
maps_tool_search: bool = False,
forced: str | None = None,
) -> tuple[TranslatedTool, ...]:
runs_tool_search: Final = keeps_tool_search(tools, maps_tool_search=maps_tool_search)
translated: Final = tuple(responses_tool(tool, runs_tool_search=runs_tool_search) for tool in tools)
runs_tool_search: Final = keeps_tool_search(tools, maps_tool_search=maps_tool_search, forced=forced)
translated: Final = tuple(
responses_tool(tool, runs_tool_search=runs_tool_search, forced=forced) for tool in tools
)
untranslatable: Final = tuple(tool for tool in translated if isinstance(tool, UntranslatableTool))
if untranslatable and not drop_params:
raise untranslatable_tools_error(untranslatable)
@ -671,20 +675,19 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
if "top_p" in anthropic_request:
responses_kwargs["top_p"] = anthropic_request["top_p"]
tool_choice: Final = cast(AnthropicMessagesToolChoice | None, anthropic_request.get("tool_choice"))
translated_tools: Final = self.translated_tools(
cast(list[AllAnthropicToolsValues], anthropic_request.get("tools") or []),
drop_params=drop_params,
maps_tool_search=maps_tool_search,
forced=forced_tool_name(tool_choice),
)
responses_tools: Final = [dict(tool.payload) for tool in translated_tools if isinstance(tool, ResponsesTool)]
if responses_tools:
responses_kwargs["tools"] = responses_tools
tool_choice: Final = anthropic_request.get("tool_choice")
responses_tool_choice: Final = (
self.translate_tool_choice_to_responses_api(
cast(AnthropicMessagesToolChoice, tool_choice), translated_tools, drop_params=drop_params
)
self.translate_tool_choice_to_responses_api(tool_choice, translated_tools, drop_params=drop_params)
if tool_choice and (responses_tools or not translated_tools)
else None
)

View file

@ -649,6 +649,30 @@ class TestAnthropicMessagesHandlerInputProcessing:
assert {"googleMaps": {"enable_widget": True}} in data["tools"]
assert [tool["name"] for tool in data["tools"] if "name" in tool] == ["get_weather"]
@pytest.mark.asyncio
async def test_a_deferred_tool_keeps_its_defer_loading_through_a_guardrail_that_returns_the_tools(self):
handler = AnthropicMessagesHandler()
guardrail = MockPassThroughGuardrail(guardrail_name="test")
data = {
"model": "gpt-5.6-luna",
"messages": [{"role": "user", "content": "what is AAPL at?"}],
"tools": [
{"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"},
{"name": "get_stock", "input_schema": {"type": "object", "properties": {}}, "defer_loading": True},
{"name": "get_weather", "input_schema": {"type": "object", "properties": {}}, "defer_loading": False},
{"name": "get_time", "input_schema": {"type": "object", "properties": {}}},
],
}
await handler.process_input_messages(data=data, guardrail_to_apply=guardrail)
assert [(tool["name"], tool.get("defer_loading")) for tool in data["tools"]] == [
("tool_search_tool_regex", None),
("get_stock", True),
("get_weather", False),
("get_time", None),
]
@pytest.mark.asyncio
async def test_midturn_system_correction_is_guardrailed_when_top_level_system_is_skipped(
self,

View file

@ -962,6 +962,110 @@ async def test_additional_drop_params_tools_sends_no_tool_choice_forcing_a_dropp
assert "tool_choice" not in sent
DEFERRED_STOCK: Final = {
"name": "get_stock",
"description": "Price for a ticker",
"input_schema": {"type": "object", "properties": {"ticker": {"type": "string"}}, "required": ["ticker"]},
"defer_loading": True,
}
FORCE_STOCK: Final = {"type": "tool", "name": "get_stock"}
def _tool_search_wire(upstream: respx.Route) -> list[tuple[object, ...]]:
return [(tool["type"], tool.get("name"), tool.get("defer_loading")) for tool in _tools_sent(upstream)]
@pytest.mark.asyncio
async def test_a_tool_choice_forcing_a_deferred_function_sends_that_function_loaded_beside_tool_search(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
upstream: Final = await _openai_request_for(
respx_mock,
monkeypatch,
[TOOL_SEARCH, DEFERRED_STOCK, DEFERRED_WEATHER],
drop_params=False,
tool_choice=dict(FORCE_STOCK),
)
assert _tool_search_wire(upstream) == [
("tool_search", None, None),
("function", "get_stock", None),
("function", "get_weather", True),
]
assert json.loads(upstream.calls.last.request.content)["tool_choice"] == {"type": "function", "name": "get_stock"}
@pytest.mark.asyncio
async def test_drop_params_drops_tool_search_when_the_forced_function_is_the_only_deferred_one(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
upstream: Final = await _openai_request_for(
respx_mock, monkeypatch, [TOOL_SEARCH, DEFERRED_STOCK], drop_params=True, tool_choice=dict(FORCE_STOCK)
)
assert _tool_search_wire(upstream) == [("function", "get_stock", None)]
assert json.loads(upstream.calls.last.request.content)["tool_choice"] == {"type": "function", "name": "get_stock"}
@pytest.mark.asyncio
async def test_additional_drop_params_tool_choice_sends_tool_search_without_the_choice_that_forces_it(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
upstream: Final = await _openai_request_for(
respx_mock,
monkeypatch,
[TOOL_SEARCH, DEFERRED_WEATHER],
drop_params=False,
additional_drop_params=["tool_choice"],
tool_choice={"type": "tool", "name": "tool_search_tool_bm25"},
)
assert "tool_choice" not in json.loads(upstream.calls.last.request.content)
assert _tool_search_wire(upstream) == [("tool_search", None, None), ("function", "get_weather", True)]
@pytest.mark.asyncio
async def test_additional_drop_params_of_every_defer_loading_answers_400_for_tool_search_without_drop_params(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
with pytest.raises(litellm.UnsupportedParamsError) as error:
await _openai_request_for(
respx_mock,
monkeypatch,
[TOOL_SEARCH, DEFERRED_WEATHER],
drop_params=False,
additional_drop_params=["tools[*].defer_loading"],
)
assert "tool_search_tool_bm25_20251119" in error.value.message
assert not respx_mock.calls
@pytest.mark.asyncio
async def test_a_deferral_inside_the_custom_object_keeps_tool_search_with_the_tool_deferred(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
wrapped: Final = {**UNDEFERRED_WEATHER, "custom": {"defer_loading": True}}
upstream: Final = await _openai_request_for(respx_mock, monkeypatch, [TOOL_SEARCH, wrapped], drop_params=False)
assert _tool_search_wire(upstream) == [("tool_search", None, None), ("function", "get_weather", True)]
assert all("custom" not in tool for tool in _tools_sent(upstream))
@pytest.mark.asyncio
async def test_an_unversioned_typed_tool_name_reaches_openai_with_the_clients_own_schema(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
schema: Final = {"type": "object", "properties": {"script": {"type": "string"}}, "required": ["script"]}
upstream: Final = await _openai_request_for(
respx_mock, monkeypatch, [{"type": "bash", "name": "bash", "input_schema": schema}], drop_params=False
)
assert [(tool["name"], tool["parameters"]) for tool in _tools_sent(upstream)] == [("bash", schema)]
@pytest.mark.asyncio
async def test_a_tool_openai_cannot_run_answers_400_before_any_openai_call_without_drop_params(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch