diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 31b0f06251d..7d0435976b6 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -258,6 +258,27 @@ def _with_original_name(tool: ChatCompletionToolParam, original_names: Mapping[s return {**tool, "function": {**function, "name": original}} +def _request_deferrals(request: Mapping[str, object]) -> Mapping[str, object]: + tools: Final = tuple( + fields for tool in list_field(request, "tools") if (fields := as_str_mapping(tool)) is not None + ) + return MappingProxyType( + { + name: tool["defer_loading"] + for tool in tools + if isinstance(name := tool.get("name"), str) and "defer_loading" in tool + } + ) + + +def _with_request_deferral(tool: AllAnthropicToolsValues, deferrals: Mapping[str, object]) -> Mapping[str, object]: + fields: Final = as_str_mapping(tool) or {} + name: Final = fields.get("name") + if "defer_loading" in fields or not isinstance(name, str) or name not in deferrals: + return fields + return {**fields, "defer_loading": deferrals[name]} + + def _write_back_system_block(system: object, block_idx: int, response: str) -> None: if not isinstance(system, list): return @@ -688,9 +709,8 @@ class AnthropicMessagesHandler(BaseTranslation): model: Final = data.get("model") if model: inputs["model"] = model - original_tool_names: Final = _original_tool_names( - cast(Mapping[str, object], data) # cast-ok: Anthropic request body - ) + request_body: Final = cast(Mapping[str, object], data) # cast-ok: Anthropic request body + original_tool_names: Final = _original_tool_names(request_body) guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail( inputs=inputs, request_data=data, @@ -703,13 +723,14 @@ class AnthropicMessagesHandler(BaseTranslation): if guardrailed_tools is not None: # Convert tools back from OpenAI format to Anthropic format anthropic_config: Final = AnthropicConfig() - anthropic_tools: Final[list[AllAnthropicToolsValues]] = [] + deferrals: Final = _request_deferrals(request_body) + anthropic_tools: Final[list[Mapping[str, object]]] = [] for tool in guardrailed_tools: converted_tool, mcp_server = anthropic_config.map_tool_helper( _with_original_name(tool, original_tool_names.declared) ) if converted_tool is not None: - anthropic_tools.append(converted_tool) + anthropic_tools.append(_with_request_deferral(converted_tool, deferrals)) # Note: MCP servers are handled separately in the main transformation data["tools"] = ( merge_returned_tools_into_request_tools( diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index 3df19d8151c..0a6f68f9354 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -183,6 +183,9 @@ def _provider_returns_encrypted_reasoning(model: str, custom_llm_provider: objec return responses_config is not None and "include" in responses_config.get_supported_openai_params(provider_model) +_DEFERRAL_DROP_PATH: Final = "tools[*].defer_loading" + + def _maps_tool_search(model: str, forwarded_kwargs: Mapping[str, object]) -> bool: provider: Final = _resolved_provider(model, forwarded_kwargs.get("custom_llm_provider")) return provider == LlmProviders.OPENAI.value and openai_runs_tool_search( @@ -315,8 +318,9 @@ def _build_responses_kwargs( request_data["temperature"] = temperature if thinking: request_data["thinking"] = thinking - drops_tools: Final = "tools" in drop_policy(forwarded_kwargs).additional_drop_params - if tool_choice and not drops_tools: + dropped: Final = drop_policy(forwarded_kwargs).additional_drop_params + drops_tools: Final = "tools" in dropped + if tool_choice and not drops_tools and "tool_choice" not in dropped: request_data["tool_choice"] = tool_choice if tools and not drops_tools: request_data["tools"] = tools @@ -333,7 +337,7 @@ def _build_responses_kwargs( ), auto_summary_level=auto_reasoning_summary(model, forwarded_kwargs.get("custom_llm_provider")), drop_params=drops_unsupported_params(forwarded_kwargs.get("drop_params")), - maps_tool_search=_maps_tool_search(model, forwarded_kwargs), + maps_tool_search=_maps_tool_search(model, forwarded_kwargs) and _DEFERRAL_DROP_PATH not in dropped, ) # Normalize reasoning effort based on model capabilities diff --git a/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py b/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py index 13a41f93d47..4d593a369fa 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py +++ b/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py @@ -236,16 +236,38 @@ def is_tool_search(tool: Mapping[str, object]) -> bool: return isinstance(tool_type, str) and tool_type in ANTHROPIC_TOOL_SEARCH_TOOL_TYPES -def _defers_a_function(tool: Mapping[str, object]) -> bool: - translated: Final = _translated_tool(tool, False) if tool.get("defer_loading") is True else None +def forced_tool_name(tool_choice: object) -> str | None: + match tool_choice: + case {"type": "tool", "name": str() as name}: + return name + case _: + return None + + +def _defer_loading(tool: Mapping[str, object], forced: str | None) -> object: + if forced is not None and tool.get("name") == forced: + return None + if "defer_loading" in tool: + return tool["defer_loading"] + match tool.get("custom"): + case {"defer_loading": bool() as wrapped}: + return wrapped + case _: + return None + + +def _defers_a_function(tool: Mapping[str, object], forced: str | None) -> bool: + translated: Final = _translated_tool(tool, False) if _defer_loading(tool, forced) is True else None return isinstance(translated, ResponsesTool) and translated.payload.get("type") == "function" -def keeps_tool_search(tools: Sequence[Mapping[str, object]], *, maps_tool_search: bool) -> bool: +def keeps_tool_search( + tools: Sequence[Mapping[str, object]], *, maps_tool_search: bool, forced: str | None = None +) -> bool: return ( maps_tool_search and any(is_tool_search(tool) for tool in tools) - and any(_defers_a_function(tool) for tool in tools) + and any(_defers_a_function(tool, forced) for tool in tools) ) @@ -257,9 +279,11 @@ def _with_deferral(translated: TranslatedTool, defer_loading: object) -> Transla return translated -def responses_tool(tool: Mapping[str, object], *, runs_tool_search: bool = False) -> TranslatedTool: +def responses_tool( + tool: Mapping[str, object], *, runs_tool_search: bool = False, forced: str | None = None +) -> TranslatedTool: translated: Final = _translated_tool(tool, runs_tool_search) - return _with_deferral(translated, tool.get("defer_loading")) if runs_tool_search else translated + return _with_deferral(translated, _defer_loading(tool, forced)) if runs_tool_search else translated def _translated_tool(tool: Mapping[str, object], runs_tool_search: bool) -> TranslatedTool: @@ -272,6 +296,8 @@ def _translated_tool(tool: Mapping[str, object], runs_tool_search: bool) -> Tran return _hosted(name, {"type": "web_search_preview"}) if runs_tool_search and is_tool_search(tool): return _tool_search(name, tool) + if not version and family not in _UNVERSIONED_SERVER_TOOL_TYPES: + return _custom_function(name, tool) match family: case "bash": return _bash_tool(name) @@ -281,10 +307,8 @@ def _translated_tool(tool: Mapping[str, object], runs_tool_search: bool) -> Tran return _memory_tool(name) case "computer": return _computer_tool(name, version, tool) - case _ if version or family in _UNVERSIONED_SERVER_TOOL_TYPES: - return UntranslatableTool(name=name, type=type_name) case _: - return _custom_function(name, tool) + return UntranslatableTool(name=name, type=type_name) _DROP_PARAMS_HOW_TO: Final = ( diff --git a/litellm/llms/anthropic/messages/responses_adapters/transformation.py b/litellm/llms/anthropic/messages/responses_adapters/transformation.py index c0765f18670..cd83b75163b 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/messages/responses_adapters/transformation.py @@ -30,6 +30,7 @@ from litellm.llms.anthropic.messages.responses_adapters.tool_translation import ResponsesTool, TranslatedTool, UntranslatableTool, + forced_tool_name, keeps_tool_search, log_dropped_tools, responses_tool, @@ -471,9 +472,12 @@ class LiteLLMAnthropicToResponsesAPIAdapter: *, drop_params: bool = True, maps_tool_search: bool = False, + forced: str | None = None, ) -> tuple[TranslatedTool, ...]: - runs_tool_search: Final = keeps_tool_search(tools, maps_tool_search=maps_tool_search) - translated: Final = tuple(responses_tool(tool, runs_tool_search=runs_tool_search) for tool in tools) + runs_tool_search: Final = keeps_tool_search(tools, maps_tool_search=maps_tool_search, forced=forced) + translated: Final = tuple( + responses_tool(tool, runs_tool_search=runs_tool_search, forced=forced) for tool in tools + ) untranslatable: Final = tuple(tool for tool in translated if isinstance(tool, UntranslatableTool)) if untranslatable and not drop_params: raise untranslatable_tools_error(untranslatable) @@ -671,20 +675,19 @@ class LiteLLMAnthropicToResponsesAPIAdapter: if "top_p" in anthropic_request: responses_kwargs["top_p"] = anthropic_request["top_p"] + tool_choice: Final = cast(AnthropicMessagesToolChoice | None, anthropic_request.get("tool_choice")) translated_tools: Final = self.translated_tools( cast(list[AllAnthropicToolsValues], anthropic_request.get("tools") or []), drop_params=drop_params, maps_tool_search=maps_tool_search, + forced=forced_tool_name(tool_choice), ) responses_tools: Final = [dict(tool.payload) for tool in translated_tools if isinstance(tool, ResponsesTool)] if responses_tools: responses_kwargs["tools"] = responses_tools - tool_choice: Final = anthropic_request.get("tool_choice") responses_tool_choice: Final = ( - self.translate_tool_choice_to_responses_api( - cast(AnthropicMessagesToolChoice, tool_choice), translated_tools, drop_params=drop_params - ) + self.translate_tool_choice_to_responses_api(tool_choice, translated_tools, drop_params=drop_params) if tool_choice and (responses_tools or not translated_tools) else None ) diff --git a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index 9e88b59adc9..6d40173034c 100644 --- a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -649,6 +649,30 @@ class TestAnthropicMessagesHandlerInputProcessing: assert {"googleMaps": {"enable_widget": True}} in data["tools"] assert [tool["name"] for tool in data["tools"] if "name" in tool] == ["get_weather"] + @pytest.mark.asyncio + async def test_a_deferred_tool_keeps_its_defer_loading_through_a_guardrail_that_returns_the_tools(self): + handler = AnthropicMessagesHandler() + guardrail = MockPassThroughGuardrail(guardrail_name="test") + data = { + "model": "gpt-5.6-luna", + "messages": [{"role": "user", "content": "what is AAPL at?"}], + "tools": [ + {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"}, + {"name": "get_stock", "input_schema": {"type": "object", "properties": {}}, "defer_loading": True}, + {"name": "get_weather", "input_schema": {"type": "object", "properties": {}}, "defer_loading": False}, + {"name": "get_time", "input_schema": {"type": "object", "properties": {}}}, + ], + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert [(tool["name"], tool.get("defer_loading")) for tool in data["tools"]] == [ + ("tool_search_tool_regex", None), + ("get_stock", True), + ("get_weather", False), + ("get_time", None), + ] + @pytest.mark.asyncio async def test_midturn_system_correction_is_guardrailed_when_top_level_system_is_skipped( self, diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py index c9a750f43ec..2ccb0f6ad6f 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py @@ -962,6 +962,110 @@ async def test_additional_drop_params_tools_sends_no_tool_choice_forcing_a_dropp assert "tool_choice" not in sent +DEFERRED_STOCK: Final = { + "name": "get_stock", + "description": "Price for a ticker", + "input_schema": {"type": "object", "properties": {"ticker": {"type": "string"}}, "required": ["ticker"]}, + "defer_loading": True, +} +FORCE_STOCK: Final = {"type": "tool", "name": "get_stock"} + + +def _tool_search_wire(upstream: respx.Route) -> list[tuple[object, ...]]: + return [(tool["type"], tool.get("name"), tool.get("defer_loading")) for tool in _tools_sent(upstream)] + + +@pytest.mark.asyncio +async def test_a_tool_choice_forcing_a_deferred_function_sends_that_function_loaded_beside_tool_search( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + upstream: Final = await _openai_request_for( + respx_mock, + monkeypatch, + [TOOL_SEARCH, DEFERRED_STOCK, DEFERRED_WEATHER], + drop_params=False, + tool_choice=dict(FORCE_STOCK), + ) + + assert _tool_search_wire(upstream) == [ + ("tool_search", None, None), + ("function", "get_stock", None), + ("function", "get_weather", True), + ] + assert json.loads(upstream.calls.last.request.content)["tool_choice"] == {"type": "function", "name": "get_stock"} + + +@pytest.mark.asyncio +async def test_drop_params_drops_tool_search_when_the_forced_function_is_the_only_deferred_one( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + upstream: Final = await _openai_request_for( + respx_mock, monkeypatch, [TOOL_SEARCH, DEFERRED_STOCK], drop_params=True, tool_choice=dict(FORCE_STOCK) + ) + + assert _tool_search_wire(upstream) == [("function", "get_stock", None)] + assert json.loads(upstream.calls.last.request.content)["tool_choice"] == {"type": "function", "name": "get_stock"} + + +@pytest.mark.asyncio +async def test_additional_drop_params_tool_choice_sends_tool_search_without_the_choice_that_forces_it( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + upstream: Final = await _openai_request_for( + respx_mock, + monkeypatch, + [TOOL_SEARCH, DEFERRED_WEATHER], + drop_params=False, + additional_drop_params=["tool_choice"], + tool_choice={"type": "tool", "name": "tool_search_tool_bm25"}, + ) + + assert "tool_choice" not in json.loads(upstream.calls.last.request.content) + assert _tool_search_wire(upstream) == [("tool_search", None, None), ("function", "get_weather", True)] + + +@pytest.mark.asyncio +async def test_additional_drop_params_of_every_defer_loading_answers_400_for_tool_search_without_drop_params( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + with pytest.raises(litellm.UnsupportedParamsError) as error: + await _openai_request_for( + respx_mock, + monkeypatch, + [TOOL_SEARCH, DEFERRED_WEATHER], + drop_params=False, + additional_drop_params=["tools[*].defer_loading"], + ) + + assert "tool_search_tool_bm25_20251119" in error.value.message + assert not respx_mock.calls + + +@pytest.mark.asyncio +async def test_a_deferral_inside_the_custom_object_keeps_tool_search_with_the_tool_deferred( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + wrapped: Final = {**UNDEFERRED_WEATHER, "custom": {"defer_loading": True}} + + upstream: Final = await _openai_request_for(respx_mock, monkeypatch, [TOOL_SEARCH, wrapped], drop_params=False) + + assert _tool_search_wire(upstream) == [("tool_search", None, None), ("function", "get_weather", True)] + assert all("custom" not in tool for tool in _tools_sent(upstream)) + + +@pytest.mark.asyncio +async def test_an_unversioned_typed_tool_name_reaches_openai_with_the_clients_own_schema( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + schema: Final = {"type": "object", "properties": {"script": {"type": "string"}}, "required": ["script"]} + + upstream: Final = await _openai_request_for( + respx_mock, monkeypatch, [{"type": "bash", "name": "bash", "input_schema": schema}], drop_params=False + ) + + assert [(tool["name"], tool["parameters"]) for tool in _tools_sent(upstream)] == [("bash", schema)] + + @pytest.mark.asyncio async def test_a_tool_openai_cannot_run_answers_400_before_any_openai_call_without_drop_params( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch