From 9766f7cc02336672b05d11f8804910eee4ebccb9 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 01:30:48 -0700 Subject: [PATCH 01/11] fix(anthropic): send Anthropic typed tools to the Responses API with their schemas --- .../responses_adapters/tool_translation.py | 252 ++++++++++++++++++ .../responses_adapters/transformation.py | 91 ++++--- .../test_responses_adapters_transformation.py | 141 ++++++++++ 3 files changed, 447 insertions(+), 37 deletions(-) create mode 100644 litellm/llms/anthropic/messages/responses_adapters/tool_translation.py diff --git a/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py b/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py new file mode 100644 index 00000000000..dbe8f7593b5 --- /dev/null +++ b/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py @@ -0,0 +1,252 @@ +import re +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import Final, TypeAlias + +from litellm.exceptions import UnsupportedParamsError +from litellm.llms.anthropic.messages.utils import json_copy_or_original + +_VERSIONED_TOOL_TYPE: Final = re.compile(r"(?P[a-z_]+?)_(?P\d{8})") +_UNTRANSLATABLE_FAMILIES: Final = frozenset({"web_fetch", "mcp_toolset", "computer_toolset"}) +_TEXT_EDITOR_INSERT_TEXT_SINCE: Final = "20250429" +_COMPUTER_MODIFIERS_SINCE: Final = "20250124" +_COMPUTER_ZOOM_SINCE: Final = "20251124" +_COMPUTER_ACTIONS_20241022: Final = ( + "key", + "type", + "mouse_move", + "left_click", + "left_click_drag", + "right_click", + "middle_click", + "double_click", + "screenshot", + "cursor_position", +) +_COMPUTER_ACTIONS_20250124: Final = ( + *_COMPUTER_ACTIONS_20241022, + "left_mouse_down", + "left_mouse_up", + "scroll", + "hold_key", + "wait", + "triple_click", +) +_COMPUTER_ACTIONS_20251124: Final = (*_COMPUTER_ACTIONS_20250124, "zoom") + + +@dataclass(frozen=True, slots=True) +class ResponsesTool: + name: str + payload: Mapping[str, object] + choice: Mapping[str, object] + + +@dataclass(frozen=True, slots=True) +class UntranslatableTool: + name: str + type: str + + +TranslatedTool: TypeAlias = ResponsesTool | UntranslatableTool + + +def _family_and_version(tool_type: str) -> tuple[str, str]: + versioned: Final = _VERSIONED_TOOL_TYPE.fullmatch(tool_type) + if versioned is None: + return tool_type, "" + return versioned["family"], versioned["version"] + + +def _string(description: str) -> Mapping[str, object]: + return {"type": "string", "description": description} + + +def _integer(description: str) -> Mapping[str, object]: + return {"type": "integer", "description": description} + + +def _integers(count: int, description: str) -> Mapping[str, object]: + return { + "type": "array", + "items": {"type": "integer"}, + "minItems": count, + "maxItems": count, + "description": description, + } + + +def _enum(values: Sequence[str]) -> Mapping[str, object]: + return {"type": "string", "enum": list(values)} + + +def _function(name: str, description: str, properties: Mapping[str, object], required: Sequence[str]) -> ResponsesTool: + return ResponsesTool( + name=name, + payload={ + "type": "function", + "name": name, + "strict": False, + "description": description, + "parameters": {"type": "object", "properties": properties, "required": list(required)}, + }, + choice={"type": "function", "name": name}, + ) + + +def _hosted(name: str, payload: Mapping[str, object]) -> ResponsesTool: + return ResponsesTool(name=name, payload=payload, choice={"type": payload["type"]}) + + +def _bash_tool(name: str) -> ResponsesTool: + return _function( + name, + "Run a command in a persistent bash session. The working directory, environment, and files carry over " + "between calls.", + { + "command": _string("The bash command to run. Required unless restart is true."), + "restart": {"type": "boolean", "description": "Set to true to restart the bash session."}, + }, + (), + ) + + +def _text_editor_tool(name: str, version: str) -> ResponsesTool: + inserts_insert_text: Final = version >= _TEXT_EDITOR_INSERT_TEXT_SINCE + edit_properties: Final[Mapping[str, object]] = { + "command": _enum( + ("view", "create", "str_replace", "insert") + if inserts_insert_text + else ("view", "create", "str_replace", "insert", "undo_edit") + ), + "path": _string("Absolute path to the file or directory."), + "view_range": _integers(2, "Optional [start, end] line numbers for view. An end of -1 reads to the end."), + "file_text": _string("Content of the new file, for create."), + "old_str": _string("Exact text to replace, for str_replace. It must match exactly one place in the file."), + "new_str": _string( + "Replacement text, for str_replace." + if inserts_insert_text + else "Replacement text for str_replace, or the text to add for insert." + ), + "insert_line": _integer("Line number to insert after, for insert. 0 inserts at the start of the file."), + } + return _function( + name, + "View, create, and edit text files. view lists a directory or shows a file with line numbers.", + ( + {**edit_properties, "insert_text": _string("Text to add, for insert.")} + if inserts_insert_text + else edit_properties + ), + ("command", "path"), + ) + + +def _memory_tool(name: str) -> ResponsesTool: + return _function( + name, + "Read and write files in the /memories directory, which persists between conversations. View it before " + "starting a task, and record progress and learnings as you work.", + { + "command": _enum(("view", "create", "str_replace", "insert", "delete", "rename")), + "path": _string("Path under /memories, for every command but rename."), + "view_range": _integers(2, "Optional [start, end] line numbers for view."), + "file_text": _string("Content of the new file, for create."), + "old_str": _string("Exact text to replace, for str_replace."), + "new_str": _string("Replacement text, for str_replace."), + "insert_line": _integer("Line number to insert after, for insert."), + "insert_text": _string("Text to add, for insert."), + "old_path": _string("Current path, for rename."), + "new_path": _string("New path, for rename."), + }, + ("command",), + ) + + +def _computer_properties(version: str) -> Mapping[str, object]: + pointer: Final[Mapping[str, object]] = { + "text": _string("Text to type for type, or keys to press for key and hold_key, such as ctrl+s."), + "coordinate": _integers(2, "[x, y] pixel position for a click, mouse_move, or the end of left_click_drag."), + "start_coordinate": _integers(2, "[x, y] pixel position where left_click_drag starts."), + } + if version < _COMPUTER_MODIFIERS_SINCE: + return {"action": _enum(_COMPUTER_ACTIONS_20241022), **pointer} + modifiers: Final[Mapping[str, object]] = { + **pointer, + "scroll_direction": _enum(("up", "down", "left", "right")), + "scroll_amount": _integer("Number of scroll steps, for scroll."), + "duration": {"type": "number", "description": "Seconds to wait for wait, or to hold for hold_key."}, + "key": _string("Modifier key to hold during a click, such as shift."), + } + if version < _COMPUTER_ZOOM_SINCE: + return {"action": _enum(_COMPUTER_ACTIONS_20250124), **modifiers} + return { + "action": _enum(_COMPUTER_ACTIONS_20251124), + **modifiers, + "region": _integers(4, "[x0, y0, x1, y1] screen region to show at full resolution, for zoom."), + } + + +def _computer_tool(name: str, version: str, tool: Mapping[str, object]) -> ResponsesTool: + width: Final = tool.get("display_width_px") + height: Final = tool.get("display_height_px") + display: Final = ( + f"a {width}x{height} pixel display" if isinstance(width, int) and isinstance(height, int) else "a display" + ) + return _function( + name, + f"Control {display} with the mouse and keyboard, and take screenshots to see it. Coordinates are pixels " + "from the top left.", + _computer_properties(version), + ("action",), + ) + + +def _custom_function(name: str, tool: Mapping[str, object]) -> ResponsesTool: + description: Final = tool.get("description") + input_schema: Final = tool.get("input_schema") + return ResponsesTool( + name=name, + payload={ + "type": "function", + "name": name, + "strict": bool(tool.get("strict")), + **({"description": description} if "description" in tool else {}), + **({"parameters": json_copy_or_original(input_schema)} if "input_schema" in tool else {}), + }, + choice={"type": "function", "name": name}, + ) + + +def responses_tool(tool: Mapping[str, object]) -> TranslatedTool: + tool_type: Final = tool.get("type") + tool_name: Final = tool.get("name") + name: Final = tool_name if isinstance(tool_name, str) else "" + family, version = _family_and_version(tool_type if isinstance(tool_type, str) else "") + if family == "web_search" or name == "web_search": + return _hosted(name, {"type": "web_search_preview"}) + if family in _UNTRANSLATABLE_FAMILIES: + return UntranslatableTool(name=name, type=str(tool_type)) + match family: + case "code_execution": + return _hosted(name, {"type": "code_interpreter", "container": {"type": "auto"}}) + case "bash": + return _bash_tool(name) + case "text_editor": + return _text_editor_tool(name, version) + case "memory": + return _memory_tool(name) + case "computer": + return _computer_tool(name, version, tool) + case _: + return _custom_function(name, tool) + + +def untranslatable_tools_error(tools: Sequence[UntranslatableTool]) -> UnsupportedParamsError: + types: Final = ", ".join(dict.fromkeys(tool.type for tool in tools)) + return UnsupportedParamsError( + message=( + f"The Responses API has no equivalent for these Anthropic tools: {types}. To drop them, set " + "`litellm.drop_params=True`, or `drop_params: true` under `litellm_settings` on the proxy" + ) + ) diff --git a/litellm/llms/anthropic/messages/responses_adapters/transformation.py b/litellm/llms/anthropic/messages/responses_adapters/transformation.py index 1dd07c1739b..080bffa2f92 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/messages/responses_adapters/transformation.py @@ -25,9 +25,15 @@ from litellm.llms.anthropic.messages.native.utils import ( refusal_stop_details, responses_output_refusal_text, ) +from litellm.llms.anthropic.messages.responses_adapters.tool_translation import ( + ResponsesTool, + TranslatedTool, + UntranslatableTool, + responses_tool, + untranslatable_tools_error, +) from litellm.llms.anthropic.messages.utils import ( is_reasoning_auto_summary_enabled, - json_copy_or_original, prompt_cache_key_from_user_id, ) from litellm.types.llms.anthropic import ( @@ -465,45 +471,51 @@ class LiteLLMAnthropicToResponsesAPIAdapter: return input_items @staticmethod - def _translate_tool_to_responses_api(tool_dict: Mapping[str, object]) -> dict[str, object]: - tool_type: Final = tool_dict.get("type", "") - tool_name: Final = tool_dict.get("name", "") - # web_search tool - if (isinstance(tool_type, str) and tool_type.startswith("web_search")) or tool_name == "web_search": - return {"type": "web_search_preview"} - # Responses turns strict mode on when `strict` is omitted, silently rewriting - # `required` to every property. Anthropic tools are non-strict unless asked. - func_tool: Final[dict[str, object]] = { - "type": "function", - "name": tool_name, - "strict": bool(tool_dict.get("strict")), - } - if "description" in tool_dict: - func_tool["description"] = tool_dict["description"] - if "input_schema" in tool_dict: - func_tool["parameters"] = json_copy_or_original(tool_dict["input_schema"]) - return func_tool + def translated_tools( + tools: Sequence[AllAnthropicToolsValues], + *, + drop_params: bool = True, + ) -> tuple[TranslatedTool, ...]: + translated: Final = tuple(responses_tool(tool) for tool in tools) + untranslatable: Final = tuple(tool for tool in translated if isinstance(tool, UntranslatableTool)) + if untranslatable and not drop_params: + raise untranslatable_tools_error(untranslatable) + return translated def translate_tools_to_responses_api( self, tools: list[AllAnthropicToolsValues], + *, + drop_params: bool = True, ) -> list[dict[str, object]]: - """Convert Anthropic tool definitions to Responses API function tools.""" - return [self._translate_tool_to_responses_api(tool) for tool in tools] + """Convert Anthropic tool definitions to Responses API tools.""" + return [ + dict(tool.payload) + for tool in self.translated_tools(tools, drop_params=drop_params) + if isinstance(tool, ResponsesTool) + ] @staticmethod def translate_tool_choice_to_responses_api( tool_choice: AnthropicMessagesToolChoice, - ) -> str | dict[str, object]: - """Convert Anthropic tool_choice to Responses API tool_choice.""" + tools: Sequence[TranslatedTool] = (), + ) -> str | dict[str, object] | None: + """Convert Anthropic tool_choice to Responses API tool_choice, or None when it names a dropped tool.""" tc_type: Final = tool_choice.get("type") if tc_type == "any": return "required" - elif tc_type == "tool": - return {"type": "function", "name": tool_choice.get("name", "")} - elif tc_type == "none": + if tc_type == "none": return "none" - return "auto" + if tc_type != "tool": + return "auto" + name: Final = tool_choice.get("name", "") + match next((tool for tool in tools if tool.name == name), None): + case ResponsesTool(choice=choice): + return dict(choice) + case UntranslatableTool(): + return None + case None: + return {"type": "function", "name": name} @staticmethod def translate_context_management_to_responses_api( @@ -653,19 +665,24 @@ class LiteLLMAnthropicToResponsesAPIAdapter: if "top_p" in anthropic_request: responses_kwargs["top_p"] = anthropic_request["top_p"] - # tools - tools: Final = anthropic_request.get("tools") - if tools: - responses_kwargs["tools"] = self.translate_tools_to_responses_api( - cast(list[AllAnthropicToolsValues], tools) - ) + translated_tools: Final = self.translated_tools( + cast(list[AllAnthropicToolsValues], anthropic_request.get("tools") or []), + drop_params=drop_params, + ) + responses_tools: Final = [dict(tool.payload) for tool in translated_tools if isinstance(tool, ResponsesTool)] + if responses_tools: + responses_kwargs["tools"] = responses_tools - # tool_choice tool_choice: Final = anthropic_request.get("tool_choice") - if tool_choice: - responses_kwargs["tool_choice"] = self.translate_tool_choice_to_responses_api( - cast(AnthropicMessagesToolChoice, tool_choice) + responses_tool_choice: Final = ( + self.translate_tool_choice_to_responses_api( + cast(AnthropicMessagesToolChoice, tool_choice), translated_tools ) + if tool_choice and (responses_tools or not translated_tools) + else None + ) + if responses_tool_choice is not None: + responses_kwargs["tool_choice"] = responses_tool_choice # thinking -> reasoning thinking: Final = anthropic_request.get("thinking") diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py index f6fcb9e6f1e..87bff52ab52 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py @@ -17,6 +17,7 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, ) +from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, @@ -948,6 +949,146 @@ class TestTranslateToolChoiceToResponsesAPI: assert result == {"type": "function", "name": "get_weather"} +class TestAnthropicTypedToolsOnResponses: + """Anthropic-defined tools carry no input_schema, so the adapter has to supply the schema Claude has built in.""" + + @staticmethod + def _only_tool(tool: Dict[str, Any]) -> Dict[str, Any]: + tools: Final = _ADAPTER.translate_request(_make_request(tools=[tool]), drop_params=False)["tools"] + assert len(tools) == 1 + return tools[0] + + @staticmethod + def _enum(tool: Dict[str, Any], prop: str) -> List[str]: + return tool["parameters"]["properties"][prop]["enum"] + + def test_bash_gets_a_command_parameter(self): + tool: Final = self._only_tool({"type": "bash_20250124", "name": "bash"}) + + assert tool["type"] == "function" + assert tool["name"] == "bash" + assert tool["strict"] is False + assert set(tool["parameters"]["properties"]) == {"command", "restart"} + assert tool["description"] + + def test_text_editor_from_20250429_inserts_insert_text_and_has_no_undo(self): + tool: Final = self._only_tool({"type": "text_editor_20250728", "name": "str_replace_based_edit_tool"}) + + assert tool["name"] == "str_replace_based_edit_tool" + assert self._enum(tool, "command") == ["view", "create", "str_replace", "insert"] + assert "insert_text" in tool["parameters"]["properties"] + assert tool["parameters"]["required"] == ["command", "path"] + + def test_text_editor_before_20250429_inserts_new_str_and_can_undo(self): + tool: Final = self._only_tool({"type": "text_editor_20250124", "name": "str_replace_editor"}) + + assert "undo_edit" in self._enum(tool, "command") + assert "insert_text" not in tool["parameters"]["properties"] + assert "new_str" in tool["parameters"]["properties"] + + def test_memory_gets_every_memory_command(self): + tool: Final = self._only_tool({"type": "memory_20250818", "name": "memory"}) + + assert set(self._enum(tool, "command")) == {"view", "create", "str_replace", "insert", "delete", "rename"} + assert {"old_path", "new_path"} <= set(tool["parameters"]["properties"]) + + @pytest.mark.parametrize( + ("tool_type", "has_scroll", "has_zoom"), + [("computer_20241022", False, False), ("computer_20250124", True, False), ("computer_20251124", True, True)], + ) + def test_computer_actions_follow_the_tool_version(self, tool_type: str, has_scroll: bool, has_zoom: bool): + tool: Final = self._only_tool( + {"type": tool_type, "name": "computer", "display_width_px": 1280, "display_height_px": 800} + ) + actions: Final = self._enum(tool, "action") + properties: Final = tool["parameters"]["properties"] + + assert {"screenshot", "left_click", "type", "key"} <= set(actions) + assert ("scroll" in actions) is has_scroll + assert ("scroll_direction" in properties) is has_scroll + assert ("zoom" in actions) is has_zoom + assert ("region" in properties) is has_zoom + assert "1280x800" in tool["description"] + + def test_code_execution_runs_on_the_hosted_code_interpreter(self): + assert self._only_tool({"type": "code_execution_20250825", "name": "code_execution"}) == { + "type": "code_interpreter", + "container": {"type": "auto"}, + } + + def test_custom_tool_named_like_a_typed_tool_keeps_its_own_schema(self): + schema: Final = {"type": "object", "properties": {"cmd": {"type": "string"}}, "required": ["cmd"]} + + tool: Final = self._only_tool({"name": "bash", "input_schema": schema}) + + assert tool["parameters"] == schema + assert "description" not in tool + + @pytest.mark.parametrize( + "tool", + [ + {"type": "web_fetch_20250910", "name": "web_fetch"}, + {"type": "mcp_toolset", "mcp_server_name": "docs"}, + {"type": "computer_toolset_20260801"}, + ], + ) + def test_tool_with_no_responses_equivalent_is_a_400_without_drop_params(self, tool: Dict[str, Any]): + with pytest.raises(UnsupportedParamsError) as error: + _ADAPTER.translate_request(_make_request(tools=[tool]), drop_params=False) + + assert error.value.status_code == 400 + assert tool["type"] in str(error.value) + + def test_tool_with_no_responses_equivalent_drops_under_drop_params(self): + kwargs: Final = _ADAPTER.translate_request( + _make_request( + tools=[{"type": "web_fetch_20250910", "name": "web_fetch"}, {"type": "bash_20250124", "name": "bash"}] + ), + drop_params=True, + ) + + assert [tool["name"] for tool in kwargs["tools"]] == ["bash"] + + def test_tool_choice_of_a_hosted_tool_names_the_hosted_type(self): + kwargs: Final = _ADAPTER.translate_request( + _make_request( + tools=[ + {"type": "code_execution_20250825", "name": "code_execution"}, + {"type": "web_search_20250305", "name": "web_search"}, + ], + tool_choice={"type": "tool", "name": "code_execution"}, + ) + ) + + assert kwargs["tool_choice"] == {"type": "code_interpreter"} + + def test_tool_choice_of_a_dropped_tool_is_dropped_with_it(self): + kwargs: Final = _ADAPTER.translate_request( + _make_request( + tools=[{"type": "web_fetch_20250910", "name": "web_fetch"}, {"type": "bash_20250124", "name": "bash"}], + tool_choice={"type": "tool", "name": "web_fetch"}, + ), + drop_params=True, + ) + + assert [tool["name"] for tool in kwargs["tools"]] == ["bash"] + assert "tool_choice" not in kwargs + + def test_tool_choice_is_dropped_when_every_tool_was_dropped(self): + kwargs: Final = _ADAPTER.translate_request( + _make_request(tools=[{"type": "web_fetch_20250910", "name": "web_fetch"}], tool_choice={"type": "any"}), + drop_params=True, + ) + + assert "tools" not in kwargs + assert "tool_choice" not in kwargs + + def test_tool_choice_without_tools_still_reaches_the_provider(self): + kwargs: Final = _ADAPTER.translate_request(_make_request(tool_choice={"type": "any"})) + + assert kwargs["tool_choice"] == "required" + + # --------------------------------------------------------------------------- # translate_thinking_to_reasoning # --------------------------------------------------------------------------- From 744bf129571e50058496b165d424825e504cd916 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 06:19:55 -0700 Subject: [PATCH 02/11] fix(anthropic): keep web search types and 400 on unmappable Anthropic tools Every type starting with web_search maps to OpenAI web search again, as it did before this branch. code_execution, tool search, and every other dated Anthropic tool type with no Responses equivalent answer 400 without drop_params and drop with it, since code_interpreter opens paid containers that spend does not record yet. computer_20251124 offers zoom only when enable_zoom is true, and the 400 names per-model drop_params. --- .../responses_adapters/tool_translation.py | 24 +++++------ .../test_responses_adapters_transformation.py | 41 +++++++++++++------ 2 files changed, 41 insertions(+), 24 deletions(-) diff --git a/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py b/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py index dbe8f7593b5..432954a25ed 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py +++ b/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py @@ -6,8 +6,8 @@ from typing import Final, TypeAlias from litellm.exceptions import UnsupportedParamsError from litellm.llms.anthropic.messages.utils import json_copy_or_original -_VERSIONED_TOOL_TYPE: Final = re.compile(r"(?P[a-z_]+?)_(?P\d{8})") -_UNTRANSLATABLE_FAMILIES: Final = frozenset({"web_fetch", "mcp_toolset", "computer_toolset"}) +_VERSIONED_TOOL_TYPE: Final = re.compile(r"(?P[a-z0-9_]+?)_(?P\d{8})") +_UNVERSIONED_SERVER_TOOL_TYPES: Final = frozenset({"mcp_toolset", "computer_toolset"}) _TEXT_EDITOR_INSERT_TEXT_SINCE: Final = "20250429" _COMPUTER_MODIFIERS_SINCE: Final = "20250124" _COMPUTER_ZOOM_SINCE: Final = "20251124" @@ -163,7 +163,7 @@ def _memory_tool(name: str) -> ResponsesTool: ) -def _computer_properties(version: str) -> Mapping[str, object]: +def _computer_properties(version: str, zoom: bool) -> Mapping[str, object]: pointer: Final[Mapping[str, object]] = { "text": _string("Text to type for type, or keys to press for key and hold_key, such as ctrl+s."), "coordinate": _integers(2, "[x, y] pixel position for a click, mouse_move, or the end of left_click_drag."), @@ -178,7 +178,7 @@ def _computer_properties(version: str) -> Mapping[str, object]: "duration": {"type": "number", "description": "Seconds to wait for wait, or to hold for hold_key."}, "key": _string("Modifier key to hold during a click, such as shift."), } - if version < _COMPUTER_ZOOM_SINCE: + if not zoom: return {"action": _enum(_COMPUTER_ACTIONS_20250124), **modifiers} return { "action": _enum(_COMPUTER_ACTIONS_20251124), @@ -197,7 +197,7 @@ def _computer_tool(name: str, version: str, tool: Mapping[str, object]) -> Respo name, f"Control {display} with the mouse and keyboard, and take screenshots to see it. Coordinates are pixels " "from the top left.", - _computer_properties(version), + _computer_properties(version, zoom=version >= _COMPUTER_ZOOM_SINCE and tool.get("enable_zoom") is True), ("action",), ) @@ -222,14 +222,11 @@ def responses_tool(tool: Mapping[str, object]) -> TranslatedTool: tool_type: Final = tool.get("type") tool_name: Final = tool.get("name") name: Final = tool_name if isinstance(tool_name, str) else "" - family, version = _family_and_version(tool_type if isinstance(tool_type, str) else "") - if family == "web_search" or name == "web_search": + type_name: Final = tool_type if isinstance(tool_type, str) else "" + family, version = _family_and_version(type_name) + if type_name.startswith("web_search") or name == "web_search": return _hosted(name, {"type": "web_search_preview"}) - if family in _UNTRANSLATABLE_FAMILIES: - return UntranslatableTool(name=name, type=str(tool_type)) match family: - case "code_execution": - return _hosted(name, {"type": "code_interpreter", "container": {"type": "auto"}}) case "bash": return _bash_tool(name) case "text_editor": @@ -238,6 +235,8 @@ def responses_tool(tool: Mapping[str, object]) -> TranslatedTool: return _memory_tool(name) case "computer": return _computer_tool(name, version, tool) + case _ if version or family in _UNVERSIONED_SERVER_TOOL_TYPES: + return UntranslatableTool(name=name, type=type_name) case _: return _custom_function(name, tool) @@ -247,6 +246,7 @@ def untranslatable_tools_error(tools: Sequence[UntranslatableTool]) -> Unsupport return UnsupportedParamsError( message=( f"The Responses API has no equivalent for these Anthropic tools: {types}. To drop them, set " - "`litellm.drop_params=True`, or `drop_params: true` under `litellm_settings` on the proxy" + "`drop_params: true` in the model's `litellm_params` or under `litellm_settings` on the proxy, or " + "`litellm.drop_params=True` in the SDK" ) ) diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py index 87bff52ab52..cfbc663b7bd 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py @@ -993,12 +993,25 @@ class TestAnthropicTypedToolsOnResponses: assert {"old_path", "new_path"} <= set(tool["parameters"]["properties"]) @pytest.mark.parametrize( - ("tool_type", "has_scroll", "has_zoom"), - [("computer_20241022", False, False), ("computer_20250124", True, False), ("computer_20251124", True, True)], + ("tool_type", "enable_zoom", "has_scroll", "has_zoom"), + [ + ("computer_20241022", True, False, False), + ("computer_20250124", True, True, False), + ("computer_20251124", False, True, False), + ("computer_20251124", True, True, True), + ], ) - def test_computer_actions_follow_the_tool_version(self, tool_type: str, has_scroll: bool, has_zoom: bool): + def test_computer_actions_follow_the_tool_version( + self, tool_type: str, enable_zoom: bool, has_scroll: bool, has_zoom: bool + ): tool: Final = self._only_tool( - {"type": tool_type, "name": "computer", "display_width_px": 1280, "display_height_px": 800} + { + "type": tool_type, + "name": "computer", + "display_width_px": 1280, + "display_height_px": 800, + "enable_zoom": enable_zoom, + } ) actions: Final = self._enum(tool, "action") properties: Final = tool["parameters"]["properties"] @@ -1010,11 +1023,11 @@ class TestAnthropicTypedToolsOnResponses: assert ("region" in properties) is has_zoom assert "1280x800" in tool["description"] - def test_code_execution_runs_on_the_hosted_code_interpreter(self): - assert self._only_tool({"type": "code_execution_20250825", "name": "code_execution"}) == { - "type": "code_interpreter", - "container": {"type": "auto"}, - } + @pytest.mark.parametrize( + "tool_type", ["web_search_20250305", "web_search_preview", "web_search_preview_2025_03_11", "web_search_2025_08_26"] + ) + def test_every_web_search_type_runs_the_hosted_web_search(self, tool_type: str): + assert self._only_tool({"type": tool_type, "name": "web_search"}) == {"type": "web_search_preview"} def test_custom_tool_named_like_a_typed_tool_keeps_its_own_schema(self): schema: Final = {"type": "object", "properties": {"cmd": {"type": "string"}}, "required": ["cmd"]} @@ -1028,6 +1041,9 @@ class TestAnthropicTypedToolsOnResponses: "tool", [ {"type": "web_fetch_20250910", "name": "web_fetch"}, + {"type": "code_execution_20250825", "name": "code_execution"}, + {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"}, + {"type": "tool_search_tool_bm25_20251119", "name": "tool_search_tool_bm25"}, {"type": "mcp_toolset", "mcp_server_name": "docs"}, {"type": "computer_toolset_20260801"}, ], @@ -1038,6 +1054,7 @@ class TestAnthropicTypedToolsOnResponses: assert error.value.status_code == 400 assert tool["type"] in str(error.value) + assert "litellm_params" in str(error.value) def test_tool_with_no_responses_equivalent_drops_under_drop_params(self): kwargs: Final = _ADAPTER.translate_request( @@ -1053,14 +1070,14 @@ class TestAnthropicTypedToolsOnResponses: kwargs: Final = _ADAPTER.translate_request( _make_request( tools=[ - {"type": "code_execution_20250825", "name": "code_execution"}, + {"type": "bash_20250124", "name": "bash"}, {"type": "web_search_20250305", "name": "web_search"}, ], - tool_choice={"type": "tool", "name": "code_execution"}, + tool_choice={"type": "tool", "name": "web_search"}, ) ) - assert kwargs["tool_choice"] == {"type": "code_interpreter"} + assert kwargs["tool_choice"] == {"type": "web_search_preview"} def test_tool_choice_of_a_dropped_tool_is_dropped_with_it(self): kwargs: Final = _ADAPTER.translate_request( From 79f13175170afa1df917c83b71bf876d82b62804 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 10:07:37 -0700 Subject: [PATCH 03/11] fix(anthropic): run Anthropic tool search on OpenAI tool_search from /v1/messages Tool search reaches the existing OpenAI tool_search mapping instead of a 400, a custom tool keeps defer_loading only beside a search tool, and a tool_choice naming the search tool is omitted. A custom tool named web_search with its own input_schema stays a function, the computer tool documents its text field, and the untranslatable-tool error says how to set drop_params. --- .../responses_adapters/tool_translation.py | 29 ++++++-- .../responses_adapters/transformation.py | 10 +-- .../test_openai_responses_tool_search_wire.py | 9 ++- .../test_responses_adapters_handler.py | 70 +++++++++++++++++++ .../test_responses_adapters_transformation.py | 35 ++++++++-- 5 files changed, 137 insertions(+), 16 deletions(-) diff --git a/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py b/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py index 432954a25ed..1e7168ede03 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py +++ b/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py @@ -5,6 +5,7 @@ from typing import Final, TypeAlias from litellm.exceptions import UnsupportedParamsError from litellm.llms.anthropic.messages.utils import json_copy_or_original +from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_TOOL_TYPES _VERSIONED_TOOL_TYPE: Final = re.compile(r"(?P[a-z0-9_]+?)_(?P\d{8})") _UNVERSIONED_SERVER_TOOL_TYPES: Final = frozenset({"mcp_toolset", "computer_toolset"}) @@ -39,7 +40,7 @@ _COMPUTER_ACTIONS_20251124: Final = (*_COMPUTER_ACTIONS_20250124, "zoom") class ResponsesTool: name: str payload: Mapping[str, object] - choice: Mapping[str, object] + choice: Mapping[str, object] | None @dataclass(frozen=True, slots=True) @@ -175,6 +176,10 @@ def _computer_properties(version: str, zoom: bool) -> Mapping[str, object]: **pointer, "scroll_direction": _enum(("up", "down", "left", "right")), "scroll_amount": _integer("Number of scroll steps, for scroll."), + "text": _string( + "Text to type for type, keys to press for key and hold_key, such as ctrl+s, or a modifier key to hold " + "during scroll, such as shift." + ), "duration": {"type": "number", "description": "Seconds to wait for wait, or to hold for hold_key."}, "key": _string("Modifier key to hold during a click, such as shift."), } @@ -202,9 +207,14 @@ def _computer_tool(name: str, version: str, tool: Mapping[str, object]) -> Respo ) -def _custom_function(name: str, tool: Mapping[str, object]) -> ResponsesTool: +def _tool_search(name: str, tool: Mapping[str, object]) -> ResponsesTool: + return ResponsesTool(name=name, payload=dict(tool), choice=None) + + +def _custom_function(name: str, tool: Mapping[str, object], keeps_deferral: bool) -> ResponsesTool: description: Final = tool.get("description") input_schema: Final = tool.get("input_schema") + defer_loading: Final = tool.get("defer_loading") return ResponsesTool( name=name, payload={ @@ -213,19 +223,26 @@ def _custom_function(name: str, tool: Mapping[str, object]) -> ResponsesTool: "strict": bool(tool.get("strict")), **({"description": description} if "description" in tool else {}), **({"parameters": json_copy_or_original(input_schema)} if "input_schema" in tool else {}), + **({"defer_loading": defer_loading} if keeps_deferral and isinstance(defer_loading, bool) else {}), }, choice={"type": "function", "name": name}, ) -def responses_tool(tool: Mapping[str, object]) -> TranslatedTool: +def is_tool_search(tool: Mapping[str, object]) -> bool: + return tool.get("type") in ANTHROPIC_TOOL_SEARCH_TOOL_TYPES + + +def responses_tool(tool: Mapping[str, object], *, keeps_deferral: bool = False) -> TranslatedTool: tool_type: Final = tool.get("type") tool_name: Final = tool.get("name") name: Final = tool_name if isinstance(tool_name, str) else "" type_name: Final = tool_type if isinstance(tool_type, str) else "" family, version = _family_and_version(type_name) - if type_name.startswith("web_search") or name == "web_search": + if type_name.startswith("web_search") or (name == "web_search" and "input_schema" not in tool): return _hosted(name, {"type": "web_search_preview"}) + if is_tool_search(tool): + return _tool_search(name, tool) match family: case "bash": return _bash_tool(name) @@ -238,14 +255,14 @@ def responses_tool(tool: Mapping[str, object]) -> TranslatedTool: case _ if version or family in _UNVERSIONED_SERVER_TOOL_TYPES: return UntranslatableTool(name=name, type=type_name) case _: - return _custom_function(name, tool) + return _custom_function(name, tool, keeps_deferral) def untranslatable_tools_error(tools: Sequence[UntranslatableTool]) -> UnsupportedParamsError: types: Final = ", ".join(dict.fromkeys(tool.type for tool in tools)) return UnsupportedParamsError( message=( - f"The Responses API has no equivalent for these Anthropic tools: {types}. To drop them, set " + f"LiteLLM does not translate these Anthropic tools to the Responses API: {types}. To drop them, set " "`drop_params: true` in the model's `litellm_params` or under `litellm_settings` on the proxy, or " "`litellm.drop_params=True` in the SDK" ) diff --git a/litellm/llms/anthropic/messages/responses_adapters/transformation.py b/litellm/llms/anthropic/messages/responses_adapters/transformation.py index 080bffa2f92..f653304f6a2 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/messages/responses_adapters/transformation.py @@ -29,6 +29,7 @@ from litellm.llms.anthropic.messages.responses_adapters.tool_translation import ResponsesTool, TranslatedTool, UntranslatableTool, + is_tool_search, responses_tool, untranslatable_tools_error, ) @@ -476,7 +477,8 @@ class LiteLLMAnthropicToResponsesAPIAdapter: *, drop_params: bool = True, ) -> tuple[TranslatedTool, ...]: - translated: Final = tuple(responses_tool(tool) for tool in tools) + keeps_deferral: Final = any(is_tool_search(tool) for tool in tools) + translated: Final = tuple(responses_tool(tool, keeps_deferral=keeps_deferral) for tool in tools) untranslatable: Final = tuple(tool for tool in translated if isinstance(tool, UntranslatableTool)) if untranslatable and not drop_params: raise untranslatable_tools_error(untranslatable) @@ -500,7 +502,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: tool_choice: AnthropicMessagesToolChoice, tools: Sequence[TranslatedTool] = (), ) -> str | dict[str, object] | None: - """Convert Anthropic tool_choice to Responses API tool_choice, or None when it names a dropped tool.""" + """Map Anthropic tool_choice to Responses API tool_choice, or None when the named tool cannot be forced.""" tc_type: Final = tool_choice.get("type") if tc_type == "any": return "required" @@ -510,9 +512,9 @@ class LiteLLMAnthropicToResponsesAPIAdapter: return "auto" name: Final = tool_choice.get("name", "") match next((tool for tool in tools if tool.name == name), None): - case ResponsesTool(choice=choice): + case ResponsesTool(choice=choice) if choice is not None: return dict(choice) - case UntranslatableTool(): + case ResponsesTool() | UntranslatableTool(): return None case None: return {"type": "function", "name": name} diff --git a/tests/integration/providers/test_openai_responses_tool_search_wire.py b/tests/integration/providers/test_openai_responses_tool_search_wire.py index 39a955be13b..59febb3f713 100644 --- a/tests/integration/providers/test_openai_responses_tool_search_wire.py +++ b/tests/integration/providers/test_openai_responses_tool_search_wire.py @@ -629,7 +629,7 @@ def test_chat_completions_through_the_responses_bridge_follow_drop_params( assert [tuple(tool.get(key) for key in wire_keys) for tool in tools] == forwarded, tools -def test_messages_to_an_openai_model_with_the_search_tool_still_answers( +def test_messages_to_an_openai_model_runs_the_search_tool_on_openai_with_the_tool_still_deferred( openai_gateway: Gateway, openai_api: ControlPlane ) -> None: exchange: Final = _exchange( @@ -654,7 +654,12 @@ def test_messages_to_an_openai_model_with_the_search_tool_still_answers( ) assert exchange.response.status_code == 200, exchange.response.text - assert len(exchange.requests) == 1, exchange.requests + tools: Final = _ITEMS.validate_python(exchange.only_body()["tools"]) + wire_keys: Final = ("type", "execution", "name", "defer_loading") + assert [tuple(tool.get(key) for key in wire_keys) for tool in tools] == [ + ("tool_search", "server", None, None), + ("function", None, "get_weather", True), + ], tools def test_a_global_drop_params_drops_the_search_for_an_openai_model_that_cannot_run_it( diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py index 7de722e1171..d9c88b211ec 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py @@ -739,3 +739,73 @@ async def test_recomputed_cost_of_a_served_bridge_response_prices_the_reasoning_ assert logging_obj.cost_breakdown is not None assert logging_obj.cost_breakdown["output_cost"] == pytest.approx(expected_output) assert logging_obj.cost_breakdown["reasoning_cost"] == pytest.approx(expected_reasoning) + + +TOOL_SEARCH: Final = {"type": "tool_search_tool_bm25_20251119", "name": "tool_search_tool_bm25"} +DEFERRED_WEATHER: Final = { + "name": "get_weather", + "description": "Weather for a city", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + "defer_loading": True, +} +WEB_FETCH: Final = {"type": "web_fetch_20250910", "name": "web_fetch"} +TYPED_BASH: Final = {"type": "bash_20250124", "name": "bash"} + + +async def _openai_request_for( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, tools: list[Mapping[str, object]], **extra: object +) -> respx.Route: + monkeypatch.setenv("OPENAI_API_KEY", "sk-typed-tools-test") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond( + status_code=200, json=TOOL_CALL_RESPONSE + ) + await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=1024, + messages=MESSAGES, + model="openai/gpt-5.6-luna", + tools=[dict(tool) for tool in tools], + custom_llm_provider="openai", + **extra, + ) + return upstream + + +def _tools_sent(upstream: respx.Route) -> list[Mapping[str, object]]: + return json.loads(upstream.calls.last.request.content)["tools"] + + +@pytest.mark.asyncio +async def test_anthropic_tool_search_reaches_openai_as_its_server_tool_search_with_the_tool_still_deferred( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + upstream: Final = await _openai_request_for(respx_mock, monkeypatch, [TOOL_SEARCH, DEFERRED_WEATHER]) + + wire_keys: Final = ("type", "execution", "name", "defer_loading") + assert [tuple(tool.get(key) for key in wire_keys) for tool in _tools_sent(upstream)] == [ + ("tool_search", "server", None, None), + ("function", None, "get_weather", True), + ] + + +@pytest.mark.asyncio +async def test_a_tool_openai_cannot_run_answers_400_before_any_openai_call_without_drop_params( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + with pytest.raises(litellm.UnsupportedParamsError) as error: + await _openai_request_for(respx_mock, monkeypatch, [WEB_FETCH, TYPED_BASH], drop_params=False) + + assert error.value.status_code == 400 + assert not respx_mock.calls + + +@pytest.mark.asyncio +async def test_drop_params_drops_a_tool_openai_cannot_run_and_keeps_the_rest( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + upstream: Final = await _openai_request_for(respx_mock, monkeypatch, [WEB_FETCH, TYPED_BASH], drop_params=True) + + tools: Final = _tools_sent(upstream) + assert [tool["name"] for tool in tools] == ["bash"] + assert "command" in cast(Mapping[str, Mapping[str, object]], tools[0]["parameters"])["properties"] diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py index cfbc663b7bd..4a5d82e9ed6 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py @@ -949,6 +949,15 @@ class TestTranslateToolChoiceToResponsesAPI: assert result == {"type": "function", "name": "get_weather"} +_TOOL_SEARCH: Final = {"type": "tool_search_tool_bm25_20251119", "name": "tool_search_tool_bm25"} +_DEFERRED_WEATHER: Final = { + "name": "get_weather", + "description": "Weather for a city", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + "defer_loading": True, +} + + class TestAnthropicTypedToolsOnResponses: """Anthropic-defined tools carry no input_schema, so the adapter has to supply the schema Claude has built in.""" @@ -1029,10 +1038,11 @@ class TestAnthropicTypedToolsOnResponses: def test_every_web_search_type_runs_the_hosted_web_search(self, tool_type: str): assert self._only_tool({"type": tool_type, "name": "web_search"}) == {"type": "web_search_preview"} - def test_custom_tool_named_like_a_typed_tool_keeps_its_own_schema(self): + @pytest.mark.parametrize("name", ["bash", "web_search"]) + def test_custom_tool_named_like_a_typed_tool_keeps_its_own_schema(self, name: str): schema: Final = {"type": "object", "properties": {"cmd": {"type": "string"}}, "required": ["cmd"]} - tool: Final = self._only_tool({"name": "bash", "input_schema": schema}) + tool: Final = self._only_tool({"name": name, "input_schema": schema}) assert tool["parameters"] == schema assert "description" not in tool @@ -1042,8 +1052,6 @@ class TestAnthropicTypedToolsOnResponses: [ {"type": "web_fetch_20250910", "name": "web_fetch"}, {"type": "code_execution_20250825", "name": "code_execution"}, - {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"}, - {"type": "tool_search_tool_bm25_20251119", "name": "tool_search_tool_bm25"}, {"type": "mcp_toolset", "mcp_server_name": "docs"}, {"type": "computer_toolset_20260801"}, ], @@ -1100,6 +1108,25 @@ class TestAnthropicTypedToolsOnResponses: assert "tools" not in kwargs assert "tool_choice" not in kwargs + def test_tool_choice_naming_the_tool_search_leaves_the_choice_to_the_model(self): + kwargs: Final = _ADAPTER.translate_request( + _make_request( + tools=[_TOOL_SEARCH, _DEFERRED_WEATHER], + tool_choice={"type": "tool", "name": "tool_search_tool_bm25"}, + ), + drop_params=False, + ) + + assert [tool["name"] for tool in kwargs["tools"]] == ["tool_search_tool_bm25", "get_weather"] + assert "tool_choice" not in kwargs + + @pytest.mark.parametrize(("tools", "deferred"), [([_TOOL_SEARCH, _DEFERRED_WEATHER], True), ([_DEFERRED_WEATHER], None)]) + def test_a_custom_tool_stays_deferred_only_beside_a_tool_search(self, tools: List[Dict[str, Any]], deferred: Any): + kwargs: Final = _ADAPTER.translate_request(_make_request(tools=tools), drop_params=False) + + weather: Final = next(tool for tool in kwargs["tools"] if tool.get("name") == "get_weather") + assert weather.get("defer_loading") is deferred + def test_tool_choice_without_tools_still_reaches_the_provider(self): kwargs: Final = _ADAPTER.translate_request(_make_request(tool_choice={"type": "any"})) From 551c2f51d5dbe3bab683c0c6ddfdc45e3040be4e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 13:56:31 -0700 Subject: [PATCH 04/11] fix(anthropic): map tool search only for OpenAI's API and keep typed tool deferral Anthropic tool search now maps to OpenAI's tool_search only when the deployment targets api.openai.com. Another Responses server gets it as an unsupported tool, a 400 naming drop_params without it and a drop, deferral included, with it. A deferred typed tool (bash, text_editor, memory, computer) keeps defer_loading beside a search tool the target runs. Token counting drops tool search so a count never fails on it. A tool_choice forcing the search answers 400 without drop_params. Dropped tools log a warning naming them. A custom tool named web_search maps to hosted web search again, as on main, since changing it here would break existing callers. --- .../messages/responses_adapters/handler.py | 17 +++++- .../responses_adapters/tool_translation.py | 56 +++++++++++++++---- .../responses_adapters/transformation.py | 23 ++++++-- .../test_responses_adapters_handler.py | 53 ++++++++++++++++-- .../test_responses_adapters_transformation.py | 50 +++++++++++++---- ...test_openai_count_tokens_transformation.py | 20 +++++++ 6 files changed, 185 insertions(+), 34 deletions(-) diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index 99a07a1e9bf..83a58e73143 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -13,6 +13,7 @@ import litellm from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, supports_openai_prompt_cache_breakpoint, + targets_openai_api, ) from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.prompt_templates.common_utils import cache_control_as_prompt_cache_breakpoint @@ -29,6 +30,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager from ..adapters.handler import request_safeguards_evaluator @@ -56,15 +58,23 @@ def _forwarded_kwargs(extra_kwargs: Mapping[str, object] | None) -> Mapping[str, return extra_kwargs or {} +def _resolved_provider(model: str, custom_llm_provider: object) -> str: + return custom_llm_provider if isinstance(custom_llm_provider, str) else litellm.get_llm_provider(model=model)[1] + + def _provider_returns_encrypted_reasoning(model: str, custom_llm_provider: object) -> bool: - provider: Final = ( - custom_llm_provider if isinstance(custom_llm_provider, str) else litellm.get_llm_provider(model=model)[1] - ) + provider: Final = _resolved_provider(model, custom_llm_provider) provider_model: Final = local_model_name(model, provider) responses_config: Final = ProviderConfigManager.get_provider_responses_api_config(provider, provider_model) return responses_config is not None and "include" in responses_config.get_supported_openai_params(provider_model) +def _maps_tool_search(model: str, forwarded_kwargs: Mapping[str, object]) -> bool: + return _resolved_provider(model, forwarded_kwargs.get("custom_llm_provider")) == LlmProviders.OPENAI.value and ( + targets_openai_api(forwarded_kwargs.get("api_base") or forwarded_kwargs.get("base_url")) + ) + + def _reads_prompt_cache_breakpoint(model: str, forwarded_kwargs: Mapping[str, object]) -> bool: provider: Final = forwarded_kwargs.get("custom_llm_provider") return AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( @@ -193,6 +203,7 @@ def _build_responses_kwargs( ), auto_summary_level=auto_reasoning_summary(model, forwarded_kwargs.get("custom_llm_provider")), drop_params=drops_unsupported_params(forwarded_kwargs.get("drop_params")), + maps_tool_search=_maps_tool_search(model, forwarded_kwargs), ) # Normalize reasoning effort based on model capabilities diff --git a/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py b/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py index 1e7168ede03..496784e78e8 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py +++ b/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py @@ -1,8 +1,9 @@ import re from collections.abc import Mapping, Sequence -from dataclasses import dataclass +from dataclasses import dataclass, replace from typing import Final, TypeAlias +from litellm._logging import verbose_logger from litellm.exceptions import UnsupportedParamsError from litellm.llms.anthropic.messages.utils import json_copy_or_original from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_TOOL_TYPES @@ -211,10 +212,9 @@ def _tool_search(name: str, tool: Mapping[str, object]) -> ResponsesTool: return ResponsesTool(name=name, payload=dict(tool), choice=None) -def _custom_function(name: str, tool: Mapping[str, object], keeps_deferral: bool) -> ResponsesTool: +def _custom_function(name: str, tool: Mapping[str, object]) -> ResponsesTool: description: Final = tool.get("description") input_schema: Final = tool.get("input_schema") - defer_loading: Final = tool.get("defer_loading") return ResponsesTool( name=name, payload={ @@ -223,7 +223,6 @@ def _custom_function(name: str, tool: Mapping[str, object], keeps_deferral: bool "strict": bool(tool.get("strict")), **({"description": description} if "description" in tool else {}), **({"parameters": json_copy_or_original(input_schema)} if "input_schema" in tool else {}), - **({"defer_loading": defer_loading} if keeps_deferral and isinstance(defer_loading, bool) else {}), }, choice={"type": "function", "name": name}, ) @@ -233,15 +232,28 @@ def is_tool_search(tool: Mapping[str, object]) -> bool: return tool.get("type") in ANTHROPIC_TOOL_SEARCH_TOOL_TYPES -def responses_tool(tool: Mapping[str, object], *, keeps_deferral: bool = False) -> TranslatedTool: +def _with_deferral(translated: TranslatedTool, defer_loading: object) -> TranslatedTool: + match translated: + case ResponsesTool(payload=payload) if payload.get("type") == "function" and isinstance(defer_loading, bool): + return replace(translated, payload={**payload, "defer_loading": defer_loading}) + case _: + return translated + + +def responses_tool(tool: Mapping[str, object], *, runs_tool_search: bool = False) -> TranslatedTool: + translated: Final = _translated_tool(tool, runs_tool_search) + return _with_deferral(translated, tool.get("defer_loading")) if runs_tool_search else translated + + +def _translated_tool(tool: Mapping[str, object], runs_tool_search: bool) -> TranslatedTool: tool_type: Final = tool.get("type") tool_name: Final = tool.get("name") name: Final = tool_name if isinstance(tool_name, str) else "" type_name: Final = tool_type if isinstance(tool_type, str) else "" family, version = _family_and_version(type_name) - if type_name.startswith("web_search") or (name == "web_search" and "input_schema" not in tool): + if type_name.startswith("web_search") or name == "web_search": return _hosted(name, {"type": "web_search_preview"}) - if is_tool_search(tool): + if runs_tool_search and is_tool_search(tool): return _tool_search(name, tool) match family: case "bash": @@ -255,15 +267,35 @@ def responses_tool(tool: Mapping[str, object], *, keeps_deferral: bool = False) case _ if version or family in _UNVERSIONED_SERVER_TOOL_TYPES: return UntranslatableTool(name=name, type=type_name) case _: - return _custom_function(name, tool, keeps_deferral) + return _custom_function(name, tool) + + +_DROP_PARAMS_HOW_TO: Final = ( + "set `drop_params: true` in the model's `litellm_params` or under `litellm_settings` on the proxy, or " + "`litellm.drop_params=True` in the SDK" +) + + +def _tool_types(tools: Sequence[UntranslatableTool]) -> str: + return ", ".join(dict.fromkeys(tool.type for tool in tools)) def untranslatable_tools_error(tools: Sequence[UntranslatableTool]) -> UnsupportedParamsError: - types: Final = ", ".join(dict.fromkeys(tool.type for tool in tools)) return UnsupportedParamsError( message=( - f"LiteLLM does not translate these Anthropic tools to the Responses API: {types}. To drop them, set " - "`drop_params: true` in the model's `litellm_params` or under `litellm_settings` on the proxy, or " - "`litellm.drop_params=True` in the SDK" + f"LiteLLM does not translate these Anthropic tools to the Responses API: {_tool_types(tools)}. " + f"To drop them, {_DROP_PARAMS_HOW_TO}" ) ) + + +def log_dropped_tools(tools: Sequence[UntranslatableTool]) -> None: + verbose_logger.warning( + "Dropping Anthropic tools the Responses API cannot run under drop_params: %s", _tool_types(tools) + ) + + +def unforceable_tool_choice_error(name: str) -> UnsupportedParamsError: + return UnsupportedParamsError( + message=f"OpenAI cannot force the tool search tool {name} through `tool_choice`. To drop it, {_DROP_PARAMS_HOW_TO}" + ) diff --git a/litellm/llms/anthropic/messages/responses_adapters/transformation.py b/litellm/llms/anthropic/messages/responses_adapters/transformation.py index f653304f6a2..846bb3a415b 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/messages/responses_adapters/transformation.py @@ -30,7 +30,9 @@ from litellm.llms.anthropic.messages.responses_adapters.tool_translation import TranslatedTool, UntranslatableTool, is_tool_search, + log_dropped_tools, responses_tool, + unforceable_tool_choice_error, untranslatable_tools_error, ) from litellm.llms.anthropic.messages.utils import ( @@ -476,12 +478,15 @@ class LiteLLMAnthropicToResponsesAPIAdapter: tools: Sequence[AllAnthropicToolsValues], *, drop_params: bool = True, + maps_tool_search: bool = False, ) -> tuple[TranslatedTool, ...]: - keeps_deferral: Final = any(is_tool_search(tool) for tool in tools) - translated: Final = tuple(responses_tool(tool, keeps_deferral=keeps_deferral) for tool in tools) + runs_tool_search: Final = maps_tool_search and any(is_tool_search(tool) for tool in tools) + translated: Final = tuple(responses_tool(tool, runs_tool_search=runs_tool_search) for tool in tools) untranslatable: Final = tuple(tool for tool in translated if isinstance(tool, UntranslatableTool)) if untranslatable and not drop_params: raise untranslatable_tools_error(untranslatable) + if untranslatable: + log_dropped_tools(untranslatable) return translated def translate_tools_to_responses_api( @@ -489,11 +494,12 @@ class LiteLLMAnthropicToResponsesAPIAdapter: tools: list[AllAnthropicToolsValues], *, drop_params: bool = True, + maps_tool_search: bool = False, ) -> list[dict[str, object]]: """Convert Anthropic tool definitions to Responses API tools.""" return [ dict(tool.payload) - for tool in self.translated_tools(tools, drop_params=drop_params) + for tool in self.translated_tools(tools, drop_params=drop_params, maps_tool_search=maps_tool_search) if isinstance(tool, ResponsesTool) ] @@ -501,6 +507,8 @@ class LiteLLMAnthropicToResponsesAPIAdapter: def translate_tool_choice_to_responses_api( tool_choice: AnthropicMessagesToolChoice, tools: Sequence[TranslatedTool] = (), + *, + drop_params: bool = True, ) -> str | dict[str, object] | None: """Map Anthropic tool_choice to Responses API tool_choice, or None when the named tool cannot be forced.""" tc_type: Final = tool_choice.get("type") @@ -514,6 +522,8 @@ class LiteLLMAnthropicToResponsesAPIAdapter: match next((tool for tool in tools if tool.name == name), None): case ResponsesTool(choice=choice) if choice is not None: return dict(choice) + case ResponsesTool(name=forced) if not drop_params: + raise unforceable_tool_choice_error(forced) case ResponsesTool() | UntranslatableTool(): return None case None: @@ -608,6 +618,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: auto_summary_level: str = "detailed", *, drop_params: bool = True, + maps_tool_search: bool = False, ) -> dict[str, Any]: """ Translate a full Anthropic /v1/messages request dict to @@ -617,7 +628,8 @@ class LiteLLMAnthropicToResponsesAPIAdapter: on every call, so a reasoning model's items can be replayed intact next turn even when the client sent no ``thinking`` block; pass False for a provider whose Responses API rejects ``include``. ``auto_summary_level`` is the ``reasoning.summary`` the - opt-in auto summary injects when the client sent none. + opt-in auto summary injects when the client sent none. ``maps_tool_search`` passes Anthropic tool search + through for a target that runs OpenAI's tool search, and it is otherwise an unsupported tool. """ model: Final[str] = anthropic_request["model"] messages_list: Final = cast( @@ -670,6 +682,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: translated_tools: Final = self.translated_tools( cast(list[AllAnthropicToolsValues], anthropic_request.get("tools") or []), drop_params=drop_params, + maps_tool_search=maps_tool_search, ) responses_tools: Final = [dict(tool.payload) for tool in translated_tools if isinstance(tool, ResponsesTool)] if responses_tools: @@ -678,7 +691,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: tool_choice: Final = anthropic_request.get("tool_choice") responses_tool_choice: Final = ( self.translate_tool_choice_to_responses_api( - cast(AnthropicMessagesToolChoice, tool_choice), translated_tools + cast(AnthropicMessagesToolChoice, tool_choice), translated_tools, drop_params=drop_params ) if tool_choice and (responses_tools or not translated_tools) else None diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py index d9c88b211ec..2cd18929562 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py @@ -750,23 +750,28 @@ DEFERRED_WEATHER: Final = { } WEB_FETCH: Final = {"type": "web_fetch_20250910", "name": "web_fetch"} TYPED_BASH: Final = {"type": "bash_20250124", "name": "bash"} +OPENAI_API_BASE: Final = "https://api.openai.com/v1" +OTHER_RESPONSES_API_BASE: Final = "https://responses.example.test/v1" async def _openai_request_for( - respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, tools: list[Mapping[str, object]], **extra: object + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + tools: list[Mapping[str, object]], + api_base: str = OPENAI_API_BASE, + **extra: object, ) -> respx.Route: monkeypatch.setenv("OPENAI_API_KEY", "sk-typed-tools-test") monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") litellm.in_memory_llm_clients_cache.flush_cache() - upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond( - status_code=200, json=TOOL_CALL_RESPONSE - ) + upstream: Final = respx_mock.post(f"{api_base}/responses").respond(status_code=200, json=TOOL_CALL_RESPONSE) await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( max_tokens=1024, messages=MESSAGES, model="openai/gpt-5.6-luna", tools=[dict(tool) for tool in tools], custom_llm_provider="openai", + api_base=api_base, **extra, ) return upstream @@ -789,6 +794,46 @@ async def test_anthropic_tool_search_reaches_openai_as_its_server_tool_search_wi ] +@pytest.mark.asyncio +async def test_a_deferred_typed_tool_keeps_the_tool_search_under_drop_params( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + upstream: Final = await _openai_request_for( + respx_mock, monkeypatch, [TOOL_SEARCH, {**TYPED_BASH, "defer_loading": True}], drop_params=True + ) + + assert [(tool["type"], tool.get("defer_loading")) for tool in _tools_sent(upstream)] == [ + ("tool_search", None), + ("function", True), + ] + + +@pytest.mark.asyncio +async def test_tool_search_for_a_responses_server_other_than_openai_answers_400_without_drop_params( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + with pytest.raises(litellm.UnsupportedParamsError) as error: + await _openai_request_for( + respx_mock, monkeypatch, [TOOL_SEARCH, DEFERRED_WEATHER], api_base=OTHER_RESPONSES_API_BASE, drop_params=False + ) + + assert error.value.status_code == 400 + assert not respx_mock.calls + + +@pytest.mark.asyncio +async def test_drop_params_drops_tool_search_and_its_deferral_for_a_responses_server_other_than_openai( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + upstream: Final = await _openai_request_for( + respx_mock, monkeypatch, [TOOL_SEARCH, DEFERRED_WEATHER], api_base=OTHER_RESPONSES_API_BASE, drop_params=True + ) + + assert [(tool["type"], tool["name"], tool.get("defer_loading")) for tool in _tools_sent(upstream)] == [ + ("function", "get_weather", None) + ] + + @pytest.mark.asyncio async def test_a_tool_openai_cannot_run_answers_400_before_any_openai_call_without_drop_params( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py index 4a5d82e9ed6..03ff2537fa6 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py @@ -956,6 +956,7 @@ _DEFERRED_WEATHER: Final = { "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, "defer_loading": True, } +_DEFERRED_BASH: Final = {"type": "bash_20250124", "name": "bash", "defer_loading": True} class TestAnthropicTypedToolsOnResponses: @@ -1038,11 +1039,10 @@ class TestAnthropicTypedToolsOnResponses: def test_every_web_search_type_runs_the_hosted_web_search(self, tool_type: str): assert self._only_tool({"type": tool_type, "name": "web_search"}) == {"type": "web_search_preview"} - @pytest.mark.parametrize("name", ["bash", "web_search"]) - def test_custom_tool_named_like_a_typed_tool_keeps_its_own_schema(self, name: str): + def test_custom_tool_named_like_a_typed_tool_keeps_its_own_schema(self): schema: Final = {"type": "object", "properties": {"cmd": {"type": "string"}}, "required": ["cmd"]} - tool: Final = self._only_tool({"name": name, "input_schema": schema}) + tool: Final = self._only_tool({"name": "bash", "input_schema": schema}) assert tool["parameters"] == schema assert "description" not in tool @@ -1054,6 +1054,7 @@ class TestAnthropicTypedToolsOnResponses: {"type": "code_execution_20250825", "name": "code_execution"}, {"type": "mcp_toolset", "mcp_server_name": "docs"}, {"type": "computer_toolset_20260801"}, + _TOOL_SEARCH, ], ) def test_tool_with_no_responses_equivalent_is_a_400_without_drop_params(self, tool: Dict[str, Any]): @@ -1108,24 +1109,53 @@ class TestAnthropicTypedToolsOnResponses: assert "tools" not in kwargs assert "tool_choice" not in kwargs - def test_tool_choice_naming_the_tool_search_leaves_the_choice_to_the_model(self): + def test_tool_choice_forcing_the_tool_search_is_a_400_without_drop_params(self): + with pytest.raises(UnsupportedParamsError) as error: + _ADAPTER.translate_request( + _make_request( + tools=[_TOOL_SEARCH, _DEFERRED_WEATHER], + tool_choice={"type": "tool", "name": "tool_search_tool_bm25"}, + ), + drop_params=False, + maps_tool_search=True, + ) + + assert error.value.status_code == 400 + assert "tool_search_tool_bm25" in str(error.value) + assert "drop_params" in str(error.value) + + def test_tool_choice_forcing_the_tool_search_is_dropped_under_drop_params(self): kwargs: Final = _ADAPTER.translate_request( _make_request( tools=[_TOOL_SEARCH, _DEFERRED_WEATHER], tool_choice={"type": "tool", "name": "tool_search_tool_bm25"}, ), - drop_params=False, + drop_params=True, + maps_tool_search=True, ) assert [tool["name"] for tool in kwargs["tools"]] == ["tool_search_tool_bm25", "get_weather"] assert "tool_choice" not in kwargs - @pytest.mark.parametrize(("tools", "deferred"), [([_TOOL_SEARCH, _DEFERRED_WEATHER], True), ([_DEFERRED_WEATHER], None)]) - def test_a_custom_tool_stays_deferred_only_beside_a_tool_search(self, tools: List[Dict[str, Any]], deferred: Any): - kwargs: Final = _ADAPTER.translate_request(_make_request(tools=tools), drop_params=False) + @pytest.mark.parametrize( + ("tools", "maps_tool_search", "deferred"), + [ + ([_TOOL_SEARCH, _DEFERRED_WEATHER], True, True), + ([_TOOL_SEARCH, _DEFERRED_BASH], True, True), + ([_DEFERRED_WEATHER], True, None), + ([_TOOL_SEARCH, _DEFERRED_WEATHER], False, None), + ], + ) + def test_a_tool_stays_deferred_only_beside_a_tool_search_the_target_runs( + self, tools: List[Dict[str, Any]], maps_tool_search: bool, deferred: Any + ): + kwargs: Final = _ADAPTER.translate_request( + _make_request(tools=tools), drop_params=True, maps_tool_search=maps_tool_search + ) - weather: Final = next(tool for tool in kwargs["tools"] if tool.get("name") == "get_weather") - assert weather.get("defer_loading") is deferred + deferrable: Final = next(tool for tool in kwargs["tools"] if tool.get("type") == "function") + assert deferrable.get("defer_loading") is deferred + assert any(tool["type"] == _TOOL_SEARCH["type"] for tool in kwargs["tools"]) is (deferred is True) def test_tool_choice_without_tools_still_reaches_the_provider(self): kwargs: Final = _ADAPTER.translate_request(_make_request(tool_choice={"type": "any"})) diff --git a/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py b/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py index 575183772f9..9f540366d05 100644 --- a/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py @@ -603,6 +603,26 @@ def test_anthropic_messages_keep_every_block_the_chat_converter_drops() -> None: ) +def test_a_count_with_anthropic_tool_search_counts_the_tools_without_the_search() -> None: + _, _, tools, _ = OpenAICountTokensConfig.anthropic_messages_to_responses_input( + model="gpt-5.6-luna", + messages=[{"role": "user", "content": "Find a weather tool."}], + system=None, + tools=[ + {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"}, + { + "name": "get_weather", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}}, + "defer_loading": True, + }, + ], + ) + + assert [(tool.get("type"), tool.get("name"), tool.get("defer_loading")) for tool in tools or ()] == [ + ("function", "get_weather", None) + ] + + def test_anthropic_list_system_becomes_joined_instruction_text() -> None: _, instructions, _, _ = OpenAICountTokensConfig.anthropic_messages_to_responses_input( model="gpt-4o", From ece1b1add343f2e7cd507c9f8030387a78db03be Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 16:09:15 -0700 Subject: [PATCH 05/11] fix(anthropic): run tool search only on OpenAI models that support it and count it as inference sends it The /v1/messages bridge now maps Anthropic tool search to OpenAI's tool_search only when the target is OpenAI's API and the cost map says the model runs tool search. Other models get LiteLLM's 400 naming drop_params, or the search and its deferral dropped under drop_params, instead of OpenAI's raw 400. count_tokens maps tool search the same way, so the count sees the tool_search tool and the deferred tools inference sends. --- .../messages/responses_adapters/handler.py | 7 +-- .../responses_adapters/tool_translation.py | 9 ++++ .../responses/count_tokens/token_counter.py | 10 +++- .../responses/count_tokens/transformation.py | 10 +++- .../tool_search.py | 13 ++--- .../test_responses_adapters_handler.py | 34 +++++++++++- .../count_tokens/test_token_counter.py | 54 +++++++++++++++++++ 7 files changed, 124 insertions(+), 13 deletions(-) diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index 83a58e73143..5be250f65c9 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -13,7 +13,6 @@ import litellm from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, supports_openai_prompt_cache_breakpoint, - targets_openai_api, ) from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.prompt_templates.common_utils import cache_control_as_prompt_cache_breakpoint @@ -38,6 +37,7 @@ from ..convertible_blocks import reject_malformed_tool_result_text from ..safeguards import with_safeguard_results from ..utils import litellm_logging_obj_from_kwargs, local_model_name from .streaming_iterator import AnthropicResponsesStreamWrapper +from .tool_translation import openai_runs_tool_search from .transformation import LiteLLMAnthropicToResponsesAPIAdapter AnthropicRequestMessages: TypeAlias = list[AllAnthropicMessageValues] | list[dict[str, object]] @@ -70,8 +70,9 @@ def _provider_returns_encrypted_reasoning(model: str, custom_llm_provider: objec def _maps_tool_search(model: str, forwarded_kwargs: Mapping[str, object]) -> bool: - return _resolved_provider(model, forwarded_kwargs.get("custom_llm_provider")) == LlmProviders.OPENAI.value and ( - targets_openai_api(forwarded_kwargs.get("api_base") or forwarded_kwargs.get("base_url")) + provider: Final = _resolved_provider(model, forwarded_kwargs.get("custom_llm_provider")) + return provider == LlmProviders.OPENAI.value and openai_runs_tool_search( + local_model_name(model, provider), forwarded_kwargs.get("api_base") or forwarded_kwargs.get("base_url") ) diff --git a/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py b/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py index 496784e78e8..ecd95b60794 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py +++ b/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py @@ -3,10 +3,13 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass, replace from typing import Final, TypeAlias +import litellm from litellm._logging import verbose_logger from litellm.exceptions import UnsupportedParamsError +from litellm.integrations.anthropic_cache_control_hook import targets_openai_api from litellm.llms.anthropic.messages.utils import json_copy_or_original from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_TOOL_TYPES +from litellm.types.utils import LlmProviders _VERSIONED_TOOL_TYPE: Final = re.compile(r"(?P[a-z0-9_]+?)_(?P\d{8})") _UNVERSIONED_SERVER_TOOL_TYPES: Final = frozenset({"mcp_toolset", "computer_toolset"}) @@ -299,3 +302,9 @@ def unforceable_tool_choice_error(name: str) -> UnsupportedParamsError: return UnsupportedParamsError( message=f"OpenAI cannot force the tool search tool {name} through `tool_choice`. To drop it, {_DROP_PARAMS_HOW_TO}" ) + + +def openai_runs_tool_search(model: str, api_base: object) -> bool: + return targets_openai_api(api_base) and litellm.supports_tool_search( + model=model, custom_llm_provider=LlmProviders.OPENAI.value + ) diff --git a/litellm/llms/openai/responses/count_tokens/token_counter.py b/litellm/llms/openai/responses/count_tokens/token_counter.py index 6ae61e15953..f61eeab1aa6 100644 --- a/litellm/llms/openai/responses/count_tokens/token_counter.py +++ b/litellm/llms/openai/responses/count_tokens/token_counter.py @@ -16,6 +16,7 @@ from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.tool_fields import DropPolicy from litellm.llms.anthropic.messages.convertible_blocks import deployment_drops_params, unconvertible_blocks_error +from litellm.llms.anthropic.messages.responses_adapters.tool_translation import openai_runs_tool_search from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) @@ -118,11 +119,17 @@ class OpenAITokenCounter(BaseTokenCounter): system: object, messages_format: TokenCountMessagesFormat, output_format: JsonValue, + api_base: str | None, ) -> ResponsesCountRequestParts: match messages_format: case "anthropic": return OpenAICountTokensConfig.anthropic_messages_to_responses_input( - model=model_to_use, messages=messages, system=system, tools=tools, output_format=output_format + model=model_to_use, + messages=messages, + system=system, + tools=tools, + output_format=output_format, + maps_tool_search=openai_runs_tool_search(model_to_use, api_base), ) case "openai": input_items, instructions = OpenAICountTokensConfig.messages_to_responses_input( @@ -186,6 +193,7 @@ class OpenAITokenCounter(BaseTokenCounter): system=system, messages_format=messages_format, output_format=output_format, + api_base=resolved.api_base, ) # If no input items were produced (e.g., system-only messages), fall back to local counting diff --git a/litellm/llms/openai/responses/count_tokens/transformation.py b/litellm/llms/openai/responses/count_tokens/transformation.py index a45a0448326..f723245a096 100644 --- a/litellm/llms/openai/responses/count_tokens/transformation.py +++ b/litellm/llms/openai/responses/count_tokens/transformation.py @@ -20,6 +20,7 @@ from litellm.llms.anthropic.messages.native.utils import normalize_native_messag from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) +from litellm.responses.litellm_completion_transformation.tool_search import tools_with_anthropic_tool_search_mapped from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.anthropic import AnthropicMessagesRequest, AnthropicOutputSchema from litellm.types.llms.openai import ResponsesAPIRequestParams, replayable_responses_input @@ -343,6 +344,7 @@ class OpenAICountTokensConfig: system: object, tools: Sequence[Mapping[str, object]] | None, output_format: JsonValue = None, + maps_tool_search: bool = False, ) -> ResponsesCountRequestParts: """Convert with the adapter /v1/messages inference uses, so the count sees the same input it bills.""" anthropic_request: Final = AnthropicMessagesRequest( @@ -354,15 +356,19 @@ class OpenAICountTokensConfig: ) translated: Final = _TRANSLATED_REQUEST.validate_python( LiteLLMAnthropicToResponsesAPIAdapter().translate_request( - anthropic_request, include_encrypted_reasoning=False + anthropic_request, include_encrypted_reasoning=False, maps_tool_search=maps_tool_search ) ) - return OpenAICountTokensConfig.count_request_parts( + input_items, instructions, translated_tools, text = OpenAICountTokensConfig.count_request_parts( ResponsesAPIRequestUtils.restore_encrypted_content_item_ids_in_input(translated.get("input")), translated.get("instructions"), translated.get("tools"), translated.get("text"), ) + counted_tools: Final = ( + tools_with_anthropic_tool_search_mapped(translated_tools) if maps_tool_search else translated_tools + ) + return input_items, instructions, None if counted_tools is None else tuple(counted_tools), text @staticmethod def count_request_parts( diff --git a/litellm/responses/litellm_completion_transformation/tool_search.py b/litellm/responses/litellm_completion_transformation/tool_search.py index 780a23e02b2..448d0edcf78 100644 --- a/litellm/responses/litellm_completion_transformation/tool_search.py +++ b/litellm/responses/litellm_completion_transformation/tool_search.py @@ -1,12 +1,12 @@ from collections.abc import Iterable, Iterator, Mapping, Sequence, Set -from typing import TYPE_CHECKING, Final, Literal, TypeAlias +from typing import TYPE_CHECKING, Final, Literal, TypeAlias, TypeVar from pydantic import ConfigDict, TypeAdapter, ValidationError from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import is_anthropic_tool_search_use from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_TOOL_TYPES, AnthropicToolSearchToolRegex from litellm.types.llms.base import LiteLLMBaseModel -from litellm.types.llms.openai import ALL_RESPONSES_API_TOOL_PARAMS, OpenAIToolSearchToolParam, ResponseInputParam +from litellm.types.llms.openai import OpenAIToolSearchToolParam, ResponseInputParam from litellm.types.responses.main import OutputToolSearchCall from litellm.types.utils import LlmProviders, Message @@ -48,6 +48,7 @@ ANTHROPIC_TOOL_SEARCH_TOOL_TYPE: Final = "tool_search_tool_regex_20251119" ANTHROPIC_TOOL_SEARCH_TOOL_NAME: Final = "tool_search_tool_regex" ToolSearchCallStatus: TypeAlias = Literal["in_progress", "completed", "incomplete"] +_Tool: Final = TypeVar("_Tool", bound=Mapping[str, object]) _ARGUMENTS_ADAPTER: Final = TypeAdapter(dict[str, object]) _MESSAGE_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) @@ -100,8 +101,8 @@ def is_openai_tool_search_tool(tool: object) -> bool: def tools_with_anthropic_tool_search_mapped( - tools: Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None, -) -> Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None: + tools: Sequence[_Tool] | None, +) -> Sequence[_Tool | OpenAIToolSearchToolParam] | None: if tools is None or not any(is_anthropic_tool_search_chat_tool(tool) for tool in tools): return tools if any(is_openai_tool_search_tool(tool) for tool in tools): @@ -118,8 +119,8 @@ def tools_with_anthropic_tool_search_mapped( def tools_without_anthropic_tool_search( - tools: Sequence[ALL_RESPONSES_API_TOOL_PARAMS], -) -> tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]: + tools: Sequence[_Tool], +) -> tuple[_Tool, ...]: return tuple(tool for tool in tools if not is_anthropic_tool_search_chat_tool(tool)) diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py index 2cd18929562..665368ee45f 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py @@ -759,6 +759,7 @@ async def _openai_request_for( monkeypatch: pytest.MonkeyPatch, tools: list[Mapping[str, object]], api_base: str = OPENAI_API_BASE, + model: str = "openai/gpt-5.6-luna", **extra: object, ) -> respx.Route: monkeypatch.setenv("OPENAI_API_KEY", "sk-typed-tools-test") @@ -768,7 +769,7 @@ async def _openai_request_for( await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( max_tokens=1024, messages=MESSAGES, - model="openai/gpt-5.6-luna", + model=model, tools=[dict(tool) for tool in tools], custom_llm_provider="openai", api_base=api_base, @@ -834,6 +835,37 @@ async def test_drop_params_drops_tool_search_and_its_deferral_for_a_responses_se ] +MODEL_WITHOUT_TOOL_SEARCH: Final = "openai/a-model-the-cost-map-does-not-mark-for-tool-search" + + +@pytest.mark.asyncio +async def test_tool_search_on_an_openai_model_that_cannot_run_it_answers_400_naming_drop_params( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + with pytest.raises(litellm.UnsupportedParamsError) as error: + await _openai_request_for( + respx_mock, monkeypatch, [TOOL_SEARCH, DEFERRED_WEATHER], model=MODEL_WITHOUT_TOOL_SEARCH, drop_params=False + ) + + assert error.value.status_code == 400 + assert "tool_search_tool_bm25_20251119" in error.value.message + assert "drop_params" in error.value.message + assert not respx_mock.calls + + +@pytest.mark.asyncio +async def test_drop_params_drops_tool_search_and_its_deferral_on_an_openai_model_that_cannot_run_it( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + upstream: Final = await _openai_request_for( + respx_mock, monkeypatch, [TOOL_SEARCH, DEFERRED_WEATHER], model=MODEL_WITHOUT_TOOL_SEARCH, drop_params=True + ) + + assert [(tool["type"], tool["name"], tool.get("defer_loading")) for tool in _tools_sent(upstream)] == [ + ("function", "get_weather", None) + ] + + @pytest.mark.asyncio async def test_a_tool_openai_cannot_run_answers_400_before_any_openai_call_without_drop_params( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch diff --git a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py index 17c1b35e211..89936c21acc 100644 --- a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py +++ b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py @@ -194,6 +194,60 @@ async def _sent_count_body( return json.loads(route.calls.last.request.content) +TOOL_SEARCH_TOOLS: Final[list[dict[str, object]]] = [ + {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"}, + { + "name": "get_weather", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}}, + "defer_loading": True, + }, +] + + +async def _counted_tools( + monkeypatch: pytest.MonkeyPatch, model: str, litellm_params: Mapping[str, object] +) -> list[tuple[object, object, object]]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock: + route: Final = respx.post(f"{litellm_params.get('api_base', 'https://api.openai.com/v1')}/responses/input_tokens") + route.mock(return_value=httpx.Response(200, json={"input_tokens": 1})) + await OpenAITokenCounter().count_tokens( + model_to_use=model, + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + contents=None, + deployment={"litellm_params": {"api_key": "sk-test", **litellm_params}}, + request_model=f"openai/{model}", + tools=TOOL_SEARCH_TOOLS, + messages_format="anthropic", + ) + tools: Final = json.loads(route.calls.last.request.content)["tools"] + return [(tool.get("type"), tool.get("name"), tool.get("defer_loading")) for tool in tools] + + +@pytest.mark.asyncio +async def test_anthropic_tool_search_is_counted_as_the_openai_tool_search_inference_sends( + monkeypatch: pytest.MonkeyPatch, +) -> None: + assert await _counted_tools(monkeypatch, "gpt-5.6-luna", {}) == [ + ("tool_search", None, None), + ("function", "get_weather", True), + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model", "litellm_params"), + [ + pytest.param("a-model-the-cost-map-does-not-mark-for-tool-search", {}, id="model-without-tool-search"), + pytest.param("gpt-5.6-luna", {"api_base": "https://responses.example.test/v1"}, id="other-responses-server"), + ], +) +async def test_anthropic_tool_search_is_left_out_of_the_count_where_inference_cannot_run_it( + monkeypatch: pytest.MonkeyPatch, model: str, litellm_params: Mapping[str, object] +) -> None: + assert await _counted_tools(monkeypatch, model, litellm_params) == [("function", "get_weather", None)] + + @pytest.mark.asyncio async def test_anthropic_format_sends_the_document_image_and_tool_turns_to_the_count_endpoint( monkeypatch: pytest.MonkeyPatch, From b2b965f8d058737c0ed1f7c14c7cf259da725f97 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 20:54:49 -0700 Subject: [PATCH 06/11] fix(count_tokens): drop tool search from the count when inference drops it under drop_params --- .../responses/count_tokens/token_counter.py | 3 ++ .../responses/count_tokens/transformation.py | 19 +++++++++-- .../llms/openai/responses/transformation.py | 10 ++++-- .../count_tokens/test_token_counter.py | 33 ++++++++++++++++--- 4 files changed, 56 insertions(+), 9 deletions(-) diff --git a/litellm/llms/openai/responses/count_tokens/token_counter.py b/litellm/llms/openai/responses/count_tokens/token_counter.py index 75fe22ee928..02e6c98ca07 100644 --- a/litellm/llms/openai/responses/count_tokens/token_counter.py +++ b/litellm/llms/openai/responses/count_tokens/token_counter.py @@ -148,6 +148,7 @@ class OpenAITokenCounter(BaseTokenCounter): messages_format: TokenCountMessagesFormat, output_format: JsonValue, api_base: str | None, + drop_params: bool, ) -> ResponsesCountRequestParts: match messages_format: case "anthropic": @@ -158,6 +159,7 @@ class OpenAITokenCounter(BaseTokenCounter): tools=tools, output_format=output_format, maps_tool_search=openai_runs_tool_search(model_to_use, api_base), + drop_params=drop_params, ) case "openai": input_items, instructions = OpenAICountTokensConfig.messages_to_responses_input( @@ -237,6 +239,7 @@ class OpenAITokenCounter(BaseTokenCounter): messages_format=messages_format, output_format=output_format, api_base=resolved.api_base, + drop_params=deployment_drops_params(deployment), ) # If no input items were produced (e.g., system-only messages), fall back to local counting diff --git a/litellm/llms/openai/responses/count_tokens/transformation.py b/litellm/llms/openai/responses/count_tokens/transformation.py index f723245a096..8aad129da40 100644 --- a/litellm/llms/openai/responses/count_tokens/transformation.py +++ b/litellm/llms/openai/responses/count_tokens/transformation.py @@ -20,7 +20,11 @@ from litellm.llms.anthropic.messages.native.utils import normalize_native_messag from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) -from litellm.responses.litellm_completion_transformation.tool_search import tools_with_anthropic_tool_search_mapped +from litellm.llms.openai.responses.transformation import keeps_anthropic_tool_search +from litellm.responses.litellm_completion_transformation.tool_search import ( + tools_with_anthropic_tool_search_mapped, + tools_without_anthropic_tool_search, +) from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.anthropic import AnthropicMessagesRequest, AnthropicOutputSchema from litellm.types.llms.openai import ResponsesAPIRequestParams, replayable_responses_input @@ -161,6 +165,14 @@ def chat_content_blocks_to_responses_content( return "\n".join(part["text"] for part in parts if part["type"] == "input_text") +def _tools_inference_sends( + model: str, tools: Sequence[Mapping[str, object]], *, drop_params: bool +) -> Sequence[Mapping[str, object]] | None: + if keeps_anthropic_tool_search(model, tools, drop_params=drop_params): + return tools_with_anthropic_tool_search_mapped(tools) + return tools_without_anthropic_tool_search(tools) + + class OpenAICountTokensConfig: """ Configuration and transformation logic for OpenAI Responses API token counting. @@ -345,6 +357,7 @@ class OpenAICountTokensConfig: tools: Sequence[Mapping[str, object]] | None, output_format: JsonValue = None, maps_tool_search: bool = False, + drop_params: bool = False, ) -> ResponsesCountRequestParts: """Convert with the adapter /v1/messages inference uses, so the count sees the same input it bills.""" anthropic_request: Final = AnthropicMessagesRequest( @@ -366,7 +379,9 @@ class OpenAICountTokensConfig: translated.get("text"), ) counted_tools: Final = ( - tools_with_anthropic_tool_search_mapped(translated_tools) if maps_tool_search else translated_tools + _tools_inference_sends(model, translated_tools, drop_params=drop_params) + if maps_tool_search and translated_tools is not None + else translated_tools ) return input_items, instructions, None if counted_tools is None else tuple(counted_tools), text diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 9b8515a9ecd..3e2b58f7fbf 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -158,12 +158,17 @@ def _tool_deferral(tool: object) -> _ToolDeferral: return _ToolDeferral() -def _runs_tool_search(model: str, tools: Sequence[ALL_RESPONSES_API_TOOL_PARAMS]) -> bool: +def _runs_tool_search(model: str, tools: Sequence[object]) -> bool: return any(_tool_deferral(tool).defers for tool in tools) and litellm.supports_tool_search( model=model, custom_llm_provider=LlmProviders.OPENAI.value ) +def keeps_anthropic_tool_search(model: str, tools: Sequence[object], *, drop_params: bool) -> bool: + callers_own_search: Final = any(is_openai_tool_search_tool(tool) for tool in tools) + return callers_own_search or not drop_params or _runs_tool_search(model, tools) + + def _without_defer_loading(tool: ALL_RESPONSES_API_TOOL_PARAMS) -> ALL_RESPONSES_API_TOOL_PARAMS: deferral: Final = _tool_deferral(tool) if not deferral.defers: @@ -728,8 +733,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): ) -> Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None: if tools is None or not any(is_anthropic_tool_search_chat_tool(tool) for tool in tools): return tools - callers_own_search: Final = any(is_openai_tool_search_tool(tool) for tool in tools) - if callers_own_search or not drops_params(litellm_params) or _runs_tool_search(model, tools): + if keeps_anthropic_tool_search(model, tools, drop_params=drops_params(litellm_params)): return tools_with_anthropic_tool_search_mapped(tools) verbose_logger.warning( "Dropping tool search under drop_params: OpenAI model '%s' cannot run it or no tool is deferred, so every " diff --git a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py index 3ac356a5b07..f6ff8aab381 100644 --- a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py +++ b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py @@ -204,10 +204,17 @@ TOOL_SEARCH_TOOLS: Final[list[dict[str, object]]] = [ "defer_loading": True, }, ] +UNDEFERRED_TOOL_SEARCH_TOOLS: Final[list[dict[str, object]]] = [ + TOOL_SEARCH_TOOLS[0], + {key: value for key, value in TOOL_SEARCH_TOOLS[1].items() if key != "defer_loading"}, +] async def _counted_tools( - monkeypatch: pytest.MonkeyPatch, model: str, litellm_params: Mapping[str, object] + monkeypatch: pytest.MonkeyPatch, + model: str, + litellm_params: Mapping[str, object], + tools: list[dict[str, object]] = TOOL_SEARCH_TOOLS, ) -> list[tuple[object, object, object]]: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) with respx.mock: @@ -219,11 +226,11 @@ async def _counted_tools( contents=None, deployment={"litellm_params": {"api_key": "sk-test", **litellm_params}}, request_model=f"openai/{model}", - tools=TOOL_SEARCH_TOOLS, + tools=tools, messages_format="anthropic", ) - tools: Final = json.loads(route.calls.last.request.content)["tools"] - return [(tool.get("type"), tool.get("name"), tool.get("defer_loading")) for tool in tools] + counted: Final = json.loads(route.calls.last.request.content)["tools"] + return [(tool.get("type"), tool.get("name"), tool.get("defer_loading")) for tool in counted] @pytest.mark.asyncio @@ -236,6 +243,24 @@ async def test_anthropic_tool_search_is_counted_as_the_openai_tool_search_infere ] +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("tools", "expected"), + [ + pytest.param( + TOOL_SEARCH_TOOLS, + [("tool_search", None, None), ("function", "get_weather", True)], + id="a-tool-is-deferred", + ), + pytest.param(UNDEFERRED_TOOL_SEARCH_TOOLS, [("function", "get_weather", None)], id="nothing-is-deferred"), + ], +) +async def test_drop_params_counts_anthropic_tool_search_only_where_inference_keeps_it( + monkeypatch: pytest.MonkeyPatch, tools: list[dict[str, object]], expected: list[tuple[object, object, object]] +) -> None: + assert await _counted_tools(monkeypatch, "gpt-5.6-luna", {"drop_params": True}, tools) == expected + + @pytest.mark.asyncio @pytest.mark.parametrize( ("model", "litellm_params"), From 6ae64adfd57cab0da73ee6b9377e4078aa87afed Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 03:53:49 -0700 Subject: [PATCH 07/11] fix(anthropic): map tool search only beside a deferred tool, guard non-string tool types, and honor additional_drop_params tools --- .../messages/responses_adapters/handler.py | 3 +- .../responses_adapters/tool_translation.py | 16 +++++- .../responses_adapters/transformation.py | 4 +- .../responses/count_tokens/token_counter.py | 3 -- .../responses/count_tokens/transformation.py | 19 +------ .../llms/openai/responses/transformation.py | 10 ++-- .../test_responses_adapters_handler.py | 50 +++++++++++++++++++ .../count_tokens/test_token_counter.py | 13 +++-- 8 files changed, 84 insertions(+), 34 deletions(-) diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index 1df439bab4e..ef023e99ec6 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -24,6 +24,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( prompt_cache_breakpoint_for_wire, with_prompt_cache_breakpoint, ) +from litellm.litellm_core_utils.tool_fields import drop_policy from litellm.responses.utils import auto_reasoning_summary from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS from litellm.types.llms.anthropic import ( @@ -316,7 +317,7 @@ def _build_responses_kwargs( request_data["thinking"] = thinking if tool_choice: request_data["tool_choice"] = tool_choice - if tools: + if tools and "tools" not in drop_policy(forwarded_kwargs).additional_drop_params: request_data["tools"] = tools if top_p is not None: request_data["top_p"] = top_p diff --git a/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py b/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py index ecd95b60794..13a41f93d47 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py +++ b/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py @@ -232,7 +232,21 @@ def _custom_function(name: str, tool: Mapping[str, object]) -> ResponsesTool: def is_tool_search(tool: Mapping[str, object]) -> bool: - return tool.get("type") in ANTHROPIC_TOOL_SEARCH_TOOL_TYPES + tool_type: Final = tool.get("type") + return isinstance(tool_type, str) and tool_type in ANTHROPIC_TOOL_SEARCH_TOOL_TYPES + + +def _defers_a_function(tool: Mapping[str, object]) -> bool: + translated: Final = _translated_tool(tool, False) if tool.get("defer_loading") is True else None + return isinstance(translated, ResponsesTool) and translated.payload.get("type") == "function" + + +def keeps_tool_search(tools: Sequence[Mapping[str, object]], *, maps_tool_search: bool) -> bool: + return ( + maps_tool_search + and any(is_tool_search(tool) for tool in tools) + and any(_defers_a_function(tool) for tool in tools) + ) def _with_deferral(translated: TranslatedTool, defer_loading: object) -> TranslatedTool: diff --git a/litellm/llms/anthropic/messages/responses_adapters/transformation.py b/litellm/llms/anthropic/messages/responses_adapters/transformation.py index 34c3d530fb5..c0765f18670 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/messages/responses_adapters/transformation.py @@ -30,7 +30,7 @@ from litellm.llms.anthropic.messages.responses_adapters.tool_translation import ResponsesTool, TranslatedTool, UntranslatableTool, - is_tool_search, + keeps_tool_search, log_dropped_tools, responses_tool, unforceable_tool_choice_error, @@ -472,7 +472,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: drop_params: bool = True, maps_tool_search: bool = False, ) -> tuple[TranslatedTool, ...]: - runs_tool_search: Final = maps_tool_search and any(is_tool_search(tool) for tool in tools) + runs_tool_search: Final = keeps_tool_search(tools, maps_tool_search=maps_tool_search) translated: Final = tuple(responses_tool(tool, runs_tool_search=runs_tool_search) for tool in tools) untranslatable: Final = tuple(tool for tool in translated if isinstance(tool, UntranslatableTool)) if untranslatable and not drop_params: diff --git a/litellm/llms/openai/responses/count_tokens/token_counter.py b/litellm/llms/openai/responses/count_tokens/token_counter.py index 02e6c98ca07..75fe22ee928 100644 --- a/litellm/llms/openai/responses/count_tokens/token_counter.py +++ b/litellm/llms/openai/responses/count_tokens/token_counter.py @@ -148,7 +148,6 @@ class OpenAITokenCounter(BaseTokenCounter): messages_format: TokenCountMessagesFormat, output_format: JsonValue, api_base: str | None, - drop_params: bool, ) -> ResponsesCountRequestParts: match messages_format: case "anthropic": @@ -159,7 +158,6 @@ class OpenAITokenCounter(BaseTokenCounter): tools=tools, output_format=output_format, maps_tool_search=openai_runs_tool_search(model_to_use, api_base), - drop_params=drop_params, ) case "openai": input_items, instructions = OpenAICountTokensConfig.messages_to_responses_input( @@ -239,7 +237,6 @@ class OpenAITokenCounter(BaseTokenCounter): messages_format=messages_format, output_format=output_format, api_base=resolved.api_base, - drop_params=deployment_drops_params(deployment), ) # If no input items were produced (e.g., system-only messages), fall back to local counting diff --git a/litellm/llms/openai/responses/count_tokens/transformation.py b/litellm/llms/openai/responses/count_tokens/transformation.py index 8aad129da40..f723245a096 100644 --- a/litellm/llms/openai/responses/count_tokens/transformation.py +++ b/litellm/llms/openai/responses/count_tokens/transformation.py @@ -20,11 +20,7 @@ from litellm.llms.anthropic.messages.native.utils import normalize_native_messag from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) -from litellm.llms.openai.responses.transformation import keeps_anthropic_tool_search -from litellm.responses.litellm_completion_transformation.tool_search import ( - tools_with_anthropic_tool_search_mapped, - tools_without_anthropic_tool_search, -) +from litellm.responses.litellm_completion_transformation.tool_search import tools_with_anthropic_tool_search_mapped from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.anthropic import AnthropicMessagesRequest, AnthropicOutputSchema from litellm.types.llms.openai import ResponsesAPIRequestParams, replayable_responses_input @@ -165,14 +161,6 @@ def chat_content_blocks_to_responses_content( return "\n".join(part["text"] for part in parts if part["type"] == "input_text") -def _tools_inference_sends( - model: str, tools: Sequence[Mapping[str, object]], *, drop_params: bool -) -> Sequence[Mapping[str, object]] | None: - if keeps_anthropic_tool_search(model, tools, drop_params=drop_params): - return tools_with_anthropic_tool_search_mapped(tools) - return tools_without_anthropic_tool_search(tools) - - class OpenAICountTokensConfig: """ Configuration and transformation logic for OpenAI Responses API token counting. @@ -357,7 +345,6 @@ class OpenAICountTokensConfig: tools: Sequence[Mapping[str, object]] | None, output_format: JsonValue = None, maps_tool_search: bool = False, - drop_params: bool = False, ) -> ResponsesCountRequestParts: """Convert with the adapter /v1/messages inference uses, so the count sees the same input it bills.""" anthropic_request: Final = AnthropicMessagesRequest( @@ -379,9 +366,7 @@ class OpenAICountTokensConfig: translated.get("text"), ) counted_tools: Final = ( - _tools_inference_sends(model, translated_tools, drop_params=drop_params) - if maps_tool_search and translated_tools is not None - else translated_tools + tools_with_anthropic_tool_search_mapped(translated_tools) if maps_tool_search else translated_tools ) return input_items, instructions, None if counted_tools is None else tuple(counted_tools), text diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 3e2b58f7fbf..9b8515a9ecd 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -158,17 +158,12 @@ def _tool_deferral(tool: object) -> _ToolDeferral: return _ToolDeferral() -def _runs_tool_search(model: str, tools: Sequence[object]) -> bool: +def _runs_tool_search(model: str, tools: Sequence[ALL_RESPONSES_API_TOOL_PARAMS]) -> bool: return any(_tool_deferral(tool).defers for tool in tools) and litellm.supports_tool_search( model=model, custom_llm_provider=LlmProviders.OPENAI.value ) -def keeps_anthropic_tool_search(model: str, tools: Sequence[object], *, drop_params: bool) -> bool: - callers_own_search: Final = any(is_openai_tool_search_tool(tool) for tool in tools) - return callers_own_search or not drop_params or _runs_tool_search(model, tools) - - def _without_defer_loading(tool: ALL_RESPONSES_API_TOOL_PARAMS) -> ALL_RESPONSES_API_TOOL_PARAMS: deferral: Final = _tool_deferral(tool) if not deferral.defers: @@ -733,7 +728,8 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): ) -> Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None: if tools is None or not any(is_anthropic_tool_search_chat_tool(tool) for tool in tools): return tools - if keeps_anthropic_tool_search(model, tools, drop_params=drops_params(litellm_params)): + callers_own_search: Final = any(is_openai_tool_search_tool(tool) for tool in tools) + if callers_own_search or not drops_params(litellm_params) or _runs_tool_search(model, tools): return tools_with_anthropic_tool_search_mapped(tools) verbose_logger.warning( "Dropping tool search under drop_params: OpenAI model '%s' cannot run it or no tool is deferred, so every " diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py index 650d412acbf..fb96039e73a 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py @@ -889,6 +889,56 @@ async def test_drop_params_drops_tool_search_and_its_deferral_on_an_openai_model ] +UNDEFERRED_WEATHER: Final = {key: value for key, value in DEFERRED_WEATHER.items() if key != "defer_loading"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "weather", + [ + pytest.param(UNDEFERRED_WEATHER, id="no-defer-loading"), + pytest.param({**DEFERRED_WEATHER, "defer_loading": "true"}, id="defer-loading-string"), + ], +) +async def test_tool_search_with_no_deferred_tool_answers_400_naming_drop_params_before_any_openai_call( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, weather: Mapping[str, object] +): + with pytest.raises(litellm.UnsupportedParamsError) as error: + await _openai_request_for(respx_mock, monkeypatch, [TOOL_SEARCH, weather], drop_params=False) + + assert error.value.status_code == 400 + assert "tool_search_tool_bm25_20251119" in error.value.message + assert "drop_params" in error.value.message + assert not respx_mock.calls + + +@pytest.mark.asyncio +@pytest.mark.parametrize("tool_type", [pytest.param(["bash"], id="list"), pytest.param({"name": "bash"}, id="object")]) +async def test_a_tool_whose_type_is_not_a_string_reaches_openai_as_a_function_beside_tool_search( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, tool_type: object +): + upstream: Final = await _openai_request_for( + respx_mock, monkeypatch, [TOOL_SEARCH, DEFERRED_WEATHER, {"type": tool_type, "name": "odd_tool"}] + ) + + assert [(tool["type"], tool.get("name")) for tool in _tools_sent(upstream)] == [ + ("tool_search", None), + ("function", "get_weather"), + ("function", "odd_tool"), + ] + + +@pytest.mark.asyncio +async def test_additional_drop_params_tools_sends_no_tools_even_beside_one_openai_cannot_run( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + upstream: Final = await _openai_request_for( + respx_mock, monkeypatch, [WEB_FETCH, TYPED_BASH], drop_params=False, additional_drop_params=["tools"] + ) + + assert "tools" not in json.loads(upstream.calls.last.request.content) + + @pytest.mark.asyncio async def test_a_tool_openai_cannot_run_answers_400_before_any_openai_call_without_drop_params( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch diff --git a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py index f6ff8aab381..3de097476c8 100644 --- a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py +++ b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py @@ -255,10 +255,17 @@ async def test_anthropic_tool_search_is_counted_as_the_openai_tool_search_infere pytest.param(UNDEFERRED_TOOL_SEARCH_TOOLS, [("function", "get_weather", None)], id="nothing-is-deferred"), ], ) -async def test_drop_params_counts_anthropic_tool_search_only_where_inference_keeps_it( - monkeypatch: pytest.MonkeyPatch, tools: list[dict[str, object]], expected: list[tuple[object, object, object]] +@pytest.mark.parametrize( + "litellm_params", + [pytest.param({}, id="without-drop-params"), pytest.param({"drop_params": True}, id="drop-params")], +) +async def test_anthropic_tool_search_is_counted_only_beside_a_deferred_tool( + monkeypatch: pytest.MonkeyPatch, + tools: list[dict[str, object]], + expected: list[tuple[object, object, object]], + litellm_params: Mapping[str, object], ) -> None: - assert await _counted_tools(monkeypatch, "gpt-5.6-luna", {"drop_params": True}, tools) == expected + assert await _counted_tools(monkeypatch, "gpt-5.6-luna", litellm_params, tools) == expected @pytest.mark.asyncio From 0150e87bf7d32c6e527824ceaf59e076911a9207 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 05:56:34 -0700 Subject: [PATCH 08/11] fix(anthropic): omit tool_choice when additional_drop_params drops tools --- .../messages/responses_adapters/handler.py | 5 ++-- .../test_responses_adapters_handler.py | 23 +++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index ef023e99ec6..3df19d8151c 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -315,9 +315,10 @@ def _build_responses_kwargs( request_data["temperature"] = temperature if thinking: request_data["thinking"] = thinking - if tool_choice: + drops_tools: Final = "tools" in drop_policy(forwarded_kwargs).additional_drop_params + if tool_choice and not drops_tools: request_data["tool_choice"] = tool_choice - if tools and "tools" not in drop_policy(forwarded_kwargs).additional_drop_params: + if tools and not drops_tools: request_data["tools"] = tools if top_p is not None: request_data["top_p"] = top_p diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py index fb96039e73a..c9a750f43ec 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py @@ -939,6 +939,29 @@ async def test_additional_drop_params_tools_sends_no_tools_even_beside_one_opena assert "tools" not in json.loads(upstream.calls.last.request.content) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "tool_choice", + [{"type": "tool", "name": "web_fetch"}, {"type": "tool", "name": "bash"}, {"type": "any"}], + ids=["forcing-a-tool-openai-cannot-run", "forcing-a-client-tool", "forcing-any-tool"], +) +async def test_additional_drop_params_tools_sends_no_tool_choice_forcing_a_dropped_tool( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, tool_choice: Mapping[str, object] +): + upstream: Final = await _openai_request_for( + respx_mock, + monkeypatch, + [WEB_FETCH, TYPED_BASH], + drop_params=True, + additional_drop_params=["tools"], + tool_choice=dict(tool_choice), + ) + + sent: Final = json.loads(upstream.calls.last.request.content) + assert "tools" not in sent + assert "tool_choice" not in sent + + @pytest.mark.asyncio async def test_a_tool_openai_cannot_run_answers_400_before_any_openai_call_without_drop_params( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch From 45d9d5c95c7efc07dead496609c2bd0b161f759d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 08:53:56 -0700 Subject: [PATCH 09/11] fix(anthropic): keep tool search deferral through forced tools, drop lists, guardrails, and custom A tool_choice forcing a deferred function now sends that function loaded, so OpenAI can find it. additional_drop_params naming tool_choice or tools[*].defer_loading is read before tool search is kept. A wrapped custom.defer_loading counts as deferral, a guardrail write-back keeps defer_loading, and an unversioned typed tool keeps the client's own schema. --- .../chat/guardrail_translation/handler.py | 31 +++++- .../messages/responses_adapters/handler.py | 10 +- .../responses_adapters/tool_translation.py | 42 +++++-- .../responses_adapters/transformation.py | 15 ++- .../test_anthropic_guardrail_handler.py | 24 ++++ .../test_responses_adapters_handler.py | 104 ++++++++++++++++++ 6 files changed, 203 insertions(+), 23 deletions(-) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 31b0f06251d..7d0435976b6 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -258,6 +258,27 @@ def _with_original_name(tool: ChatCompletionToolParam, original_names: Mapping[s return {**tool, "function": {**function, "name": original}} +def _request_deferrals(request: Mapping[str, object]) -> Mapping[str, object]: + tools: Final = tuple( + fields for tool in list_field(request, "tools") if (fields := as_str_mapping(tool)) is not None + ) + return MappingProxyType( + { + name: tool["defer_loading"] + for tool in tools + if isinstance(name := tool.get("name"), str) and "defer_loading" in tool + } + ) + + +def _with_request_deferral(tool: AllAnthropicToolsValues, deferrals: Mapping[str, object]) -> Mapping[str, object]: + fields: Final = as_str_mapping(tool) or {} + name: Final = fields.get("name") + if "defer_loading" in fields or not isinstance(name, str) or name not in deferrals: + return fields + return {**fields, "defer_loading": deferrals[name]} + + def _write_back_system_block(system: object, block_idx: int, response: str) -> None: if not isinstance(system, list): return @@ -688,9 +709,8 @@ class AnthropicMessagesHandler(BaseTranslation): model: Final = data.get("model") if model: inputs["model"] = model - original_tool_names: Final = _original_tool_names( - cast(Mapping[str, object], data) # cast-ok: Anthropic request body - ) + request_body: Final = cast(Mapping[str, object], data) # cast-ok: Anthropic request body + original_tool_names: Final = _original_tool_names(request_body) guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail( inputs=inputs, request_data=data, @@ -703,13 +723,14 @@ class AnthropicMessagesHandler(BaseTranslation): if guardrailed_tools is not None: # Convert tools back from OpenAI format to Anthropic format anthropic_config: Final = AnthropicConfig() - anthropic_tools: Final[list[AllAnthropicToolsValues]] = [] + deferrals: Final = _request_deferrals(request_body) + anthropic_tools: Final[list[Mapping[str, object]]] = [] for tool in guardrailed_tools: converted_tool, mcp_server = anthropic_config.map_tool_helper( _with_original_name(tool, original_tool_names.declared) ) if converted_tool is not None: - anthropic_tools.append(converted_tool) + anthropic_tools.append(_with_request_deferral(converted_tool, deferrals)) # Note: MCP servers are handled separately in the main transformation data["tools"] = ( merge_returned_tools_into_request_tools( diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index 3df19d8151c..0a6f68f9354 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -183,6 +183,9 @@ def _provider_returns_encrypted_reasoning(model: str, custom_llm_provider: objec return responses_config is not None and "include" in responses_config.get_supported_openai_params(provider_model) +_DEFERRAL_DROP_PATH: Final = "tools[*].defer_loading" + + def _maps_tool_search(model: str, forwarded_kwargs: Mapping[str, object]) -> bool: provider: Final = _resolved_provider(model, forwarded_kwargs.get("custom_llm_provider")) return provider == LlmProviders.OPENAI.value and openai_runs_tool_search( @@ -315,8 +318,9 @@ def _build_responses_kwargs( request_data["temperature"] = temperature if thinking: request_data["thinking"] = thinking - drops_tools: Final = "tools" in drop_policy(forwarded_kwargs).additional_drop_params - if tool_choice and not drops_tools: + dropped: Final = drop_policy(forwarded_kwargs).additional_drop_params + drops_tools: Final = "tools" in dropped + if tool_choice and not drops_tools and "tool_choice" not in dropped: request_data["tool_choice"] = tool_choice if tools and not drops_tools: request_data["tools"] = tools @@ -333,7 +337,7 @@ def _build_responses_kwargs( ), auto_summary_level=auto_reasoning_summary(model, forwarded_kwargs.get("custom_llm_provider")), drop_params=drops_unsupported_params(forwarded_kwargs.get("drop_params")), - maps_tool_search=_maps_tool_search(model, forwarded_kwargs), + maps_tool_search=_maps_tool_search(model, forwarded_kwargs) and _DEFERRAL_DROP_PATH not in dropped, ) # Normalize reasoning effort based on model capabilities diff --git a/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py b/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py index 13a41f93d47..4d593a369fa 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py +++ b/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py @@ -236,16 +236,38 @@ def is_tool_search(tool: Mapping[str, object]) -> bool: return isinstance(tool_type, str) and tool_type in ANTHROPIC_TOOL_SEARCH_TOOL_TYPES -def _defers_a_function(tool: Mapping[str, object]) -> bool: - translated: Final = _translated_tool(tool, False) if tool.get("defer_loading") is True else None +def forced_tool_name(tool_choice: object) -> str | None: + match tool_choice: + case {"type": "tool", "name": str() as name}: + return name + case _: + return None + + +def _defer_loading(tool: Mapping[str, object], forced: str | None) -> object: + if forced is not None and tool.get("name") == forced: + return None + if "defer_loading" in tool: + return tool["defer_loading"] + match tool.get("custom"): + case {"defer_loading": bool() as wrapped}: + return wrapped + case _: + return None + + +def _defers_a_function(tool: Mapping[str, object], forced: str | None) -> bool: + translated: Final = _translated_tool(tool, False) if _defer_loading(tool, forced) is True else None return isinstance(translated, ResponsesTool) and translated.payload.get("type") == "function" -def keeps_tool_search(tools: Sequence[Mapping[str, object]], *, maps_tool_search: bool) -> bool: +def keeps_tool_search( + tools: Sequence[Mapping[str, object]], *, maps_tool_search: bool, forced: str | None = None +) -> bool: return ( maps_tool_search and any(is_tool_search(tool) for tool in tools) - and any(_defers_a_function(tool) for tool in tools) + and any(_defers_a_function(tool, forced) for tool in tools) ) @@ -257,9 +279,11 @@ def _with_deferral(translated: TranslatedTool, defer_loading: object) -> Transla return translated -def responses_tool(tool: Mapping[str, object], *, runs_tool_search: bool = False) -> TranslatedTool: +def responses_tool( + tool: Mapping[str, object], *, runs_tool_search: bool = False, forced: str | None = None +) -> TranslatedTool: translated: Final = _translated_tool(tool, runs_tool_search) - return _with_deferral(translated, tool.get("defer_loading")) if runs_tool_search else translated + return _with_deferral(translated, _defer_loading(tool, forced)) if runs_tool_search else translated def _translated_tool(tool: Mapping[str, object], runs_tool_search: bool) -> TranslatedTool: @@ -272,6 +296,8 @@ def _translated_tool(tool: Mapping[str, object], runs_tool_search: bool) -> Tran return _hosted(name, {"type": "web_search_preview"}) if runs_tool_search and is_tool_search(tool): return _tool_search(name, tool) + if not version and family not in _UNVERSIONED_SERVER_TOOL_TYPES: + return _custom_function(name, tool) match family: case "bash": return _bash_tool(name) @@ -281,10 +307,8 @@ def _translated_tool(tool: Mapping[str, object], runs_tool_search: bool) -> Tran return _memory_tool(name) case "computer": return _computer_tool(name, version, tool) - case _ if version or family in _UNVERSIONED_SERVER_TOOL_TYPES: - return UntranslatableTool(name=name, type=type_name) case _: - return _custom_function(name, tool) + return UntranslatableTool(name=name, type=type_name) _DROP_PARAMS_HOW_TO: Final = ( diff --git a/litellm/llms/anthropic/messages/responses_adapters/transformation.py b/litellm/llms/anthropic/messages/responses_adapters/transformation.py index c0765f18670..cd83b75163b 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/messages/responses_adapters/transformation.py @@ -30,6 +30,7 @@ from litellm.llms.anthropic.messages.responses_adapters.tool_translation import ResponsesTool, TranslatedTool, UntranslatableTool, + forced_tool_name, keeps_tool_search, log_dropped_tools, responses_tool, @@ -471,9 +472,12 @@ class LiteLLMAnthropicToResponsesAPIAdapter: *, drop_params: bool = True, maps_tool_search: bool = False, + forced: str | None = None, ) -> tuple[TranslatedTool, ...]: - runs_tool_search: Final = keeps_tool_search(tools, maps_tool_search=maps_tool_search) - translated: Final = tuple(responses_tool(tool, runs_tool_search=runs_tool_search) for tool in tools) + runs_tool_search: Final = keeps_tool_search(tools, maps_tool_search=maps_tool_search, forced=forced) + translated: Final = tuple( + responses_tool(tool, runs_tool_search=runs_tool_search, forced=forced) for tool in tools + ) untranslatable: Final = tuple(tool for tool in translated if isinstance(tool, UntranslatableTool)) if untranslatable and not drop_params: raise untranslatable_tools_error(untranslatable) @@ -671,20 +675,19 @@ class LiteLLMAnthropicToResponsesAPIAdapter: if "top_p" in anthropic_request: responses_kwargs["top_p"] = anthropic_request["top_p"] + tool_choice: Final = cast(AnthropicMessagesToolChoice | None, anthropic_request.get("tool_choice")) translated_tools: Final = self.translated_tools( cast(list[AllAnthropicToolsValues], anthropic_request.get("tools") or []), drop_params=drop_params, maps_tool_search=maps_tool_search, + forced=forced_tool_name(tool_choice), ) responses_tools: Final = [dict(tool.payload) for tool in translated_tools if isinstance(tool, ResponsesTool)] if responses_tools: responses_kwargs["tools"] = responses_tools - tool_choice: Final = anthropic_request.get("tool_choice") responses_tool_choice: Final = ( - self.translate_tool_choice_to_responses_api( - cast(AnthropicMessagesToolChoice, tool_choice), translated_tools, drop_params=drop_params - ) + self.translate_tool_choice_to_responses_api(tool_choice, translated_tools, drop_params=drop_params) if tool_choice and (responses_tools or not translated_tools) else None ) diff --git a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index 9e88b59adc9..6d40173034c 100644 --- a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -649,6 +649,30 @@ class TestAnthropicMessagesHandlerInputProcessing: assert {"googleMaps": {"enable_widget": True}} in data["tools"] assert [tool["name"] for tool in data["tools"] if "name" in tool] == ["get_weather"] + @pytest.mark.asyncio + async def test_a_deferred_tool_keeps_its_defer_loading_through_a_guardrail_that_returns_the_tools(self): + handler = AnthropicMessagesHandler() + guardrail = MockPassThroughGuardrail(guardrail_name="test") + data = { + "model": "gpt-5.6-luna", + "messages": [{"role": "user", "content": "what is AAPL at?"}], + "tools": [ + {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"}, + {"name": "get_stock", "input_schema": {"type": "object", "properties": {}}, "defer_loading": True}, + {"name": "get_weather", "input_schema": {"type": "object", "properties": {}}, "defer_loading": False}, + {"name": "get_time", "input_schema": {"type": "object", "properties": {}}}, + ], + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert [(tool["name"], tool.get("defer_loading")) for tool in data["tools"]] == [ + ("tool_search_tool_regex", None), + ("get_stock", True), + ("get_weather", False), + ("get_time", None), + ] + @pytest.mark.asyncio async def test_midturn_system_correction_is_guardrailed_when_top_level_system_is_skipped( self, diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py index c9a750f43ec..2ccb0f6ad6f 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py @@ -962,6 +962,110 @@ async def test_additional_drop_params_tools_sends_no_tool_choice_forcing_a_dropp assert "tool_choice" not in sent +DEFERRED_STOCK: Final = { + "name": "get_stock", + "description": "Price for a ticker", + "input_schema": {"type": "object", "properties": {"ticker": {"type": "string"}}, "required": ["ticker"]}, + "defer_loading": True, +} +FORCE_STOCK: Final = {"type": "tool", "name": "get_stock"} + + +def _tool_search_wire(upstream: respx.Route) -> list[tuple[object, ...]]: + return [(tool["type"], tool.get("name"), tool.get("defer_loading")) for tool in _tools_sent(upstream)] + + +@pytest.mark.asyncio +async def test_a_tool_choice_forcing_a_deferred_function_sends_that_function_loaded_beside_tool_search( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + upstream: Final = await _openai_request_for( + respx_mock, + monkeypatch, + [TOOL_SEARCH, DEFERRED_STOCK, DEFERRED_WEATHER], + drop_params=False, + tool_choice=dict(FORCE_STOCK), + ) + + assert _tool_search_wire(upstream) == [ + ("tool_search", None, None), + ("function", "get_stock", None), + ("function", "get_weather", True), + ] + assert json.loads(upstream.calls.last.request.content)["tool_choice"] == {"type": "function", "name": "get_stock"} + + +@pytest.mark.asyncio +async def test_drop_params_drops_tool_search_when_the_forced_function_is_the_only_deferred_one( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + upstream: Final = await _openai_request_for( + respx_mock, monkeypatch, [TOOL_SEARCH, DEFERRED_STOCK], drop_params=True, tool_choice=dict(FORCE_STOCK) + ) + + assert _tool_search_wire(upstream) == [("function", "get_stock", None)] + assert json.loads(upstream.calls.last.request.content)["tool_choice"] == {"type": "function", "name": "get_stock"} + + +@pytest.mark.asyncio +async def test_additional_drop_params_tool_choice_sends_tool_search_without_the_choice_that_forces_it( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + upstream: Final = await _openai_request_for( + respx_mock, + monkeypatch, + [TOOL_SEARCH, DEFERRED_WEATHER], + drop_params=False, + additional_drop_params=["tool_choice"], + tool_choice={"type": "tool", "name": "tool_search_tool_bm25"}, + ) + + assert "tool_choice" not in json.loads(upstream.calls.last.request.content) + assert _tool_search_wire(upstream) == [("tool_search", None, None), ("function", "get_weather", True)] + + +@pytest.mark.asyncio +async def test_additional_drop_params_of_every_defer_loading_answers_400_for_tool_search_without_drop_params( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + with pytest.raises(litellm.UnsupportedParamsError) as error: + await _openai_request_for( + respx_mock, + monkeypatch, + [TOOL_SEARCH, DEFERRED_WEATHER], + drop_params=False, + additional_drop_params=["tools[*].defer_loading"], + ) + + assert "tool_search_tool_bm25_20251119" in error.value.message + assert not respx_mock.calls + + +@pytest.mark.asyncio +async def test_a_deferral_inside_the_custom_object_keeps_tool_search_with_the_tool_deferred( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + wrapped: Final = {**UNDEFERRED_WEATHER, "custom": {"defer_loading": True}} + + upstream: Final = await _openai_request_for(respx_mock, monkeypatch, [TOOL_SEARCH, wrapped], drop_params=False) + + assert _tool_search_wire(upstream) == [("tool_search", None, None), ("function", "get_weather", True)] + assert all("custom" not in tool for tool in _tools_sent(upstream)) + + +@pytest.mark.asyncio +async def test_an_unversioned_typed_tool_name_reaches_openai_with_the_clients_own_schema( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + schema: Final = {"type": "object", "properties": {"script": {"type": "string"}}, "required": ["script"]} + + upstream: Final = await _openai_request_for( + respx_mock, monkeypatch, [{"type": "bash", "name": "bash", "input_schema": schema}], drop_params=False + ) + + assert [(tool["name"], tool["parameters"]) for tool in _tools_sent(upstream)] == [("bash", schema)] + + @pytest.mark.asyncio async def test_a_tool_openai_cannot_run_answers_400_before_any_openai_call_without_drop_params( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch From 06dcd798cfbd9183d9ab59427d3cbe93950e584b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 09:45:52 -0700 Subject: [PATCH 10/11] fix(anthropic): keep a wrapped defer_loading through a guardrail that returns the tools --- .../chat/guardrail_translation/handler.py | 24 ++++++++++++++----- .../test_anthropic_guardrail_handler.py | 12 ++++++---- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 7d0435976b6..56a09d655a5 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -258,25 +258,37 @@ def _with_original_name(tool: ChatCompletionToolParam, original_names: Mapping[s return {**tool, "function": {**function, "name": original}} -def _request_deferrals(request: Mapping[str, object]) -> Mapping[str, object]: +def _request_deferral(tool: Mapping[str, object]) -> Mapping[str, object] | None: + if "defer_loading" in tool: + return {"defer_loading": tool["defer_loading"]} + match tool.get("custom"): + case {"defer_loading": bool() as wrapped}: + return {"custom": {"defer_loading": wrapped}} + case _: + return None + + +def _request_deferrals(request: Mapping[str, object]) -> Mapping[str, Mapping[str, object]]: tools: Final = tuple( fields for tool in list_field(request, "tools") if (fields := as_str_mapping(tool)) is not None ) return MappingProxyType( { - name: tool["defer_loading"] + name: deferral for tool in tools - if isinstance(name := tool.get("name"), str) and "defer_loading" in tool + if isinstance(name := tool.get("name"), str) and (deferral := _request_deferral(tool)) is not None } ) -def _with_request_deferral(tool: AllAnthropicToolsValues, deferrals: Mapping[str, object]) -> Mapping[str, object]: +def _with_request_deferral( + tool: AllAnthropicToolsValues, deferrals: Mapping[str, Mapping[str, object]] +) -> Mapping[str, object]: fields: Final = as_str_mapping(tool) or {} name: Final = fields.get("name") - if "defer_loading" in fields or not isinstance(name, str) or name not in deferrals: + if "defer_loading" in fields or "custom" in fields or not isinstance(name, str) or name not in deferrals: return fields - return {**fields, "defer_loading": deferrals[name]} + return {**fields, **deferrals[name]} def _write_back_system_block(system: object, block_idx: int, response: str) -> None: diff --git a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index 6d40173034c..7457e42fe93 100644 --- a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -661,16 +661,18 @@ class TestAnthropicMessagesHandlerInputProcessing: {"name": "get_stock", "input_schema": {"type": "object", "properties": {}}, "defer_loading": True}, {"name": "get_weather", "input_schema": {"type": "object", "properties": {}}, "defer_loading": False}, {"name": "get_time", "input_schema": {"type": "object", "properties": {}}}, + {"name": "get_quote", "input_schema": {"type": "object", "properties": {}}, "custom": {"defer_loading": True}}, ], } await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) - assert [(tool["name"], tool.get("defer_loading")) for tool in data["tools"]] == [ - ("tool_search_tool_regex", None), - ("get_stock", True), - ("get_weather", False), - ("get_time", None), + assert [(tool["name"], tool.get("defer_loading"), tool.get("custom")) for tool in data["tools"]] == [ + ("tool_search_tool_regex", None, None), + ("get_stock", True, None), + ("get_weather", False, None), + ("get_time", None, None), + ("get_quote", None, {"defer_loading": True}), ] @pytest.mark.asyncio From 64f1ab196bbba49a20562b754c4e9e8619172a8f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:37:41 -0700 Subject: [PATCH 11/11] fix(anthropic): keep tool search off for any defer_loading drop path and restore only boolean deferrals A drop path such as tools[1].defer_loading or the SDK's tuple form now turns the tool search mapping off like tools[*].defer_loading, the count path reads the deployment's drop list, and a guardrail write-back restores only a boolean defer_loading --- .../chat/guardrail_translation/handler.py | 8 +++---- .../messages/responses_adapters/handler.py | 13 ++++++------ .../responses_adapters/tool_translation.py | 21 ++++++++++++++++--- .../responses/count_tokens/token_counter.py | 14 +++++++++++-- .../test_anthropic_guardrail_handler.py | 2 ++ .../test_responses_adapters_handler.py | 16 ++++++++++---- .../count_tokens/test_token_counter.py | 3 +++ 7 files changed, 57 insertions(+), 20 deletions(-) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 56a09d655a5..c9c9fcae673 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -259,10 +259,10 @@ def _with_original_name(tool: ChatCompletionToolParam, original_names: Mapping[s def _request_deferral(tool: Mapping[str, object]) -> Mapping[str, object] | None: - if "defer_loading" in tool: - return {"defer_loading": tool["defer_loading"]} - match tool.get("custom"): - case {"defer_loading": bool() as wrapped}: + match tool: + case {"defer_loading": bool() as deferred}: + return {"defer_loading": deferred} + case {"custom": {"defer_loading": bool() as wrapped}}: return {"custom": {"defer_loading": wrapped}} case _: return None diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index 0a6f68f9354..11e0f0fb3fb 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -50,7 +50,7 @@ from ..convertible_blocks import reject_malformed_tool_result_text from ..safeguards import with_safeguard_results from ..utils import litellm_logging_obj_from_kwargs, local_model_name from .streaming_iterator import AnthropicResponsesStreamWrapper -from .tool_translation import openai_runs_tool_search +from .tool_translation import maps_openai_tool_search from .transformation import LiteLLMAnthropicToResponsesAPIAdapter AnthropicRequestMessages: TypeAlias = list[AllAnthropicMessageValues] | list[dict[str, object]] @@ -183,13 +183,12 @@ def _provider_returns_encrypted_reasoning(model: str, custom_llm_provider: objec return responses_config is not None and "include" in responses_config.get_supported_openai_params(provider_model) -_DEFERRAL_DROP_PATH: Final = "tools[*].defer_loading" - - def _maps_tool_search(model: str, forwarded_kwargs: Mapping[str, object]) -> bool: provider: Final = _resolved_provider(model, forwarded_kwargs.get("custom_llm_provider")) - return provider == LlmProviders.OPENAI.value and openai_runs_tool_search( - local_model_name(model, provider), forwarded_kwargs.get("api_base") or forwarded_kwargs.get("base_url") + return provider == LlmProviders.OPENAI.value and maps_openai_tool_search( + local_model_name(model, provider), + forwarded_kwargs.get("api_base") or forwarded_kwargs.get("base_url"), + forwarded_kwargs.get("additional_drop_params"), ) @@ -337,7 +336,7 @@ def _build_responses_kwargs( ), auto_summary_level=auto_reasoning_summary(model, forwarded_kwargs.get("custom_llm_provider")), drop_params=drops_unsupported_params(forwarded_kwargs.get("drop_params")), - maps_tool_search=_maps_tool_search(model, forwarded_kwargs) and _DEFERRAL_DROP_PATH not in dropped, + maps_tool_search=_maps_tool_search(model, forwarded_kwargs), ) # Normalize reasoning effort based on model capabilities diff --git a/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py b/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py index 4d593a369fa..95ae765452a 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py +++ b/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py @@ -3,6 +3,8 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass, replace from typing import Final, TypeAlias +from pydantic import ConfigDict, TypeAdapter, ValidationError + import litellm from litellm._logging import verbose_logger from litellm.exceptions import UnsupportedParamsError @@ -13,6 +15,9 @@ from litellm.types.utils import LlmProviders _VERSIONED_TOOL_TYPE: Final = re.compile(r"(?P[a-z0-9_]+?)_(?P\d{8})") _UNVERSIONED_SERVER_TOOL_TYPES: Final = frozenset({"mcp_toolset", "computer_toolset"}) +_DEFERRAL_DROP_PATH: Final = re.compile(r"tools\[[^\]]*\]\.defer_loading") +_DropPaths: TypeAlias = list[object] | tuple[object, ...] +_DROP_PATHS: Final[TypeAdapter[_DropPaths]] = TypeAdapter(_DropPaths, config=ConfigDict(strict=True)) _TEXT_EDITOR_INSERT_TEXT_SINCE: Final = "20250429" _COMPUTER_MODIFIERS_SINCE: Final = "20250124" _COMPUTER_ZOOM_SINCE: Final = "20251124" @@ -342,7 +347,17 @@ def unforceable_tool_choice_error(name: str) -> UnsupportedParamsError: ) -def openai_runs_tool_search(model: str, api_base: object) -> bool: - return targets_openai_api(api_base) and litellm.supports_tool_search( - model=model, custom_llm_provider=LlmProviders.OPENAI.value +def _drops_deferral(additional_drop_params: object) -> bool: + try: + paths: Final = _DROP_PATHS.validate_python(additional_drop_params) + except ValidationError: + return False + return any(isinstance(path, str) and _DEFERRAL_DROP_PATH.fullmatch(path) is not None for path in paths) + + +def maps_openai_tool_search(model: str, api_base: object, additional_drop_params: object) -> bool: + return ( + targets_openai_api(api_base) + and not _drops_deferral(additional_drop_params) + and litellm.supports_tool_search(model=model, custom_llm_provider=LlmProviders.OPENAI.value) ) diff --git a/litellm/llms/openai/responses/count_tokens/token_counter.py b/litellm/llms/openai/responses/count_tokens/token_counter.py index 75fe22ee928..16cb4712997 100644 --- a/litellm/llms/openai/responses/count_tokens/token_counter.py +++ b/litellm/llms/openai/responses/count_tokens/token_counter.py @@ -18,7 +18,7 @@ from litellm.litellm_core_utils.tool_fields import DropPolicy from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError from litellm.llms.anthropic.messages.convertible_blocks import deployment_drops_params, unconvertible_blocks_error from litellm.llms.anthropic.messages.responses_adapters.handler import count_view_messages -from litellm.llms.anthropic.messages.responses_adapters.tool_translation import openai_runs_tool_search +from litellm.llms.anthropic.messages.responses_adapters.tool_translation import maps_openai_tool_search from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) @@ -64,6 +64,14 @@ class _CountCredential: credential: str | OpenAIWorkloadIdentityConfig +def _deployment_drop_paths(deployment: Mapping[str, object] | None) -> object: + match deployment: + case {"litellm_params": {"additional_drop_params": object() as additional_drop_params}}: + return additional_drop_params + case _: + return None + + def _count_credential(deployment: Mapping[str, object] | None, model_label: str) -> _CountCredential | OpenAIError: litellm_params: Final = as_str_mapping(None if deployment is None else deployment.get("litellm_params")) or {} api_base: Final = litellm_params.get("api_base") @@ -148,6 +156,7 @@ class OpenAITokenCounter(BaseTokenCounter): messages_format: TokenCountMessagesFormat, output_format: JsonValue, api_base: str | None, + additional_drop_params: object, ) -> ResponsesCountRequestParts: match messages_format: case "anthropic": @@ -157,7 +166,7 @@ class OpenAITokenCounter(BaseTokenCounter): system=system, tools=tools, output_format=output_format, - maps_tool_search=openai_runs_tool_search(model_to_use, api_base), + maps_tool_search=maps_openai_tool_search(model_to_use, api_base, additional_drop_params), ) case "openai": input_items, instructions = OpenAICountTokensConfig.messages_to_responses_input( @@ -237,6 +246,7 @@ class OpenAITokenCounter(BaseTokenCounter): messages_format=messages_format, output_format=output_format, api_base=resolved.api_base, + additional_drop_params=_deployment_drop_paths(deployment), ) # If no input items were produced (e.g., system-only messages), fall back to local counting diff --git a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index 7457e42fe93..42dbea6c946 100644 --- a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -662,6 +662,7 @@ class TestAnthropicMessagesHandlerInputProcessing: {"name": "get_weather", "input_schema": {"type": "object", "properties": {}}, "defer_loading": False}, {"name": "get_time", "input_schema": {"type": "object", "properties": {}}}, {"name": "get_quote", "input_schema": {"type": "object", "properties": {}}, "custom": {"defer_loading": True}}, + {"name": "get_news", "input_schema": {"type": "object", "properties": {}}, "defer_loading": "true"}, ], } @@ -673,6 +674,7 @@ class TestAnthropicMessagesHandlerInputProcessing: ("get_weather", False, None), ("get_time", None, None), ("get_quote", None, {"defer_loading": True}), + ("get_news", None, None), ] @pytest.mark.asyncio diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py index 2ccb0f6ad6f..10ac27a16a5 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py @@ -2,7 +2,7 @@ import datetime import json import os import sys -from collections.abc import AsyncIterator, Mapping +from collections.abc import AsyncIterator, Mapping, Sequence from itertools import chain from typing import Final, cast from unittest.mock import AsyncMock, patch @@ -1025,8 +1025,16 @@ async def test_additional_drop_params_tool_choice_sends_tool_search_without_the_ @pytest.mark.asyncio -async def test_additional_drop_params_of_every_defer_loading_answers_400_for_tool_search_without_drop_params( - respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +@pytest.mark.parametrize( + "additional_drop_params", + [ + pytest.param(["tools[*].defer_loading"], id="every-tool"), + pytest.param(["tools[1].defer_loading"], id="one-tool-by-index"), + pytest.param(("tools[*].defer_loading",), id="sdk-tuple"), + ], +) +async def test_additional_drop_params_of_a_defer_loading_path_answers_400_for_tool_search_without_drop_params( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, additional_drop_params: Sequence[str] ): with pytest.raises(litellm.UnsupportedParamsError) as error: await _openai_request_for( @@ -1034,7 +1042,7 @@ async def test_additional_drop_params_of_every_defer_loading_answers_400_for_too monkeypatch, [TOOL_SEARCH, DEFERRED_WEATHER], drop_params=False, - additional_drop_params=["tools[*].defer_loading"], + additional_drop_params=additional_drop_params, ) assert "tool_search_tool_bm25_20251119" in error.value.message diff --git a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py index 3de097476c8..dc04a71b2b0 100644 --- a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py +++ b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py @@ -274,6 +274,9 @@ async def test_anthropic_tool_search_is_counted_only_beside_a_deferred_tool( [ pytest.param("a-model-the-cost-map-does-not-mark-for-tool-search", {}, id="model-without-tool-search"), pytest.param("gpt-5.6-luna", {"api_base": "https://responses.example.test/v1"}, id="other-responses-server"), + pytest.param( + "gpt-5.6-luna", {"additional_drop_params": ["tools[*].defer_loading"]}, id="deployment-drops-deferral" + ), ], ) async def test_anthropic_tool_search_is_left_out_of_the_count_where_inference_cannot_run_it(