diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 7f688ae733b..14118ea67e7 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -394,6 +394,39 @@ def _with_original_name(tool: ChatCompletionToolParam, original_names: Mapping[s return {**tool, "function": {**function, "name": original}} +def _request_deferral(tool: Mapping[str, object]) -> Mapping[str, object] | None: + match tool: + case {"defer_loading": bool() as deferred}: + return {"defer_loading": deferred} + case {"custom": {"defer_loading": bool() as wrapped}}: + return {"custom": {"defer_loading": wrapped}} + case _: + return None + + +def _request_deferrals(request: Mapping[str, object]) -> Mapping[str, Mapping[str, object]]: + tools: Final = tuple( + fields for tool in list_field(request, "tools") if (fields := as_str_mapping(tool)) is not None + ) + return MappingProxyType( + { + name: deferral + for tool in tools + if isinstance(name := tool.get("name"), str) and (deferral := _request_deferral(tool)) is not None + } + ) + + +def _with_request_deferral( + tool: AllAnthropicToolsValues, deferrals: Mapping[str, Mapping[str, object]] +) -> Mapping[str, object]: + fields: Final = as_str_mapping(tool) or {} + name: Final = fields.get("name") + if "defer_loading" in fields or "custom" in fields or not isinstance(name, str) or name not in deferrals: + return fields + return {**fields, **deferrals[name]} + + def _write_back_system_block(system: object, block_idx: int, response: str) -> None: if not isinstance(system, list): return @@ -901,9 +934,8 @@ class AnthropicMessagesHandler(BaseTranslation): model: Final = data.get("model") if model: inputs["model"] = model - original_tool_names: Final = _original_tool_names( - cast(Mapping[str, object], data) # cast-ok: Anthropic request body - ) + request_body: Final = cast(Mapping[str, object], data) # cast-ok: Anthropic request body + original_tool_names: Final = _original_tool_names(request_body) guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail( inputs=inputs, request_data=data, @@ -916,13 +948,14 @@ class AnthropicMessagesHandler(BaseTranslation): if guardrailed_tools is not None: # Convert tools back from OpenAI format to Anthropic format anthropic_config: Final = AnthropicConfig() - anthropic_tools: Final[list[AllAnthropicToolsValues]] = [] + deferrals: Final = _request_deferrals(request_body) + anthropic_tools: Final[list[Mapping[str, object]]] = [] for tool in guardrailed_tools: converted_tool, mcp_server = anthropic_config.map_tool_helper( _with_original_name(tool, original_tool_names.declared) ) if converted_tool is not None: - anthropic_tools.append(converted_tool) + anthropic_tools.append(_with_request_deferral(converted_tool, deferrals)) # Note: MCP servers are handled separately in the main transformation data["tools"] = ( merge_returned_tools_into_request_tools( diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index 0eef3fd87c0..11e0f0fb3fb 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -24,6 +24,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( prompt_cache_breakpoint_for_wire, with_prompt_cache_breakpoint, ) +from litellm.litellm_core_utils.tool_fields import drop_policy from litellm.responses.utils import auto_reasoning_summary from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS from litellm.types.llms.anthropic import ( @@ -38,7 +39,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) from litellm.types.llms.openai import ResponsesAPIResponse -from litellm.types.utils import CountContextManagement +from litellm.types.utils import CountContextManagement, LlmProviders from litellm.utils import ProviderConfigManager from ..adapters.handler import context_management_explicitly_dropped, request_safeguards_evaluator @@ -49,6 +50,7 @@ from ..convertible_blocks import reject_malformed_tool_result_text from ..safeguards import with_safeguard_results from ..utils import litellm_logging_obj_from_kwargs, local_model_name from .streaming_iterator import AnthropicResponsesStreamWrapper +from .tool_translation import maps_openai_tool_search from .transformation import LiteLLMAnthropicToResponsesAPIAdapter AnthropicRequestMessages: TypeAlias = list[AllAnthropicMessageValues] | list[dict[str, object]] @@ -170,15 +172,26 @@ def _forwarded_kwargs(extra_kwargs: Mapping[str, object] | None) -> Mapping[str, return extra_kwargs or {} +def _resolved_provider(model: str, custom_llm_provider: object) -> str: + return custom_llm_provider if isinstance(custom_llm_provider, str) else litellm.get_llm_provider(model=model)[1] + + def _provider_returns_encrypted_reasoning(model: str, custom_llm_provider: object) -> bool: - provider: Final = ( - custom_llm_provider if isinstance(custom_llm_provider, str) else litellm.get_llm_provider(model=model)[1] - ) + provider: Final = _resolved_provider(model, custom_llm_provider) provider_model: Final = local_model_name(model, provider) responses_config: Final = ProviderConfigManager.get_provider_responses_api_config(provider, provider_model) return responses_config is not None and "include" in responses_config.get_supported_openai_params(provider_model) +def _maps_tool_search(model: str, forwarded_kwargs: Mapping[str, object]) -> bool: + provider: Final = _resolved_provider(model, forwarded_kwargs.get("custom_llm_provider")) + return provider == LlmProviders.OPENAI.value and maps_openai_tool_search( + local_model_name(model, provider), + forwarded_kwargs.get("api_base") or forwarded_kwargs.get("base_url"), + forwarded_kwargs.get("additional_drop_params"), + ) + + def _reads_prompt_cache_breakpoint(model: str, forwarded_kwargs: Mapping[str, object]) -> bool: provider: Final = forwarded_kwargs.get("custom_llm_provider") return AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( @@ -304,9 +317,11 @@ def _build_responses_kwargs( request_data["temperature"] = temperature if thinking: request_data["thinking"] = thinking - if tool_choice: + dropped: Final = drop_policy(forwarded_kwargs).additional_drop_params + drops_tools: Final = "tools" in dropped + if tool_choice and not drops_tools and "tool_choice" not in dropped: request_data["tool_choice"] = tool_choice - if tools: + if tools and not drops_tools: request_data["tools"] = tools if top_p is not None: request_data["top_p"] = top_p @@ -321,6 +336,7 @@ def _build_responses_kwargs( ), auto_summary_level=auto_reasoning_summary(model, forwarded_kwargs.get("custom_llm_provider")), drop_params=drops_unsupported_params(forwarded_kwargs.get("drop_params")), + maps_tool_search=_maps_tool_search(model, forwarded_kwargs), ) # Normalize reasoning effort based on model capabilities diff --git a/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py b/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py new file mode 100644 index 00000000000..95ae765452a --- /dev/null +++ b/litellm/llms/anthropic/messages/responses_adapters/tool_translation.py @@ -0,0 +1,363 @@ +import re +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, replace +from typing import Final, TypeAlias + +from pydantic import ConfigDict, TypeAdapter, ValidationError + +import litellm +from litellm._logging import verbose_logger +from litellm.exceptions import UnsupportedParamsError +from litellm.integrations.anthropic_cache_control_hook import targets_openai_api +from litellm.llms.anthropic.messages.utils import json_copy_or_original +from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_TOOL_TYPES +from litellm.types.utils import LlmProviders + +_VERSIONED_TOOL_TYPE: Final = re.compile(r"(?P[a-z0-9_]+?)_(?P\d{8})") +_UNVERSIONED_SERVER_TOOL_TYPES: Final = frozenset({"mcp_toolset", "computer_toolset"}) +_DEFERRAL_DROP_PATH: Final = re.compile(r"tools\[[^\]]*\]\.defer_loading") +_DropPaths: TypeAlias = list[object] | tuple[object, ...] +_DROP_PATHS: Final[TypeAdapter[_DropPaths]] = TypeAdapter(_DropPaths, config=ConfigDict(strict=True)) +_TEXT_EDITOR_INSERT_TEXT_SINCE: Final = "20250429" +_COMPUTER_MODIFIERS_SINCE: Final = "20250124" +_COMPUTER_ZOOM_SINCE: Final = "20251124" +_COMPUTER_ACTIONS_20241022: Final = ( + "key", + "type", + "mouse_move", + "left_click", + "left_click_drag", + "right_click", + "middle_click", + "double_click", + "screenshot", + "cursor_position", +) +_COMPUTER_ACTIONS_20250124: Final = ( + *_COMPUTER_ACTIONS_20241022, + "left_mouse_down", + "left_mouse_up", + "scroll", + "hold_key", + "wait", + "triple_click", +) +_COMPUTER_ACTIONS_20251124: Final = (*_COMPUTER_ACTIONS_20250124, "zoom") + + +@dataclass(frozen=True, slots=True) +class ResponsesTool: + name: str + payload: Mapping[str, object] + choice: Mapping[str, object] | None + + +@dataclass(frozen=True, slots=True) +class UntranslatableTool: + name: str + type: str + + +TranslatedTool: TypeAlias = ResponsesTool | UntranslatableTool + + +def _family_and_version(tool_type: str) -> tuple[str, str]: + versioned: Final = _VERSIONED_TOOL_TYPE.fullmatch(tool_type) + if versioned is None: + return tool_type, "" + return versioned["family"], versioned["version"] + + +def _string(description: str) -> Mapping[str, object]: + return {"type": "string", "description": description} + + +def _integer(description: str) -> Mapping[str, object]: + return {"type": "integer", "description": description} + + +def _integers(count: int, description: str) -> Mapping[str, object]: + return { + "type": "array", + "items": {"type": "integer"}, + "minItems": count, + "maxItems": count, + "description": description, + } + + +def _enum(values: Sequence[str]) -> Mapping[str, object]: + return {"type": "string", "enum": list(values)} + + +def _function(name: str, description: str, properties: Mapping[str, object], required: Sequence[str]) -> ResponsesTool: + return ResponsesTool( + name=name, + payload={ + "type": "function", + "name": name, + "strict": False, + "description": description, + "parameters": {"type": "object", "properties": properties, "required": list(required)}, + }, + choice={"type": "function", "name": name}, + ) + + +def _hosted(name: str, payload: Mapping[str, object]) -> ResponsesTool: + return ResponsesTool(name=name, payload=payload, choice={"type": payload["type"]}) + + +def _bash_tool(name: str) -> ResponsesTool: + return _function( + name, + "Run a command in a persistent bash session. The working directory, environment, and files carry over " + "between calls.", + { + "command": _string("The bash command to run. Required unless restart is true."), + "restart": {"type": "boolean", "description": "Set to true to restart the bash session."}, + }, + (), + ) + + +def _text_editor_tool(name: str, version: str) -> ResponsesTool: + inserts_insert_text: Final = version >= _TEXT_EDITOR_INSERT_TEXT_SINCE + edit_properties: Final[Mapping[str, object]] = { + "command": _enum( + ("view", "create", "str_replace", "insert") + if inserts_insert_text + else ("view", "create", "str_replace", "insert", "undo_edit") + ), + "path": _string("Absolute path to the file or directory."), + "view_range": _integers(2, "Optional [start, end] line numbers for view. An end of -1 reads to the end."), + "file_text": _string("Content of the new file, for create."), + "old_str": _string("Exact text to replace, for str_replace. It must match exactly one place in the file."), + "new_str": _string( + "Replacement text, for str_replace." + if inserts_insert_text + else "Replacement text for str_replace, or the text to add for insert." + ), + "insert_line": _integer("Line number to insert after, for insert. 0 inserts at the start of the file."), + } + return _function( + name, + "View, create, and edit text files. view lists a directory or shows a file with line numbers.", + ( + {**edit_properties, "insert_text": _string("Text to add, for insert.")} + if inserts_insert_text + else edit_properties + ), + ("command", "path"), + ) + + +def _memory_tool(name: str) -> ResponsesTool: + return _function( + name, + "Read and write files in the /memories directory, which persists between conversations. View it before " + "starting a task, and record progress and learnings as you work.", + { + "command": _enum(("view", "create", "str_replace", "insert", "delete", "rename")), + "path": _string("Path under /memories, for every command but rename."), + "view_range": _integers(2, "Optional [start, end] line numbers for view."), + "file_text": _string("Content of the new file, for create."), + "old_str": _string("Exact text to replace, for str_replace."), + "new_str": _string("Replacement text, for str_replace."), + "insert_line": _integer("Line number to insert after, for insert."), + "insert_text": _string("Text to add, for insert."), + "old_path": _string("Current path, for rename."), + "new_path": _string("New path, for rename."), + }, + ("command",), + ) + + +def _computer_properties(version: str, zoom: bool) -> Mapping[str, object]: + pointer: Final[Mapping[str, object]] = { + "text": _string("Text to type for type, or keys to press for key and hold_key, such as ctrl+s."), + "coordinate": _integers(2, "[x, y] pixel position for a click, mouse_move, or the end of left_click_drag."), + "start_coordinate": _integers(2, "[x, y] pixel position where left_click_drag starts."), + } + if version < _COMPUTER_MODIFIERS_SINCE: + return {"action": _enum(_COMPUTER_ACTIONS_20241022), **pointer} + modifiers: Final[Mapping[str, object]] = { + **pointer, + "scroll_direction": _enum(("up", "down", "left", "right")), + "scroll_amount": _integer("Number of scroll steps, for scroll."), + "text": _string( + "Text to type for type, keys to press for key and hold_key, such as ctrl+s, or a modifier key to hold " + "during scroll, such as shift." + ), + "duration": {"type": "number", "description": "Seconds to wait for wait, or to hold for hold_key."}, + "key": _string("Modifier key to hold during a click, such as shift."), + } + if not zoom: + return {"action": _enum(_COMPUTER_ACTIONS_20250124), **modifiers} + return { + "action": _enum(_COMPUTER_ACTIONS_20251124), + **modifiers, + "region": _integers(4, "[x0, y0, x1, y1] screen region to show at full resolution, for zoom."), + } + + +def _computer_tool(name: str, version: str, tool: Mapping[str, object]) -> ResponsesTool: + width: Final = tool.get("display_width_px") + height: Final = tool.get("display_height_px") + display: Final = ( + f"a {width}x{height} pixel display" if isinstance(width, int) and isinstance(height, int) else "a display" + ) + return _function( + name, + f"Control {display} with the mouse and keyboard, and take screenshots to see it. Coordinates are pixels " + "from the top left.", + _computer_properties(version, zoom=version >= _COMPUTER_ZOOM_SINCE and tool.get("enable_zoom") is True), + ("action",), + ) + + +def _tool_search(name: str, tool: Mapping[str, object]) -> ResponsesTool: + return ResponsesTool(name=name, payload=dict(tool), choice=None) + + +def _custom_function(name: str, tool: Mapping[str, object]) -> ResponsesTool: + description: Final = tool.get("description") + input_schema: Final = tool.get("input_schema") + return ResponsesTool( + name=name, + payload={ + "type": "function", + "name": name, + "strict": bool(tool.get("strict")), + **({"description": description} if "description" in tool else {}), + **({"parameters": json_copy_or_original(input_schema)} if "input_schema" in tool else {}), + }, + choice={"type": "function", "name": name}, + ) + + +def is_tool_search(tool: Mapping[str, object]) -> bool: + tool_type: Final = tool.get("type") + return isinstance(tool_type, str) and tool_type in ANTHROPIC_TOOL_SEARCH_TOOL_TYPES + + +def forced_tool_name(tool_choice: object) -> str | None: + match tool_choice: + case {"type": "tool", "name": str() as name}: + return name + case _: + return None + + +def _defer_loading(tool: Mapping[str, object], forced: str | None) -> object: + if forced is not None and tool.get("name") == forced: + return None + if "defer_loading" in tool: + return tool["defer_loading"] + match tool.get("custom"): + case {"defer_loading": bool() as wrapped}: + return wrapped + case _: + return None + + +def _defers_a_function(tool: Mapping[str, object], forced: str | None) -> bool: + translated: Final = _translated_tool(tool, False) if _defer_loading(tool, forced) is True else None + return isinstance(translated, ResponsesTool) and translated.payload.get("type") == "function" + + +def keeps_tool_search( + tools: Sequence[Mapping[str, object]], *, maps_tool_search: bool, forced: str | None = None +) -> bool: + return ( + maps_tool_search + and any(is_tool_search(tool) for tool in tools) + and any(_defers_a_function(tool, forced) for tool in tools) + ) + + +def _with_deferral(translated: TranslatedTool, defer_loading: object) -> TranslatedTool: + match translated: + case ResponsesTool(payload=payload) if payload.get("type") == "function" and isinstance(defer_loading, bool): + return replace(translated, payload={**payload, "defer_loading": defer_loading}) + case _: + return translated + + +def responses_tool( + tool: Mapping[str, object], *, runs_tool_search: bool = False, forced: str | None = None +) -> TranslatedTool: + translated: Final = _translated_tool(tool, runs_tool_search) + return _with_deferral(translated, _defer_loading(tool, forced)) if runs_tool_search else translated + + +def _translated_tool(tool: Mapping[str, object], runs_tool_search: bool) -> TranslatedTool: + tool_type: Final = tool.get("type") + tool_name: Final = tool.get("name") + name: Final = tool_name if isinstance(tool_name, str) else "" + type_name: Final = tool_type if isinstance(tool_type, str) else "" + family, version = _family_and_version(type_name) + if type_name.startswith("web_search") or name == "web_search": + return _hosted(name, {"type": "web_search_preview"}) + if runs_tool_search and is_tool_search(tool): + return _tool_search(name, tool) + if not version and family not in _UNVERSIONED_SERVER_TOOL_TYPES: + return _custom_function(name, tool) + match family: + case "bash": + return _bash_tool(name) + case "text_editor": + return _text_editor_tool(name, version) + case "memory": + return _memory_tool(name) + case "computer": + return _computer_tool(name, version, tool) + case _: + return UntranslatableTool(name=name, type=type_name) + + +_DROP_PARAMS_HOW_TO: Final = ( + "set `drop_params: true` in the model's `litellm_params` or under `litellm_settings` on the proxy, or " + "`litellm.drop_params=True` in the SDK" +) + + +def _tool_types(tools: Sequence[UntranslatableTool]) -> str: + return ", ".join(dict.fromkeys(tool.type for tool in tools)) + + +def untranslatable_tools_error(tools: Sequence[UntranslatableTool]) -> UnsupportedParamsError: + return UnsupportedParamsError( + message=( + f"LiteLLM does not translate these Anthropic tools to the Responses API: {_tool_types(tools)}. " + f"To drop them, {_DROP_PARAMS_HOW_TO}" + ) + ) + + +def log_dropped_tools(tools: Sequence[UntranslatableTool]) -> None: + verbose_logger.warning( + "Dropping Anthropic tools the Responses API cannot run under drop_params: %s", _tool_types(tools) + ) + + +def unforceable_tool_choice_error(name: str) -> UnsupportedParamsError: + return UnsupportedParamsError( + message=f"OpenAI cannot force the tool search tool {name} through `tool_choice`. To drop it, {_DROP_PARAMS_HOW_TO}" + ) + + +def _drops_deferral(additional_drop_params: object) -> bool: + try: + paths: Final = _DROP_PATHS.validate_python(additional_drop_params) + except ValidationError: + return False + return any(isinstance(path, str) and _DEFERRAL_DROP_PATH.fullmatch(path) is not None for path in paths) + + +def maps_openai_tool_search(model: str, api_base: object, additional_drop_params: object) -> bool: + return ( + targets_openai_api(api_base) + and not _drops_deferral(additional_drop_params) + and litellm.supports_tool_search(model=model, custom_llm_provider=LlmProviders.OPENAI.value) + ) diff --git a/litellm/llms/anthropic/messages/responses_adapters/transformation.py b/litellm/llms/anthropic/messages/responses_adapters/transformation.py index 0630bef0311..7fce12c337f 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/messages/responses_adapters/transformation.py @@ -26,9 +26,19 @@ from litellm.llms.anthropic.messages.native.utils import ( refusal_stop_details, responses_output_refusal_text, ) +from litellm.llms.anthropic.messages.responses_adapters.tool_translation import ( + ResponsesTool, + TranslatedTool, + UntranslatableTool, + forced_tool_name, + keeps_tool_search, + log_dropped_tools, + responses_tool, + unforceable_tool_choice_error, + untranslatable_tools_error, +) from litellm.llms.anthropic.messages.utils import ( is_reasoning_auto_summary_enabled, - json_copy_or_original, prompt_cache_key_from_user_id, ) from litellm.types.llms.anthropic import ( @@ -457,45 +467,63 @@ class LiteLLMAnthropicToResponsesAPIAdapter: return input_items @staticmethod - def _translate_tool_to_responses_api(tool_dict: Mapping[str, object]) -> dict[str, object]: - tool_type: Final = tool_dict.get("type", "") - tool_name: Final = tool_dict.get("name", "") - # web_search tool - if (isinstance(tool_type, str) and tool_type.startswith("web_search")) or tool_name == "web_search": - return {"type": "web_search_preview"} - # Responses turns strict mode on when `strict` is omitted, silently rewriting - # `required` to every property. Anthropic tools are non-strict unless asked. - func_tool: Final[dict[str, object]] = { - "type": "function", - "name": tool_name, - "strict": bool(tool_dict.get("strict")), - } - if "description" in tool_dict: - func_tool["description"] = tool_dict["description"] - if "input_schema" in tool_dict: - func_tool["parameters"] = json_copy_or_original(tool_dict["input_schema"]) - return func_tool + def translated_tools( + tools: Sequence[AllAnthropicToolsValues], + *, + drop_params: bool = True, + maps_tool_search: bool = False, + forced: str | None = None, + ) -> tuple[TranslatedTool, ...]: + runs_tool_search: Final = keeps_tool_search(tools, maps_tool_search=maps_tool_search, forced=forced) + translated: Final = tuple( + responses_tool(tool, runs_tool_search=runs_tool_search, forced=forced) for tool in tools + ) + untranslatable: Final = tuple(tool for tool in translated if isinstance(tool, UntranslatableTool)) + if untranslatable and not drop_params: + raise untranslatable_tools_error(untranslatable) + if untranslatable: + log_dropped_tools(untranslatable) + return translated def translate_tools_to_responses_api( self, tools: list[AllAnthropicToolsValues], + *, + drop_params: bool = True, + maps_tool_search: bool = False, ) -> list[dict[str, object]]: - """Convert Anthropic tool definitions to Responses API function tools.""" - return [self._translate_tool_to_responses_api(tool) for tool in tools] + """Convert Anthropic tool definitions to Responses API tools.""" + return [ + dict(tool.payload) + for tool in self.translated_tools(tools, drop_params=drop_params, maps_tool_search=maps_tool_search) + if isinstance(tool, ResponsesTool) + ] @staticmethod def translate_tool_choice_to_responses_api( tool_choice: AnthropicMessagesToolChoice, - ) -> str | dict[str, object]: - """Convert Anthropic tool_choice to Responses API tool_choice.""" + tools: Sequence[TranslatedTool] = (), + *, + drop_params: bool = True, + ) -> str | dict[str, object] | None: + """Map Anthropic tool_choice to Responses API tool_choice, or None when the named tool cannot be forced.""" tc_type: Final = tool_choice.get("type") if tc_type == "any": return "required" - elif tc_type == "tool": - return {"type": "function", "name": tool_choice.get("name", "")} - elif tc_type == "none": + if tc_type == "none": return "none" - return "auto" + if tc_type != "tool": + return "auto" + name: Final = tool_choice.get("name", "") + match next((tool for tool in tools if tool.name == name), None): + case ResponsesTool(choice=choice) if choice is not None: + return dict(choice) + case ResponsesTool(name=forced) if not drop_params: + raise unforceable_tool_choice_error(forced) + case ResponsesTool() | UntranslatableTool(): + return None + case None: + return {"type": "function", "name": name} @staticmethod def translate_context_management_to_responses_api( @@ -587,6 +615,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: auto_summary_level: str = "detailed", *, drop_params: bool = True, + maps_tool_search: bool = False, ) -> dict[str, Any]: """ Translate a full Anthropic /v1/messages request dict to @@ -596,7 +625,8 @@ class LiteLLMAnthropicToResponsesAPIAdapter: on every call, so a reasoning model's items can be replayed intact next turn even when the client sent no ``thinking`` block; pass False for a provider whose Responses API rejects ``include``. ``auto_summary_level`` is the ``reasoning.summary`` the - opt-in auto summary injects when the client sent none. + opt-in auto summary injects when the client sent none. ``maps_tool_search`` passes Anthropic tool search + through for a target that runs OpenAI's tool search, and it is otherwise an unsupported tool. """ model: Final[str] = anthropic_request["model"] messages_list: Final = cast( @@ -646,19 +676,24 @@ class LiteLLMAnthropicToResponsesAPIAdapter: if "top_p" in anthropic_request: responses_kwargs["top_p"] = anthropic_request["top_p"] - # tools - tools: Final = anthropic_request.get("tools") - if tools: - responses_kwargs["tools"] = self.translate_tools_to_responses_api( - cast(list[AllAnthropicToolsValues], tools) - ) + tool_choice: Final = cast(AnthropicMessagesToolChoice | None, anthropic_request.get("tool_choice")) + translated_tools: Final = self.translated_tools( + cast(list[AllAnthropicToolsValues], anthropic_request.get("tools") or []), + drop_params=drop_params, + maps_tool_search=maps_tool_search, + forced=forced_tool_name(tool_choice), + ) + responses_tools: Final = [dict(tool.payload) for tool in translated_tools if isinstance(tool, ResponsesTool)] + if responses_tools: + responses_kwargs["tools"] = responses_tools - # tool_choice - tool_choice: Final = anthropic_request.get("tool_choice") - if tool_choice: - responses_kwargs["tool_choice"] = self.translate_tool_choice_to_responses_api( - cast(AnthropicMessagesToolChoice, tool_choice) - ) + responses_tool_choice: Final = ( + self.translate_tool_choice_to_responses_api(tool_choice, translated_tools, drop_params=drop_params) + if tool_choice and (responses_tools or not translated_tools) + else None + ) + if responses_tool_choice is not None: + responses_kwargs["tool_choice"] = responses_tool_choice # thinking -> reasoning thinking: Final = anthropic_request.get("thinking") diff --git a/litellm/llms/openai/responses/count_tokens/token_counter.py b/litellm/llms/openai/responses/count_tokens/token_counter.py index 87a75dae95e..16cb4712997 100644 --- a/litellm/llms/openai/responses/count_tokens/token_counter.py +++ b/litellm/llms/openai/responses/count_tokens/token_counter.py @@ -18,6 +18,7 @@ from litellm.litellm_core_utils.tool_fields import DropPolicy from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError from litellm.llms.anthropic.messages.convertible_blocks import deployment_drops_params, unconvertible_blocks_error from litellm.llms.anthropic.messages.responses_adapters.handler import count_view_messages +from litellm.llms.anthropic.messages.responses_adapters.tool_translation import maps_openai_tool_search from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) @@ -63,6 +64,14 @@ class _CountCredential: credential: str | OpenAIWorkloadIdentityConfig +def _deployment_drop_paths(deployment: Mapping[str, object] | None) -> object: + match deployment: + case {"litellm_params": {"additional_drop_params": object() as additional_drop_params}}: + return additional_drop_params + case _: + return None + + def _count_credential(deployment: Mapping[str, object] | None, model_label: str) -> _CountCredential | OpenAIError: litellm_params: Final = as_str_mapping(None if deployment is None else deployment.get("litellm_params")) or {} api_base: Final = litellm_params.get("api_base") @@ -146,11 +155,18 @@ class OpenAITokenCounter(BaseTokenCounter): system: object, messages_format: TokenCountMessagesFormat, output_format: JsonValue, + api_base: str | None, + additional_drop_params: object, ) -> ResponsesCountRequestParts: match messages_format: case "anthropic": return OpenAICountTokensConfig.anthropic_messages_to_responses_input( - model=model_to_use, messages=messages, system=system, tools=tools, output_format=output_format + model=model_to_use, + messages=messages, + system=system, + tools=tools, + output_format=output_format, + maps_tool_search=maps_openai_tool_search(model_to_use, api_base, additional_drop_params), ) case "openai": input_items, instructions = OpenAICountTokensConfig.messages_to_responses_input( @@ -229,6 +245,8 @@ class OpenAITokenCounter(BaseTokenCounter): system=system, messages_format=messages_format, output_format=output_format, + api_base=resolved.api_base, + additional_drop_params=_deployment_drop_paths(deployment), ) # If no input items were produced (e.g., system-only messages), fall back to local counting diff --git a/litellm/llms/openai/responses/count_tokens/transformation.py b/litellm/llms/openai/responses/count_tokens/transformation.py index a45a0448326..f723245a096 100644 --- a/litellm/llms/openai/responses/count_tokens/transformation.py +++ b/litellm/llms/openai/responses/count_tokens/transformation.py @@ -20,6 +20,7 @@ from litellm.llms.anthropic.messages.native.utils import normalize_native_messag from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) +from litellm.responses.litellm_completion_transformation.tool_search import tools_with_anthropic_tool_search_mapped from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.anthropic import AnthropicMessagesRequest, AnthropicOutputSchema from litellm.types.llms.openai import ResponsesAPIRequestParams, replayable_responses_input @@ -343,6 +344,7 @@ class OpenAICountTokensConfig: system: object, tools: Sequence[Mapping[str, object]] | None, output_format: JsonValue = None, + maps_tool_search: bool = False, ) -> ResponsesCountRequestParts: """Convert with the adapter /v1/messages inference uses, so the count sees the same input it bills.""" anthropic_request: Final = AnthropicMessagesRequest( @@ -354,15 +356,19 @@ class OpenAICountTokensConfig: ) translated: Final = _TRANSLATED_REQUEST.validate_python( LiteLLMAnthropicToResponsesAPIAdapter().translate_request( - anthropic_request, include_encrypted_reasoning=False + anthropic_request, include_encrypted_reasoning=False, maps_tool_search=maps_tool_search ) ) - return OpenAICountTokensConfig.count_request_parts( + input_items, instructions, translated_tools, text = OpenAICountTokensConfig.count_request_parts( ResponsesAPIRequestUtils.restore_encrypted_content_item_ids_in_input(translated.get("input")), translated.get("instructions"), translated.get("tools"), translated.get("text"), ) + counted_tools: Final = ( + tools_with_anthropic_tool_search_mapped(translated_tools) if maps_tool_search else translated_tools + ) + return input_items, instructions, None if counted_tools is None else tuple(counted_tools), text @staticmethod def count_request_parts( diff --git a/litellm/responses/litellm_completion_transformation/tool_search.py b/litellm/responses/litellm_completion_transformation/tool_search.py index 5a1b27f9092..c84da723c8f 100644 --- a/litellm/responses/litellm_completion_transformation/tool_search.py +++ b/litellm/responses/litellm_completion_transformation/tool_search.py @@ -1,13 +1,13 @@ from collections.abc import Iterable, Iterator, Mapping, Sequence, Set from dataclasses import dataclass -from typing import TYPE_CHECKING, Final, Literal, TypeAlias +from typing import TYPE_CHECKING, Final, Literal, TypeAlias, TypeVar from pydantic import ConfigDict, TypeAdapter, ValidationError from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import is_anthropic_tool_search_use from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_TOOL_TYPES, AnthropicToolSearchToolRegex from litellm.types.llms.base import LiteLLMBaseModel -from litellm.types.llms.openai import ALL_RESPONSES_API_TOOL_PARAMS, OpenAIToolSearchToolParam, ResponseInputParam +from litellm.types.llms.openai import OpenAIToolSearchToolParam, ResponseInputParam from litellm.types.responses.main import OutputToolSearchCall from litellm.types.utils import LlmProviders, Message @@ -57,6 +57,7 @@ CLIENT_TOOL_SEARCH_FUNCTION_NAME: Final = "tool_search" ToolSearchCallStatus: TypeAlias = Literal["in_progress", "completed", "incomplete"] ToolSearchExecution: TypeAlias = Literal["server", "client"] +_Tool: Final = TypeVar("_Tool", bound=Mapping[str, object]) _ARGUMENTS_ADAPTER: Final = TypeAdapter(dict[str, object]) _MESSAGE_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) @@ -138,8 +139,8 @@ def is_openai_tool_search_tool(tool: object) -> bool: def tools_with_anthropic_tool_search_mapped( - tools: Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None, -) -> Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None: + tools: Sequence[_Tool] | None, +) -> Sequence[_Tool | OpenAIToolSearchToolParam] | None: if tools is None or not any(is_anthropic_tool_search_chat_tool(tool) for tool in tools): return tools if any(is_openai_tool_search_tool(tool) for tool in tools): @@ -156,8 +157,8 @@ def tools_with_anthropic_tool_search_mapped( def tools_without_anthropic_tool_search( - tools: Sequence[ALL_RESPONSES_API_TOOL_PARAMS], -) -> tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]: + tools: Sequence[_Tool], +) -> tuple[_Tool, ...]: return tuple(tool for tool in tools if not is_anthropic_tool_search_chat_tool(tool)) diff --git a/tests/integration/providers/test_openai_responses_tool_search_wire.py b/tests/integration/providers/test_openai_responses_tool_search_wire.py index 39a955be13b..59febb3f713 100644 --- a/tests/integration/providers/test_openai_responses_tool_search_wire.py +++ b/tests/integration/providers/test_openai_responses_tool_search_wire.py @@ -629,7 +629,7 @@ def test_chat_completions_through_the_responses_bridge_follow_drop_params( assert [tuple(tool.get(key) for key in wire_keys) for tool in tools] == forwarded, tools -def test_messages_to_an_openai_model_with_the_search_tool_still_answers( +def test_messages_to_an_openai_model_runs_the_search_tool_on_openai_with_the_tool_still_deferred( openai_gateway: Gateway, openai_api: ControlPlane ) -> None: exchange: Final = _exchange( @@ -654,7 +654,12 @@ def test_messages_to_an_openai_model_with_the_search_tool_still_answers( ) assert exchange.response.status_code == 200, exchange.response.text - assert len(exchange.requests) == 1, exchange.requests + tools: Final = _ITEMS.validate_python(exchange.only_body()["tools"]) + wire_keys: Final = ("type", "execution", "name", "defer_loading") + assert [tuple(tool.get(key) for key in wire_keys) for tool in tools] == [ + ("tool_search", "server", None, None), + ("function", None, "get_weather", True), + ], tools def test_a_global_drop_params_drops_the_search_for_an_openai_model_that_cannot_run_it( diff --git a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index c2ee58b67ed..12319e72ab8 100644 --- a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -998,6 +998,34 @@ class TestAnthropicMessagesHandlerInputProcessing: assert {"googleMaps": {"enable_widget": True}} in data["tools"] assert [tool["name"] for tool in data["tools"] if "name" in tool] == ["get_weather"] + @pytest.mark.asyncio + async def test_a_deferred_tool_keeps_its_defer_loading_through_a_guardrail_that_returns_the_tools(self): + handler = AnthropicMessagesHandler() + guardrail = MockPassThroughGuardrail(guardrail_name="test") + data = { + "model": "gpt-5.6-luna", + "messages": [{"role": "user", "content": "what is AAPL at?"}], + "tools": [ + {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"}, + {"name": "get_stock", "input_schema": {"type": "object", "properties": {}}, "defer_loading": True}, + {"name": "get_weather", "input_schema": {"type": "object", "properties": {}}, "defer_loading": False}, + {"name": "get_time", "input_schema": {"type": "object", "properties": {}}}, + {"name": "get_quote", "input_schema": {"type": "object", "properties": {}}, "custom": {"defer_loading": True}}, + {"name": "get_news", "input_schema": {"type": "object", "properties": {}}, "defer_loading": "true"}, + ], + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert [(tool["name"], tool.get("defer_loading"), tool.get("custom")) for tool in data["tools"]] == [ + ("tool_search_tool_regex", None, None), + ("get_stock", True, None), + ("get_weather", False, None), + ("get_time", None, None), + ("get_quote", None, {"defer_loading": True}), + ("get_news", None, None), + ] + @pytest.mark.asyncio async def test_midturn_system_correction_is_guardrailed_when_top_level_system_is_skipped( self, diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py index 382917fc756..10ac27a16a5 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py @@ -2,7 +2,7 @@ import datetime import json import os import sys -from collections.abc import AsyncIterator, Mapping +from collections.abc import AsyncIterator, Mapping, Sequence from itertools import chain from typing import Final, cast from unittest.mock import AsyncMock, patch @@ -764,6 +764,338 @@ async def test_recomputed_cost_of_a_served_bridge_response_prices_the_reasoning_ assert logging_obj.cost_breakdown["reasoning_cost"] == pytest.approx(expected_reasoning) +TOOL_SEARCH: Final = {"type": "tool_search_tool_bm25_20251119", "name": "tool_search_tool_bm25"} +DEFERRED_WEATHER: Final = { + "name": "get_weather", + "description": "Weather for a city", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + "defer_loading": True, +} +WEB_FETCH: Final = {"type": "web_fetch_20250910", "name": "web_fetch"} +TYPED_BASH: Final = {"type": "bash_20250124", "name": "bash"} +OPENAI_API_BASE: Final = "https://api.openai.com/v1" +OTHER_RESPONSES_API_BASE: Final = "https://responses.example.test/v1" + + +async def _openai_request_for( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + tools: list[Mapping[str, object]], + api_base: str = OPENAI_API_BASE, + model: str = "openai/gpt-5.6-luna", + **extra: object, +) -> respx.Route: + monkeypatch.setenv("OPENAI_API_KEY", "sk-typed-tools-test") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post(f"{api_base}/responses").respond(status_code=200, json=TOOL_CALL_RESPONSE) + await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=1024, + messages=MESSAGES, + model=model, + tools=[dict(tool) for tool in tools], + custom_llm_provider="openai", + api_base=api_base, + **extra, + ) + return upstream + + +def _tools_sent(upstream: respx.Route) -> list[Mapping[str, object]]: + return json.loads(upstream.calls.last.request.content)["tools"] + + +@pytest.mark.asyncio +async def test_anthropic_tool_search_reaches_openai_as_its_server_tool_search_with_the_tool_still_deferred( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + upstream: Final = await _openai_request_for(respx_mock, monkeypatch, [TOOL_SEARCH, DEFERRED_WEATHER]) + + wire_keys: Final = ("type", "execution", "name", "defer_loading") + assert [tuple(tool.get(key) for key in wire_keys) for tool in _tools_sent(upstream)] == [ + ("tool_search", "server", None, None), + ("function", None, "get_weather", True), + ] + + +@pytest.mark.asyncio +async def test_a_deferred_typed_tool_keeps_the_tool_search_under_drop_params( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + upstream: Final = await _openai_request_for( + respx_mock, monkeypatch, [TOOL_SEARCH, {**TYPED_BASH, "defer_loading": True}], drop_params=True + ) + + assert [(tool["type"], tool.get("defer_loading")) for tool in _tools_sent(upstream)] == [ + ("tool_search", None), + ("function", True), + ] + + +@pytest.mark.asyncio +async def test_tool_search_for_a_responses_server_other_than_openai_answers_400_without_drop_params( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + with pytest.raises(litellm.UnsupportedParamsError) as error: + await _openai_request_for( + respx_mock, monkeypatch, [TOOL_SEARCH, DEFERRED_WEATHER], api_base=OTHER_RESPONSES_API_BASE, drop_params=False + ) + + assert error.value.status_code == 400 + assert not respx_mock.calls + + +@pytest.mark.asyncio +async def test_drop_params_drops_tool_search_and_its_deferral_for_a_responses_server_other_than_openai( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + upstream: Final = await _openai_request_for( + respx_mock, monkeypatch, [TOOL_SEARCH, DEFERRED_WEATHER], api_base=OTHER_RESPONSES_API_BASE, drop_params=True + ) + + assert [(tool["type"], tool["name"], tool.get("defer_loading")) for tool in _tools_sent(upstream)] == [ + ("function", "get_weather", None) + ] + + +MODEL_WITHOUT_TOOL_SEARCH: Final = "openai/a-model-the-cost-map-does-not-mark-for-tool-search" + + +@pytest.mark.asyncio +async def test_tool_search_on_an_openai_model_that_cannot_run_it_answers_400_naming_drop_params( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + with pytest.raises(litellm.UnsupportedParamsError) as error: + await _openai_request_for( + respx_mock, monkeypatch, [TOOL_SEARCH, DEFERRED_WEATHER], model=MODEL_WITHOUT_TOOL_SEARCH, drop_params=False + ) + + assert error.value.status_code == 400 + assert "tool_search_tool_bm25_20251119" in error.value.message + assert "drop_params" in error.value.message + assert not respx_mock.calls + + +@pytest.mark.asyncio +async def test_drop_params_drops_tool_search_and_its_deferral_on_an_openai_model_that_cannot_run_it( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + upstream: Final = await _openai_request_for( + respx_mock, monkeypatch, [TOOL_SEARCH, DEFERRED_WEATHER], model=MODEL_WITHOUT_TOOL_SEARCH, drop_params=True + ) + + assert [(tool["type"], tool["name"], tool.get("defer_loading")) for tool in _tools_sent(upstream)] == [ + ("function", "get_weather", None) + ] + + +UNDEFERRED_WEATHER: Final = {key: value for key, value in DEFERRED_WEATHER.items() if key != "defer_loading"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "weather", + [ + pytest.param(UNDEFERRED_WEATHER, id="no-defer-loading"), + pytest.param({**DEFERRED_WEATHER, "defer_loading": "true"}, id="defer-loading-string"), + ], +) +async def test_tool_search_with_no_deferred_tool_answers_400_naming_drop_params_before_any_openai_call( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, weather: Mapping[str, object] +): + with pytest.raises(litellm.UnsupportedParamsError) as error: + await _openai_request_for(respx_mock, monkeypatch, [TOOL_SEARCH, weather], drop_params=False) + + assert error.value.status_code == 400 + assert "tool_search_tool_bm25_20251119" in error.value.message + assert "drop_params" in error.value.message + assert not respx_mock.calls + + +@pytest.mark.asyncio +@pytest.mark.parametrize("tool_type", [pytest.param(["bash"], id="list"), pytest.param({"name": "bash"}, id="object")]) +async def test_a_tool_whose_type_is_not_a_string_reaches_openai_as_a_function_beside_tool_search( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, tool_type: object +): + upstream: Final = await _openai_request_for( + respx_mock, monkeypatch, [TOOL_SEARCH, DEFERRED_WEATHER, {"type": tool_type, "name": "odd_tool"}] + ) + + assert [(tool["type"], tool.get("name")) for tool in _tools_sent(upstream)] == [ + ("tool_search", None), + ("function", "get_weather"), + ("function", "odd_tool"), + ] + + +@pytest.mark.asyncio +async def test_additional_drop_params_tools_sends_no_tools_even_beside_one_openai_cannot_run( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + upstream: Final = await _openai_request_for( + respx_mock, monkeypatch, [WEB_FETCH, TYPED_BASH], drop_params=False, additional_drop_params=["tools"] + ) + + assert "tools" not in json.loads(upstream.calls.last.request.content) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "tool_choice", + [{"type": "tool", "name": "web_fetch"}, {"type": "tool", "name": "bash"}, {"type": "any"}], + ids=["forcing-a-tool-openai-cannot-run", "forcing-a-client-tool", "forcing-any-tool"], +) +async def test_additional_drop_params_tools_sends_no_tool_choice_forcing_a_dropped_tool( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, tool_choice: Mapping[str, object] +): + upstream: Final = await _openai_request_for( + respx_mock, + monkeypatch, + [WEB_FETCH, TYPED_BASH], + drop_params=True, + additional_drop_params=["tools"], + tool_choice=dict(tool_choice), + ) + + sent: Final = json.loads(upstream.calls.last.request.content) + assert "tools" not in sent + assert "tool_choice" not in sent + + +DEFERRED_STOCK: Final = { + "name": "get_stock", + "description": "Price for a ticker", + "input_schema": {"type": "object", "properties": {"ticker": {"type": "string"}}, "required": ["ticker"]}, + "defer_loading": True, +} +FORCE_STOCK: Final = {"type": "tool", "name": "get_stock"} + + +def _tool_search_wire(upstream: respx.Route) -> list[tuple[object, ...]]: + return [(tool["type"], tool.get("name"), tool.get("defer_loading")) for tool in _tools_sent(upstream)] + + +@pytest.mark.asyncio +async def test_a_tool_choice_forcing_a_deferred_function_sends_that_function_loaded_beside_tool_search( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + upstream: Final = await _openai_request_for( + respx_mock, + monkeypatch, + [TOOL_SEARCH, DEFERRED_STOCK, DEFERRED_WEATHER], + drop_params=False, + tool_choice=dict(FORCE_STOCK), + ) + + assert _tool_search_wire(upstream) == [ + ("tool_search", None, None), + ("function", "get_stock", None), + ("function", "get_weather", True), + ] + assert json.loads(upstream.calls.last.request.content)["tool_choice"] == {"type": "function", "name": "get_stock"} + + +@pytest.mark.asyncio +async def test_drop_params_drops_tool_search_when_the_forced_function_is_the_only_deferred_one( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + upstream: Final = await _openai_request_for( + respx_mock, monkeypatch, [TOOL_SEARCH, DEFERRED_STOCK], drop_params=True, tool_choice=dict(FORCE_STOCK) + ) + + assert _tool_search_wire(upstream) == [("function", "get_stock", None)] + assert json.loads(upstream.calls.last.request.content)["tool_choice"] == {"type": "function", "name": "get_stock"} + + +@pytest.mark.asyncio +async def test_additional_drop_params_tool_choice_sends_tool_search_without_the_choice_that_forces_it( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + upstream: Final = await _openai_request_for( + respx_mock, + monkeypatch, + [TOOL_SEARCH, DEFERRED_WEATHER], + drop_params=False, + additional_drop_params=["tool_choice"], + tool_choice={"type": "tool", "name": "tool_search_tool_bm25"}, + ) + + assert "tool_choice" not in json.loads(upstream.calls.last.request.content) + assert _tool_search_wire(upstream) == [("tool_search", None, None), ("function", "get_weather", True)] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "additional_drop_params", + [ + pytest.param(["tools[*].defer_loading"], id="every-tool"), + pytest.param(["tools[1].defer_loading"], id="one-tool-by-index"), + pytest.param(("tools[*].defer_loading",), id="sdk-tuple"), + ], +) +async def test_additional_drop_params_of_a_defer_loading_path_answers_400_for_tool_search_without_drop_params( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, additional_drop_params: Sequence[str] +): + with pytest.raises(litellm.UnsupportedParamsError) as error: + await _openai_request_for( + respx_mock, + monkeypatch, + [TOOL_SEARCH, DEFERRED_WEATHER], + drop_params=False, + additional_drop_params=additional_drop_params, + ) + + assert "tool_search_tool_bm25_20251119" in error.value.message + assert not respx_mock.calls + + +@pytest.mark.asyncio +async def test_a_deferral_inside_the_custom_object_keeps_tool_search_with_the_tool_deferred( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + wrapped: Final = {**UNDEFERRED_WEATHER, "custom": {"defer_loading": True}} + + upstream: Final = await _openai_request_for(respx_mock, monkeypatch, [TOOL_SEARCH, wrapped], drop_params=False) + + assert _tool_search_wire(upstream) == [("tool_search", None, None), ("function", "get_weather", True)] + assert all("custom" not in tool for tool in _tools_sent(upstream)) + + +@pytest.mark.asyncio +async def test_an_unversioned_typed_tool_name_reaches_openai_with_the_clients_own_schema( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + schema: Final = {"type": "object", "properties": {"script": {"type": "string"}}, "required": ["script"]} + + upstream: Final = await _openai_request_for( + respx_mock, monkeypatch, [{"type": "bash", "name": "bash", "input_schema": schema}], drop_params=False + ) + + assert [(tool["name"], tool["parameters"]) for tool in _tools_sent(upstream)] == [("bash", schema)] + + +@pytest.mark.asyncio +async def test_a_tool_openai_cannot_run_answers_400_before_any_openai_call_without_drop_params( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + with pytest.raises(litellm.UnsupportedParamsError) as error: + await _openai_request_for(respx_mock, monkeypatch, [WEB_FETCH, TYPED_BASH], drop_params=False) + + assert error.value.status_code == 400 + assert not respx_mock.calls + + +@pytest.mark.asyncio +async def test_drop_params_drops_a_tool_openai_cannot_run_and_keeps_the_rest( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + upstream: Final = await _openai_request_for(respx_mock, monkeypatch, [WEB_FETCH, TYPED_BASH], drop_params=True) + + tools: Final = _tools_sent(upstream) + assert [tool["name"] for tool in tools] == ["bash"] + assert "command" in cast(Mapping[str, Mapping[str, object]], tools[0]["parameters"])["properties"] + + TEXT_RESPONSE: Final = { "id": "resp_cleared", "object": "response", diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py index ece0cf5d423..1bf0359b401 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py @@ -17,6 +17,7 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, ) +from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, @@ -949,6 +950,220 @@ class TestTranslateToolChoiceToResponsesAPI: assert result == {"type": "function", "name": "get_weather"} +_TOOL_SEARCH: Final = {"type": "tool_search_tool_bm25_20251119", "name": "tool_search_tool_bm25"} +_DEFERRED_WEATHER: Final = { + "name": "get_weather", + "description": "Weather for a city", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + "defer_loading": True, +} +_DEFERRED_BASH: Final = {"type": "bash_20250124", "name": "bash", "defer_loading": True} + + +class TestAnthropicTypedToolsOnResponses: + """Anthropic-defined tools carry no input_schema, so the adapter has to supply the schema Claude has built in.""" + + @staticmethod + def _only_tool(tool: Dict[str, Any]) -> Dict[str, Any]: + tools: Final = _ADAPTER.translate_request(_make_request(tools=[tool]), drop_params=False)["tools"] + assert len(tools) == 1 + return tools[0] + + @staticmethod + def _enum(tool: Dict[str, Any], prop: str) -> List[str]: + return tool["parameters"]["properties"][prop]["enum"] + + def test_bash_gets_a_command_parameter(self): + tool: Final = self._only_tool({"type": "bash_20250124", "name": "bash"}) + + assert tool["type"] == "function" + assert tool["name"] == "bash" + assert tool["strict"] is False + assert set(tool["parameters"]["properties"]) == {"command", "restart"} + assert tool["description"] + + def test_text_editor_from_20250429_inserts_insert_text_and_has_no_undo(self): + tool: Final = self._only_tool({"type": "text_editor_20250728", "name": "str_replace_based_edit_tool"}) + + assert tool["name"] == "str_replace_based_edit_tool" + assert self._enum(tool, "command") == ["view", "create", "str_replace", "insert"] + assert "insert_text" in tool["parameters"]["properties"] + assert tool["parameters"]["required"] == ["command", "path"] + + def test_text_editor_before_20250429_inserts_new_str_and_can_undo(self): + tool: Final = self._only_tool({"type": "text_editor_20250124", "name": "str_replace_editor"}) + + assert "undo_edit" in self._enum(tool, "command") + assert "insert_text" not in tool["parameters"]["properties"] + assert "new_str" in tool["parameters"]["properties"] + + def test_memory_gets_every_memory_command(self): + tool: Final = self._only_tool({"type": "memory_20250818", "name": "memory"}) + + assert set(self._enum(tool, "command")) == {"view", "create", "str_replace", "insert", "delete", "rename"} + assert {"old_path", "new_path"} <= set(tool["parameters"]["properties"]) + + @pytest.mark.parametrize( + ("tool_type", "enable_zoom", "has_scroll", "has_zoom"), + [ + ("computer_20241022", True, False, False), + ("computer_20250124", True, True, False), + ("computer_20251124", False, True, False), + ("computer_20251124", True, True, True), + ], + ) + def test_computer_actions_follow_the_tool_version( + self, tool_type: str, enable_zoom: bool, has_scroll: bool, has_zoom: bool + ): + tool: Final = self._only_tool( + { + "type": tool_type, + "name": "computer", + "display_width_px": 1280, + "display_height_px": 800, + "enable_zoom": enable_zoom, + } + ) + actions: Final = self._enum(tool, "action") + properties: Final = tool["parameters"]["properties"] + + assert {"screenshot", "left_click", "type", "key"} <= set(actions) + assert ("scroll" in actions) is has_scroll + assert ("scroll_direction" in properties) is has_scroll + assert ("zoom" in actions) is has_zoom + assert ("region" in properties) is has_zoom + assert "1280x800" in tool["description"] + + @pytest.mark.parametrize( + "tool_type", ["web_search_20250305", "web_search_preview", "web_search_preview_2025_03_11", "web_search_2025_08_26"] + ) + def test_every_web_search_type_runs_the_hosted_web_search(self, tool_type: str): + assert self._only_tool({"type": tool_type, "name": "web_search"}) == {"type": "web_search_preview"} + + def test_custom_tool_named_like_a_typed_tool_keeps_its_own_schema(self): + schema: Final = {"type": "object", "properties": {"cmd": {"type": "string"}}, "required": ["cmd"]} + + tool: Final = self._only_tool({"name": "bash", "input_schema": schema}) + + assert tool["parameters"] == schema + assert "description" not in tool + + @pytest.mark.parametrize( + "tool", + [ + {"type": "web_fetch_20250910", "name": "web_fetch"}, + {"type": "code_execution_20250825", "name": "code_execution"}, + {"type": "mcp_toolset", "mcp_server_name": "docs"}, + {"type": "computer_toolset_20260801"}, + _TOOL_SEARCH, + ], + ) + def test_tool_with_no_responses_equivalent_is_a_400_without_drop_params(self, tool: Dict[str, Any]): + with pytest.raises(UnsupportedParamsError) as error: + _ADAPTER.translate_request(_make_request(tools=[tool]), drop_params=False) + + assert error.value.status_code == 400 + assert tool["type"] in str(error.value) + assert "litellm_params" in str(error.value) + + def test_tool_with_no_responses_equivalent_drops_under_drop_params(self): + kwargs: Final = _ADAPTER.translate_request( + _make_request( + tools=[{"type": "web_fetch_20250910", "name": "web_fetch"}, {"type": "bash_20250124", "name": "bash"}] + ), + drop_params=True, + ) + + assert [tool["name"] for tool in kwargs["tools"]] == ["bash"] + + def test_tool_choice_of_a_hosted_tool_names_the_hosted_type(self): + kwargs: Final = _ADAPTER.translate_request( + _make_request( + tools=[ + {"type": "bash_20250124", "name": "bash"}, + {"type": "web_search_20250305", "name": "web_search"}, + ], + tool_choice={"type": "tool", "name": "web_search"}, + ) + ) + + assert kwargs["tool_choice"] == {"type": "web_search_preview"} + + def test_tool_choice_of_a_dropped_tool_is_dropped_with_it(self): + kwargs: Final = _ADAPTER.translate_request( + _make_request( + tools=[{"type": "web_fetch_20250910", "name": "web_fetch"}, {"type": "bash_20250124", "name": "bash"}], + tool_choice={"type": "tool", "name": "web_fetch"}, + ), + drop_params=True, + ) + + assert [tool["name"] for tool in kwargs["tools"]] == ["bash"] + assert "tool_choice" not in kwargs + + def test_tool_choice_is_dropped_when_every_tool_was_dropped(self): + kwargs: Final = _ADAPTER.translate_request( + _make_request(tools=[{"type": "web_fetch_20250910", "name": "web_fetch"}], tool_choice={"type": "any"}), + drop_params=True, + ) + + assert "tools" not in kwargs + assert "tool_choice" not in kwargs + + def test_tool_choice_forcing_the_tool_search_is_a_400_without_drop_params(self): + with pytest.raises(UnsupportedParamsError) as error: + _ADAPTER.translate_request( + _make_request( + tools=[_TOOL_SEARCH, _DEFERRED_WEATHER], + tool_choice={"type": "tool", "name": "tool_search_tool_bm25"}, + ), + drop_params=False, + maps_tool_search=True, + ) + + assert error.value.status_code == 400 + assert "tool_search_tool_bm25" in str(error.value) + assert "drop_params" in str(error.value) + + def test_tool_choice_forcing_the_tool_search_is_dropped_under_drop_params(self): + kwargs: Final = _ADAPTER.translate_request( + _make_request( + tools=[_TOOL_SEARCH, _DEFERRED_WEATHER], + tool_choice={"type": "tool", "name": "tool_search_tool_bm25"}, + ), + drop_params=True, + maps_tool_search=True, + ) + + assert [tool["name"] for tool in kwargs["tools"]] == ["tool_search_tool_bm25", "get_weather"] + assert "tool_choice" not in kwargs + + @pytest.mark.parametrize( + ("tools", "maps_tool_search", "deferred"), + [ + ([_TOOL_SEARCH, _DEFERRED_WEATHER], True, True), + ([_TOOL_SEARCH, _DEFERRED_BASH], True, True), + ([_DEFERRED_WEATHER], True, None), + ([_TOOL_SEARCH, _DEFERRED_WEATHER], False, None), + ], + ) + def test_a_tool_stays_deferred_only_beside_a_tool_search_the_target_runs( + self, tools: List[Dict[str, Any]], maps_tool_search: bool, deferred: Any + ): + kwargs: Final = _ADAPTER.translate_request( + _make_request(tools=tools), drop_params=True, maps_tool_search=maps_tool_search + ) + + deferrable: Final = next(tool for tool in kwargs["tools"] if tool.get("type") == "function") + assert deferrable.get("defer_loading") is deferred + assert any(tool["type"] == _TOOL_SEARCH["type"] for tool in kwargs["tools"]) is (deferred is True) + + def test_tool_choice_without_tools_still_reaches_the_provider(self): + kwargs: Final = _ADAPTER.translate_request(_make_request(tool_choice={"type": "any"})) + + assert kwargs["tool_choice"] == "required" + + # --------------------------------------------------------------------------- # translate_thinking_to_reasoning # --------------------------------------------------------------------------- diff --git a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py index 29c25c5fdbf..dc04a71b2b0 100644 --- a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py +++ b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py @@ -196,6 +196,95 @@ async def _sent_count_body( return json.loads(route.calls.last.request.content) +TOOL_SEARCH_TOOLS: Final[list[dict[str, object]]] = [ + {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"}, + { + "name": "get_weather", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}}, + "defer_loading": True, + }, +] +UNDEFERRED_TOOL_SEARCH_TOOLS: Final[list[dict[str, object]]] = [ + TOOL_SEARCH_TOOLS[0], + {key: value for key, value in TOOL_SEARCH_TOOLS[1].items() if key != "defer_loading"}, +] + + +async def _counted_tools( + monkeypatch: pytest.MonkeyPatch, + model: str, + litellm_params: Mapping[str, object], + tools: list[dict[str, object]] = TOOL_SEARCH_TOOLS, +) -> list[tuple[object, object, object]]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock: + route: Final = respx.post(f"{litellm_params.get('api_base', 'https://api.openai.com/v1')}/responses/input_tokens") + route.mock(return_value=httpx.Response(200, json={"input_tokens": 1})) + await OpenAITokenCounter().count_tokens( + model_to_use=model, + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + contents=None, + deployment={"litellm_params": {"api_key": "sk-test", **litellm_params}}, + request_model=f"openai/{model}", + tools=tools, + messages_format="anthropic", + ) + counted: Final = json.loads(route.calls.last.request.content)["tools"] + return [(tool.get("type"), tool.get("name"), tool.get("defer_loading")) for tool in counted] + + +@pytest.mark.asyncio +async def test_anthropic_tool_search_is_counted_as_the_openai_tool_search_inference_sends( + monkeypatch: pytest.MonkeyPatch, +) -> None: + assert await _counted_tools(monkeypatch, "gpt-5.6-luna", {}) == [ + ("tool_search", None, None), + ("function", "get_weather", True), + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("tools", "expected"), + [ + pytest.param( + TOOL_SEARCH_TOOLS, + [("tool_search", None, None), ("function", "get_weather", True)], + id="a-tool-is-deferred", + ), + pytest.param(UNDEFERRED_TOOL_SEARCH_TOOLS, [("function", "get_weather", None)], id="nothing-is-deferred"), + ], +) +@pytest.mark.parametrize( + "litellm_params", + [pytest.param({}, id="without-drop-params"), pytest.param({"drop_params": True}, id="drop-params")], +) +async def test_anthropic_tool_search_is_counted_only_beside_a_deferred_tool( + monkeypatch: pytest.MonkeyPatch, + tools: list[dict[str, object]], + expected: list[tuple[object, object, object]], + litellm_params: Mapping[str, object], +) -> None: + assert await _counted_tools(monkeypatch, "gpt-5.6-luna", litellm_params, tools) == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model", "litellm_params"), + [ + pytest.param("a-model-the-cost-map-does-not-mark-for-tool-search", {}, id="model-without-tool-search"), + pytest.param("gpt-5.6-luna", {"api_base": "https://responses.example.test/v1"}, id="other-responses-server"), + pytest.param( + "gpt-5.6-luna", {"additional_drop_params": ["tools[*].defer_loading"]}, id="deployment-drops-deferral" + ), + ], +) +async def test_anthropic_tool_search_is_left_out_of_the_count_where_inference_cannot_run_it( + monkeypatch: pytest.MonkeyPatch, model: str, litellm_params: Mapping[str, object] +) -> None: + assert await _counted_tools(monkeypatch, model, litellm_params) == [("function", "get_weather", None)] + + @pytest.mark.asyncio async def test_anthropic_format_sends_the_document_image_and_tool_turns_to_the_count_endpoint( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py b/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py index 575183772f9..9f540366d05 100644 --- a/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py @@ -603,6 +603,26 @@ def test_anthropic_messages_keep_every_block_the_chat_converter_drops() -> None: ) +def test_a_count_with_anthropic_tool_search_counts_the_tools_without_the_search() -> None: + _, _, tools, _ = OpenAICountTokensConfig.anthropic_messages_to_responses_input( + model="gpt-5.6-luna", + messages=[{"role": "user", "content": "Find a weather tool."}], + system=None, + tools=[ + {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"}, + { + "name": "get_weather", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}}, + "defer_loading": True, + }, + ], + ) + + assert [(tool.get("type"), tool.get("name"), tool.get("defer_loading")) for tool in tools or ()] == [ + ("function", "get_weather", None) + ] + + def test_anthropic_list_system_becomes_joined_instruction_text() -> None: _, instructions, _, _ = OpenAICountTokensConfig.anthropic_messages_to_responses_input( model="gpt-4o",