This commit is contained in:
mika-lt 2026-10-31 13:59:05 -07:00 • committed by GitHub
commit b97783be34
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 1224 additions and 63 deletions

View file

@ -394,6 +394,39 @@ def _with_original_name(tool: ChatCompletionToolParam, original_names: Mapping[s
return {**tool, "function": {**function, "name": original}}
def _request_deferral(tool: Mapping[str, object]) -> Mapping[str, object] | None:
match tool:
case {"defer_loading": bool() as deferred}:
return {"defer_loading": deferred}
case {"custom": {"defer_loading": bool() as wrapped}}:
return {"custom": {"defer_loading": wrapped}}
case _:
return None
def _request_deferrals(request: Mapping[str, object]) -> Mapping[str, Mapping[str, object]]:
tools: Final = tuple(
fields for tool in list_field(request, "tools") if (fields := as_str_mapping(tool)) is not None
)
return MappingProxyType(
{
name: deferral
for tool in tools
if isinstance(name := tool.get("name"), str) and (deferral := _request_deferral(tool)) is not None
}
)
def _with_request_deferral(
tool: AllAnthropicToolsValues, deferrals: Mapping[str, Mapping[str, object]]
) -> Mapping[str, object]:
fields: Final = as_str_mapping(tool) or {}
name: Final = fields.get("name")
if "defer_loading" in fields or "custom" in fields or not isinstance(name, str) or name not in deferrals:
return fields
return {**fields, **deferrals[name]}
def _write_back_system_block(system: object, block_idx: int, response: str) -> None:
if not isinstance(system, list):
return
@ -901,9 +934,8 @@ class AnthropicMessagesHandler(BaseTranslation):
model: Final = data.get("model")
if model:
inputs["model"] = model
original_tool_names: Final = _original_tool_names(
cast(Mapping[str, object], data) # cast-ok: Anthropic request body
)
request_body: Final = cast(Mapping[str, object], data) # cast-ok: Anthropic request body
original_tool_names: Final = _original_tool_names(request_body)
guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail(
inputs=inputs,
request_data=data,
@ -916,13 +948,14 @@ class AnthropicMessagesHandler(BaseTranslation):
if guardrailed_tools is not None:
# Convert tools back from OpenAI format to Anthropic format
anthropic_config: Final = AnthropicConfig()
anthropic_tools: Final[list[AllAnthropicToolsValues]] = []
deferrals: Final = _request_deferrals(request_body)
anthropic_tools: Final[list[Mapping[str, object]]] = []
for tool in guardrailed_tools:
converted_tool, mcp_server = anthropic_config.map_tool_helper(
_with_original_name(tool, original_tool_names.declared)
)
if converted_tool is not None:
anthropic_tools.append(converted_tool)
anthropic_tools.append(_with_request_deferral(converted_tool, deferrals))
# Note: MCP servers are handled separately in the main transformation
data["tools"] = (
merge_returned_tools_into_request_tools(

View file

@ -24,6 +24,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
prompt_cache_breakpoint_for_wire,
with_prompt_cache_breakpoint,
)
from litellm.litellm_core_utils.tool_fields import drop_policy
from litellm.responses.utils import auto_reasoning_summary
from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS
from litellm.types.llms.anthropic import (
@ -38,7 +39,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
)
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import CountContextManagement
from litellm.types.utils import CountContextManagement, LlmProviders
from litellm.utils import ProviderConfigManager
from ..adapters.handler import context_management_explicitly_dropped, request_safeguards_evaluator
@ -49,6 +50,7 @@ from ..convertible_blocks import reject_malformed_tool_result_text
from ..safeguards import with_safeguard_results
from ..utils import litellm_logging_obj_from_kwargs, local_model_name
from .streaming_iterator import AnthropicResponsesStreamWrapper
from .tool_translation import maps_openai_tool_search
from .transformation import LiteLLMAnthropicToResponsesAPIAdapter
AnthropicRequestMessages: TypeAlias = list[AllAnthropicMessageValues] | list[dict[str, object]]
@ -170,15 +172,26 @@ def _forwarded_kwargs(extra_kwargs: Mapping[str, object] | None) -> Mapping[str,
return extra_kwargs or {}
def _resolved_provider(model: str, custom_llm_provider: object) -> str:
return custom_llm_provider if isinstance(custom_llm_provider, str) else litellm.get_llm_provider(model=model)[1]
def _provider_returns_encrypted_reasoning(model: str, custom_llm_provider: object) -> bool:
provider: Final = (
custom_llm_provider if isinstance(custom_llm_provider, str) else litellm.get_llm_provider(model=model)[1]
)
provider: Final = _resolved_provider(model, custom_llm_provider)
provider_model: Final = local_model_name(model, provider)
responses_config: Final = ProviderConfigManager.get_provider_responses_api_config(provider, provider_model)
return responses_config is not None and "include" in responses_config.get_supported_openai_params(provider_model)
def _maps_tool_search(model: str, forwarded_kwargs: Mapping[str, object]) -> bool:
provider: Final = _resolved_provider(model, forwarded_kwargs.get("custom_llm_provider"))
return provider == LlmProviders.OPENAI.value and maps_openai_tool_search(
local_model_name(model, provider),
forwarded_kwargs.get("api_base") or forwarded_kwargs.get("base_url"),
forwarded_kwargs.get("additional_drop_params"),
)
def _reads_prompt_cache_breakpoint(model: str, forwarded_kwargs: Mapping[str, object]) -> bool:
provider: Final = forwarded_kwargs.get("custom_llm_provider")
return AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
@ -304,9 +317,11 @@ def _build_responses_kwargs(
request_data["temperature"] = temperature
if thinking:
request_data["thinking"] = thinking
if tool_choice:
dropped: Final = drop_policy(forwarded_kwargs).additional_drop_params
drops_tools: Final = "tools" in dropped
if tool_choice and not drops_tools and "tool_choice" not in dropped:
request_data["tool_choice"] = tool_choice
if tools:
if tools and not drops_tools:
request_data["tools"] = tools
if top_p is not None:
request_data["top_p"] = top_p
@ -321,6 +336,7 @@ def _build_responses_kwargs(
),
auto_summary_level=auto_reasoning_summary(model, forwarded_kwargs.get("custom_llm_provider")),
drop_params=drops_unsupported_params(forwarded_kwargs.get("drop_params")),
maps_tool_search=_maps_tool_search(model, forwarded_kwargs),
)
# Normalize reasoning effort based on model capabilities

View file

@ -0,0 +1,363 @@
import re
from collections.abc import Mapping, Sequence
from dataclasses import dataclass, replace
from typing import Final, TypeAlias
from pydantic import ConfigDict, TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.integrations.anthropic_cache_control_hook import targets_openai_api
from litellm.llms.anthropic.messages.utils import json_copy_or_original
from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_TOOL_TYPES
from litellm.types.utils import LlmProviders
_VERSIONED_TOOL_TYPE: Final = re.compile(r"(?P<family>[a-z0-9_]+?)_(?P<version>\d{8})")
_UNVERSIONED_SERVER_TOOL_TYPES: Final = frozenset({"mcp_toolset", "computer_toolset"})
_DEFERRAL_DROP_PATH: Final = re.compile(r"tools\[[^\]]*\]\.defer_loading")
_DropPaths: TypeAlias = list[object] | tuple[object, ...]
_DROP_PATHS: Final[TypeAdapter[_DropPaths]] = TypeAdapter(_DropPaths, config=ConfigDict(strict=True))
_TEXT_EDITOR_INSERT_TEXT_SINCE: Final = "20250429"
_COMPUTER_MODIFIERS_SINCE: Final = "20250124"
_COMPUTER_ZOOM_SINCE: Final = "20251124"
_COMPUTER_ACTIONS_20241022: Final = (
"key",
"type",
"mouse_move",
"left_click",
"left_click_drag",
"right_click",
"middle_click",
"double_click",
"screenshot",
"cursor_position",
)
_COMPUTER_ACTIONS_20250124: Final = (
*_COMPUTER_ACTIONS_20241022,
"left_mouse_down",
"left_mouse_up",
"scroll",
"hold_key",
"wait",
"triple_click",
)
_COMPUTER_ACTIONS_20251124: Final = (*_COMPUTER_ACTIONS_20250124, "zoom")
@dataclass(frozen=True, slots=True)
class ResponsesTool:
name: str
payload: Mapping[str, object]
choice: Mapping[str, object] | None
@dataclass(frozen=True, slots=True)
class UntranslatableTool:
name: str
type: str
TranslatedTool: TypeAlias = ResponsesTool | UntranslatableTool
def _family_and_version(tool_type: str) -> tuple[str, str]:
versioned: Final = _VERSIONED_TOOL_TYPE.fullmatch(tool_type)
if versioned is None:
return tool_type, ""
return versioned["family"], versioned["version"]
def _string(description: str) -> Mapping[str, object]:
return {"type": "string", "description": description}
def _integer(description: str) -> Mapping[str, object]:
return {"type": "integer", "description": description}
def _integers(count: int, description: str) -> Mapping[str, object]:
return {
"type": "array",
"items": {"type": "integer"},
"minItems": count,
"maxItems": count,
"description": description,
}
def _enum(values: Sequence[str]) -> Mapping[str, object]:
return {"type": "string", "enum": list(values)}
def _function(name: str, description: str, properties: Mapping[str, object], required: Sequence[str]) -> ResponsesTool:
return ResponsesTool(
name=name,
payload={
"type": "function",
"name": name,
"strict": False,
"description": description,
"parameters": {"type": "object", "properties": properties, "required": list(required)},
},
choice={"type": "function", "name": name},
)
def _hosted(name: str, payload: Mapping[str, object]) -> ResponsesTool:
return ResponsesTool(name=name, payload=payload, choice={"type": payload["type"]})
def _bash_tool(name: str) -> ResponsesTool:
return _function(
name,
"Run a command in a persistent bash session. The working directory, environment, and files carry over "
"between calls.",
{
"command": _string("The bash command to run. Required unless restart is true."),
"restart": {"type": "boolean", "description": "Set to true to restart the bash session."},
},
(),
)
def _text_editor_tool(name: str, version: str) -> ResponsesTool:
inserts_insert_text: Final = version >= _TEXT_EDITOR_INSERT_TEXT_SINCE
edit_properties: Final[Mapping[str, object]] = {
"command": _enum(
("view", "create", "str_replace", "insert")
if inserts_insert_text
else ("view", "create", "str_replace", "insert", "undo_edit")
),
"path": _string("Absolute path to the file or directory."),
"view_range": _integers(2, "Optional [start, end] line numbers for view. An end of -1 reads to the end."),
"file_text": _string("Content of the new file, for create."),
"old_str": _string("Exact text to replace, for str_replace. It must match exactly one place in the file."),
"new_str": _string(
"Replacement text, for str_replace."
if inserts_insert_text
else "Replacement text for str_replace, or the text to add for insert."
),
"insert_line": _integer("Line number to insert after, for insert. 0 inserts at the start of the file."),
}
return _function(
name,
"View, create, and edit text files. view lists a directory or shows a file with line numbers.",
(
{**edit_properties, "insert_text": _string("Text to add, for insert.")}
if inserts_insert_text
else edit_properties
),
("command", "path"),
)
def _memory_tool(name: str) -> ResponsesTool:
return _function(
name,
"Read and write files in the /memories directory, which persists between conversations. View it before "
"starting a task, and record progress and learnings as you work.",
{
"command": _enum(("view", "create", "str_replace", "insert", "delete", "rename")),
"path": _string("Path under /memories, for every command but rename."),
"view_range": _integers(2, "Optional [start, end] line numbers for view."),
"file_text": _string("Content of the new file, for create."),
"old_str": _string("Exact text to replace, for str_replace."),
"new_str": _string("Replacement text, for str_replace."),
"insert_line": _integer("Line number to insert after, for insert."),
"insert_text": _string("Text to add, for insert."),
"old_path": _string("Current path, for rename."),
"new_path": _string("New path, for rename."),
},
("command",),
)
def _computer_properties(version: str, zoom: bool) -> Mapping[str, object]:
pointer: Final[Mapping[str, object]] = {
"text": _string("Text to type for type, or keys to press for key and hold_key, such as ctrl+s."),
"coordinate": _integers(2, "[x, y] pixel position for a click, mouse_move, or the end of left_click_drag."),
"start_coordinate": _integers(2, "[x, y] pixel position where left_click_drag starts."),
}
if version < _COMPUTER_MODIFIERS_SINCE:
return {"action": _enum(_COMPUTER_ACTIONS_20241022), **pointer}
modifiers: Final[Mapping[str, object]] = {
**pointer,
"scroll_direction": _enum(("up", "down", "left", "right")),
"scroll_amount": _integer("Number of scroll steps, for scroll."),
"text": _string(
"Text to type for type, keys to press for key and hold_key, such as ctrl+s, or a modifier key to hold "
"during scroll, such as shift."
),
"duration": {"type": "number", "description": "Seconds to wait for wait, or to hold for hold_key."},
"key": _string("Modifier key to hold during a click, such as shift."),
}
if not zoom:
return {"action": _enum(_COMPUTER_ACTIONS_20250124), **modifiers}
return {
"action": _enum(_COMPUTER_ACTIONS_20251124),
**modifiers,
"region": _integers(4, "[x0, y0, x1, y1] screen region to show at full resolution, for zoom."),
}
def _computer_tool(name: str, version: str, tool: Mapping[str, object]) -> ResponsesTool:
width: Final = tool.get("display_width_px")
height: Final = tool.get("display_height_px")
display: Final = (
f"a {width}x{height} pixel display" if isinstance(width, int) and isinstance(height, int) else "a display"
)
return _function(
name,
f"Control {display} with the mouse and keyboard, and take screenshots to see it. Coordinates are pixels "
"from the top left.",
_computer_properties(version, zoom=version >= _COMPUTER_ZOOM_SINCE and tool.get("enable_zoom") is True),
("action",),
)
def _tool_search(name: str, tool: Mapping[str, object]) -> ResponsesTool:
return ResponsesTool(name=name, payload=dict(tool), choice=None)
def _custom_function(name: str, tool: Mapping[str, object]) -> ResponsesTool:
description: Final = tool.get("description")
input_schema: Final = tool.get("input_schema")
return ResponsesTool(
name=name,
payload={
"type": "function",
"name": name,
"strict": bool(tool.get("strict")),
**({"description": description} if "description" in tool else {}),
**({"parameters": json_copy_or_original(input_schema)} if "input_schema" in tool else {}),
},
choice={"type": "function", "name": name},
)
def is_tool_search(tool: Mapping[str, object]) -> bool:
tool_type: Final = tool.get("type")
return isinstance(tool_type, str) and tool_type in ANTHROPIC_TOOL_SEARCH_TOOL_TYPES
def forced_tool_name(tool_choice: object) -> str | None:
match tool_choice:
case {"type": "tool", "name": str() as name}:
return name
case _:
return None
def _defer_loading(tool: Mapping[str, object], forced: str | None) -> object:
if forced is not None and tool.get("name") == forced:
return None
if "defer_loading" in tool:
return tool["defer_loading"]
match tool.get("custom"):
case {"defer_loading": bool() as wrapped}:
return wrapped
case _:
return None
def _defers_a_function(tool: Mapping[str, object], forced: str | None) -> bool:
translated: Final = _translated_tool(tool, False) if _defer_loading(tool, forced) is True else None
return isinstance(translated, ResponsesTool) and translated.payload.get("type") == "function"
def keeps_tool_search(
tools: Sequence[Mapping[str, object]], *, maps_tool_search: bool, forced: str | None = None
) -> bool:
return (
maps_tool_search
and any(is_tool_search(tool) for tool in tools)
and any(_defers_a_function(tool, forced) for tool in tools)
)
def _with_deferral(translated: TranslatedTool, defer_loading: object) -> TranslatedTool:
match translated:
case ResponsesTool(payload=payload) if payload.get("type") == "function" and isinstance(defer_loading, bool):
return replace(translated, payload={**payload, "defer_loading": defer_loading})
case _:
return translated
def responses_tool(
tool: Mapping[str, object], *, runs_tool_search: bool = False, forced: str | None = None
) -> TranslatedTool:
translated: Final = _translated_tool(tool, runs_tool_search)
return _with_deferral(translated, _defer_loading(tool, forced)) if runs_tool_search else translated
def _translated_tool(tool: Mapping[str, object], runs_tool_search: bool) -> TranslatedTool:
tool_type: Final = tool.get("type")
tool_name: Final = tool.get("name")
name: Final = tool_name if isinstance(tool_name, str) else ""
type_name: Final = tool_type if isinstance(tool_type, str) else ""
family, version = _family_and_version(type_name)
if type_name.startswith("web_search") or name == "web_search":
return _hosted(name, {"type": "web_search_preview"})
if runs_tool_search and is_tool_search(tool):
return _tool_search(name, tool)
if not version and family not in _UNVERSIONED_SERVER_TOOL_TYPES:
return _custom_function(name, tool)
match family:
case "bash":
return _bash_tool(name)
case "text_editor":
return _text_editor_tool(name, version)
case "memory":
return _memory_tool(name)
case "computer":
return _computer_tool(name, version, tool)
case _:
return UntranslatableTool(name=name, type=type_name)
_DROP_PARAMS_HOW_TO: Final = (
"set `drop_params: true` in the model's `litellm_params` or under `litellm_settings` on the proxy, or "
"`litellm.drop_params=True` in the SDK"
)
def _tool_types(tools: Sequence[UntranslatableTool]) -> str:
return ", ".join(dict.fromkeys(tool.type for tool in tools))
def untranslatable_tools_error(tools: Sequence[UntranslatableTool]) -> UnsupportedParamsError:
return UnsupportedParamsError(
message=(
f"LiteLLM does not translate these Anthropic tools to the Responses API: {_tool_types(tools)}. "
f"To drop them, {_DROP_PARAMS_HOW_TO}"
)
)
def log_dropped_tools(tools: Sequence[UntranslatableTool]) -> None:
verbose_logger.warning(
"Dropping Anthropic tools the Responses API cannot run under drop_params: %s", _tool_types(tools)
)
def unforceable_tool_choice_error(name: str) -> UnsupportedParamsError:
return UnsupportedParamsError(
message=f"OpenAI cannot force the tool search tool {name} through `tool_choice`. To drop it, {_DROP_PARAMS_HOW_TO}"
)
def _drops_deferral(additional_drop_params: object) -> bool:
try:
paths: Final = _DROP_PATHS.validate_python(additional_drop_params)
except ValidationError:
return False
return any(isinstance(path, str) and _DEFERRAL_DROP_PATH.fullmatch(path) is not None for path in paths)
def maps_openai_tool_search(model: str, api_base: object, additional_drop_params: object) -> bool:
return (
targets_openai_api(api_base)
and not _drops_deferral(additional_drop_params)
and litellm.supports_tool_search(model=model, custom_llm_provider=LlmProviders.OPENAI.value)
)

View file

@ -26,9 +26,19 @@ from litellm.llms.anthropic.messages.native.utils import (
refusal_stop_details,
responses_output_refusal_text,
)
from litellm.llms.anthropic.messages.responses_adapters.tool_translation import (
ResponsesTool,
TranslatedTool,
UntranslatableTool,
forced_tool_name,
keeps_tool_search,
log_dropped_tools,
responses_tool,
unforceable_tool_choice_error,
untranslatable_tools_error,
)
from litellm.llms.anthropic.messages.utils import (
is_reasoning_auto_summary_enabled,
json_copy_or_original,
prompt_cache_key_from_user_id,
)
from litellm.types.llms.anthropic import (
@ -457,45 +467,63 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
return input_items
@staticmethod
def _translate_tool_to_responses_api(tool_dict: Mapping[str, object]) -> dict[str, object]:
tool_type: Final = tool_dict.get("type", "")
tool_name: Final = tool_dict.get("name", "")
# web_search tool
if (isinstance(tool_type, str) and tool_type.startswith("web_search")) or tool_name == "web_search":
return {"type": "web_search_preview"}
# Responses turns strict mode on when `strict` is omitted, silently rewriting
# `required` to every property. Anthropic tools are non-strict unless asked.
func_tool: Final[dict[str, object]] = {
"type": "function",
"name": tool_name,
"strict": bool(tool_dict.get("strict")),
}
if "description" in tool_dict:
func_tool["description"] = tool_dict["description"]
if "input_schema" in tool_dict:
func_tool["parameters"] = json_copy_or_original(tool_dict["input_schema"])
return func_tool
def translated_tools(
tools: Sequence[AllAnthropicToolsValues],
*,
drop_params: bool = True,
maps_tool_search: bool = False,
forced: str | None = None,
) -> tuple[TranslatedTool, ...]:
runs_tool_search: Final = keeps_tool_search(tools, maps_tool_search=maps_tool_search, forced=forced)
translated: Final = tuple(
responses_tool(tool, runs_tool_search=runs_tool_search, forced=forced) for tool in tools
)
untranslatable: Final = tuple(tool for tool in translated if isinstance(tool, UntranslatableTool))
if untranslatable and not drop_params:
raise untranslatable_tools_error(untranslatable)
if untranslatable:
log_dropped_tools(untranslatable)
return translated
def translate_tools_to_responses_api(
self,
tools: list[AllAnthropicToolsValues],
*,
drop_params: bool = True,
maps_tool_search: bool = False,
) -> list[dict[str, object]]:
"""Convert Anthropic tool definitions to Responses API function tools."""
return [self._translate_tool_to_responses_api(tool) for tool in tools]
"""Convert Anthropic tool definitions to Responses API tools."""
return [
dict(tool.payload)
for tool in self.translated_tools(tools, drop_params=drop_params, maps_tool_search=maps_tool_search)
if isinstance(tool, ResponsesTool)
]
@staticmethod
def translate_tool_choice_to_responses_api(
tool_choice: AnthropicMessagesToolChoice,
) -> str | dict[str, object]:
"""Convert Anthropic tool_choice to Responses API tool_choice."""
tools: Sequence[TranslatedTool] = (),
*,
drop_params: bool = True,
) -> str | dict[str, object] | None:
"""Map Anthropic tool_choice to Responses API tool_choice, or None when the named tool cannot be forced."""
tc_type: Final = tool_choice.get("type")
if tc_type == "any":
return "required"
elif tc_type == "tool":
return {"type": "function", "name": tool_choice.get("name", "")}
elif tc_type == "none":
if tc_type == "none":
return "none"
return "auto"
if tc_type != "tool":
return "auto"
name: Final = tool_choice.get("name", "")
match next((tool for tool in tools if tool.name == name), None):
case ResponsesTool(choice=choice) if choice is not None:
return dict(choice)
case ResponsesTool(name=forced) if not drop_params:
raise unforceable_tool_choice_error(forced)
case ResponsesTool() | UntranslatableTool():
return None
case None:
return {"type": "function", "name": name}
@staticmethod
def translate_context_management_to_responses_api(
@ -587,6 +615,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
auto_summary_level: str = "detailed",
*,
drop_params: bool = True,
maps_tool_search: bool = False,
) -> dict[str, Any]:
"""
Translate a full Anthropic /v1/messages request dict to
@ -596,7 +625,8 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
on every call, so a reasoning model's items can be replayed intact next turn even
when the client sent no ``thinking`` block; pass False for a provider whose
Responses API rejects ``include``. ``auto_summary_level`` is the ``reasoning.summary`` the
opt-in auto summary injects when the client sent none.
opt-in auto summary injects when the client sent none. ``maps_tool_search`` passes Anthropic tool search
through for a target that runs OpenAI's tool search, and it is otherwise an unsupported tool.
"""
model: Final[str] = anthropic_request["model"]
messages_list: Final = cast(
@ -646,19 +676,24 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
if "top_p" in anthropic_request:
responses_kwargs["top_p"] = anthropic_request["top_p"]
# tools
tools: Final = anthropic_request.get("tools")
if tools:
responses_kwargs["tools"] = self.translate_tools_to_responses_api(
cast(list[AllAnthropicToolsValues], tools)
)
tool_choice: Final = cast(AnthropicMessagesToolChoice | None, anthropic_request.get("tool_choice"))
translated_tools: Final = self.translated_tools(
cast(list[AllAnthropicToolsValues], anthropic_request.get("tools") or []),
drop_params=drop_params,
maps_tool_search=maps_tool_search,
forced=forced_tool_name(tool_choice),
)
responses_tools: Final = [dict(tool.payload) for tool in translated_tools if isinstance(tool, ResponsesTool)]
if responses_tools:
responses_kwargs["tools"] = responses_tools
# tool_choice
tool_choice: Final = anthropic_request.get("tool_choice")
if tool_choice:
responses_kwargs["tool_choice"] = self.translate_tool_choice_to_responses_api(
cast(AnthropicMessagesToolChoice, tool_choice)
)
responses_tool_choice: Final = (
self.translate_tool_choice_to_responses_api(tool_choice, translated_tools, drop_params=drop_params)
if tool_choice and (responses_tools or not translated_tools)
else None
)
if responses_tool_choice is not None:
responses_kwargs["tool_choice"] = responses_tool_choice
# thinking -> reasoning
thinking: Final = anthropic_request.get("thinking")

View file

@ -18,6 +18,7 @@ from litellm.litellm_core_utils.tool_fields import DropPolicy
from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError
from litellm.llms.anthropic.messages.convertible_blocks import deployment_drops_params, unconvertible_blocks_error
from litellm.llms.anthropic.messages.responses_adapters.handler import count_view_messages
from litellm.llms.anthropic.messages.responses_adapters.tool_translation import maps_openai_tool_search
from litellm.llms.anthropic.messages.responses_adapters.transformation import (
LiteLLMAnthropicToResponsesAPIAdapter,
)
@ -63,6 +64,14 @@ class _CountCredential:
credential: str | OpenAIWorkloadIdentityConfig
def _deployment_drop_paths(deployment: Mapping[str, object] | None) -> object:
match deployment:
case {"litellm_params": {"additional_drop_params": object() as additional_drop_params}}:
return additional_drop_params
case _:
return None
def _count_credential(deployment: Mapping[str, object] | None, model_label: str) -> _CountCredential | OpenAIError:
litellm_params: Final = as_str_mapping(None if deployment is None else deployment.get("litellm_params")) or {}
api_base: Final = litellm_params.get("api_base")
@ -146,11 +155,18 @@ class OpenAITokenCounter(BaseTokenCounter):
system: object,
messages_format: TokenCountMessagesFormat,
output_format: JsonValue,
api_base: str | None,
additional_drop_params: object,
) -> ResponsesCountRequestParts:
match messages_format:
case "anthropic":
return OpenAICountTokensConfig.anthropic_messages_to_responses_input(
model=model_to_use, messages=messages, system=system, tools=tools, output_format=output_format
model=model_to_use,
messages=messages,
system=system,
tools=tools,
output_format=output_format,
maps_tool_search=maps_openai_tool_search(model_to_use, api_base, additional_drop_params),
)
case "openai":
input_items, instructions = OpenAICountTokensConfig.messages_to_responses_input(
@ -229,6 +245,8 @@ class OpenAITokenCounter(BaseTokenCounter):
system=system,
messages_format=messages_format,
output_format=output_format,
api_base=resolved.api_base,
additional_drop_params=_deployment_drop_paths(deployment),
)
# If no input items were produced (e.g., system-only messages), fall back to local counting

View file

@ -20,6 +20,7 @@ from litellm.llms.anthropic.messages.native.utils import normalize_native_messag
from litellm.llms.anthropic.messages.responses_adapters.transformation import (
LiteLLMAnthropicToResponsesAPIAdapter,
)
from litellm.responses.litellm_completion_transformation.tool_search import tools_with_anthropic_tool_search_mapped
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.llms.anthropic import AnthropicMessagesRequest, AnthropicOutputSchema
from litellm.types.llms.openai import ResponsesAPIRequestParams, replayable_responses_input
@ -343,6 +344,7 @@ class OpenAICountTokensConfig:
system: object,
tools: Sequence[Mapping[str, object]] | None,
output_format: JsonValue = None,
maps_tool_search: bool = False,
) -> ResponsesCountRequestParts:
"""Convert with the adapter /v1/messages inference uses, so the count sees the same input it bills."""
anthropic_request: Final = AnthropicMessagesRequest(
@ -354,15 +356,19 @@ class OpenAICountTokensConfig:
)
translated: Final = _TRANSLATED_REQUEST.validate_python(
LiteLLMAnthropicToResponsesAPIAdapter().translate_request(
anthropic_request, include_encrypted_reasoning=False
anthropic_request, include_encrypted_reasoning=False, maps_tool_search=maps_tool_search
)
)
return OpenAICountTokensConfig.count_request_parts(
input_items, instructions, translated_tools, text = OpenAICountTokensConfig.count_request_parts(
ResponsesAPIRequestUtils.restore_encrypted_content_item_ids_in_input(translated.get("input")),
translated.get("instructions"),
translated.get("tools"),
translated.get("text"),
)
counted_tools: Final = (
tools_with_anthropic_tool_search_mapped(translated_tools) if maps_tool_search else translated_tools
)
return input_items, instructions, None if counted_tools is None else tuple(counted_tools), text
@staticmethod
def count_request_parts(

View file

@ -1,13 +1,13 @@
from collections.abc import Iterable, Iterator, Mapping, Sequence, Set
from dataclasses import dataclass
from typing import TYPE_CHECKING, Final, Literal, TypeAlias
from typing import TYPE_CHECKING, Final, Literal, TypeAlias, TypeVar
from pydantic import ConfigDict, TypeAdapter, ValidationError
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import is_anthropic_tool_search_use
from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_TOOL_TYPES, AnthropicToolSearchToolRegex
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.openai import ALL_RESPONSES_API_TOOL_PARAMS, OpenAIToolSearchToolParam, ResponseInputParam
from litellm.types.llms.openai import OpenAIToolSearchToolParam, ResponseInputParam
from litellm.types.responses.main import OutputToolSearchCall
from litellm.types.utils import LlmProviders, Message
@ -57,6 +57,7 @@ CLIENT_TOOL_SEARCH_FUNCTION_NAME: Final = "tool_search"
ToolSearchCallStatus: TypeAlias = Literal["in_progress", "completed", "incomplete"]
ToolSearchExecution: TypeAlias = Literal["server", "client"]
_Tool: Final = TypeVar("_Tool", bound=Mapping[str, object])
_ARGUMENTS_ADAPTER: Final = TypeAdapter(dict[str, object])
_MESSAGE_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
@ -138,8 +139,8 @@ def is_openai_tool_search_tool(tool: object) -> bool:
def tools_with_anthropic_tool_search_mapped(
tools: Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None,
) -> Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None:
tools: Sequence[_Tool] | None,
) -> Sequence[_Tool | OpenAIToolSearchToolParam] | None:
if tools is None or not any(is_anthropic_tool_search_chat_tool(tool) for tool in tools):
return tools
if any(is_openai_tool_search_tool(tool) for tool in tools):
@ -156,8 +157,8 @@ def tools_with_anthropic_tool_search_mapped(
def tools_without_anthropic_tool_search(
tools: Sequence[ALL_RESPONSES_API_TOOL_PARAMS],
) -> tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]:
tools: Sequence[_Tool],
) -> tuple[_Tool, ...]:
return tuple(tool for tool in tools if not is_anthropic_tool_search_chat_tool(tool))

View file

@ -629,7 +629,7 @@ def test_chat_completions_through_the_responses_bridge_follow_drop_params(
assert [tuple(tool.get(key) for key in wire_keys) for tool in tools] == forwarded, tools
def test_messages_to_an_openai_model_with_the_search_tool_still_answers(
def test_messages_to_an_openai_model_runs_the_search_tool_on_openai_with_the_tool_still_deferred(
openai_gateway: Gateway, openai_api: ControlPlane
) -> None:
exchange: Final = _exchange(
@ -654,7 +654,12 @@ def test_messages_to_an_openai_model_with_the_search_tool_still_answers(
)
assert exchange.response.status_code == 200, exchange.response.text
assert len(exchange.requests) == 1, exchange.requests
tools: Final = _ITEMS.validate_python(exchange.only_body()["tools"])
wire_keys: Final = ("type", "execution", "name", "defer_loading")
assert [tuple(tool.get(key) for key in wire_keys) for tool in tools] == [
("tool_search", "server", None, None),
("function", None, "get_weather", True),
], tools
def test_a_global_drop_params_drops_the_search_for_an_openai_model_that_cannot_run_it(

View file

@ -998,6 +998,34 @@ class TestAnthropicMessagesHandlerInputProcessing:
assert {"googleMaps": {"enable_widget": True}} in data["tools"]
assert [tool["name"] for tool in data["tools"] if "name" in tool] == ["get_weather"]
@pytest.mark.asyncio
async def test_a_deferred_tool_keeps_its_defer_loading_through_a_guardrail_that_returns_the_tools(self):
handler = AnthropicMessagesHandler()
guardrail = MockPassThroughGuardrail(guardrail_name="test")
data = {
"model": "gpt-5.6-luna",
"messages": [{"role": "user", "content": "what is AAPL at?"}],
"tools": [
{"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"},
{"name": "get_stock", "input_schema": {"type": "object", "properties": {}}, "defer_loading": True},
{"name": "get_weather", "input_schema": {"type": "object", "properties": {}}, "defer_loading": False},
{"name": "get_time", "input_schema": {"type": "object", "properties": {}}},
{"name": "get_quote", "input_schema": {"type": "object", "properties": {}}, "custom": {"defer_loading": True}},
{"name": "get_news", "input_schema": {"type": "object", "properties": {}}, "defer_loading": "true"},
],
}
await handler.process_input_messages(data=data, guardrail_to_apply=guardrail)
assert [(tool["name"], tool.get("defer_loading"), tool.get("custom")) for tool in data["tools"]] == [
("tool_search_tool_regex", None, None),
("get_stock", True, None),
("get_weather", False, None),
("get_time", None, None),
("get_quote", None, {"defer_loading": True}),
("get_news", None, None),
]
@pytest.mark.asyncio
async def test_midturn_system_correction_is_guardrailed_when_top_level_system_is_skipped(
self,

View file

@ -2,7 +2,7 @@ import datetime
import json
import os
import sys
from collections.abc import AsyncIterator, Mapping
from collections.abc import AsyncIterator, Mapping, Sequence
from itertools import chain
from typing import Final, cast
from unittest.mock import AsyncMock, patch
@ -764,6 +764,338 @@ async def test_recomputed_cost_of_a_served_bridge_response_prices_the_reasoning_
assert logging_obj.cost_breakdown["reasoning_cost"] == pytest.approx(expected_reasoning)
TOOL_SEARCH: Final = {"type": "tool_search_tool_bm25_20251119", "name": "tool_search_tool_bm25"}
DEFERRED_WEATHER: Final = {
"name": "get_weather",
"description": "Weather for a city",
"input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
"defer_loading": True,
}
WEB_FETCH: Final = {"type": "web_fetch_20250910", "name": "web_fetch"}
TYPED_BASH: Final = {"type": "bash_20250124", "name": "bash"}
OPENAI_API_BASE: Final = "https://api.openai.com/v1"
OTHER_RESPONSES_API_BASE: Final = "https://responses.example.test/v1"
async def _openai_request_for(
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
tools: list[Mapping[str, object]],
api_base: str = OPENAI_API_BASE,
model: str = "openai/gpt-5.6-luna",
**extra: object,
) -> respx.Route:
monkeypatch.setenv("OPENAI_API_KEY", "sk-typed-tools-test")
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
litellm.in_memory_llm_clients_cache.flush_cache()
upstream: Final = respx_mock.post(f"{api_base}/responses").respond(status_code=200, json=TOOL_CALL_RESPONSE)
await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler(
max_tokens=1024,
messages=MESSAGES,
model=model,
tools=[dict(tool) for tool in tools],
custom_llm_provider="openai",
api_base=api_base,
**extra,
)
return upstream
def _tools_sent(upstream: respx.Route) -> list[Mapping[str, object]]:
return json.loads(upstream.calls.last.request.content)["tools"]
@pytest.mark.asyncio
async def test_anthropic_tool_search_reaches_openai_as_its_server_tool_search_with_the_tool_still_deferred(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
upstream: Final = await _openai_request_for(respx_mock, monkeypatch, [TOOL_SEARCH, DEFERRED_WEATHER])
wire_keys: Final = ("type", "execution", "name", "defer_loading")
assert [tuple(tool.get(key) for key in wire_keys) for tool in _tools_sent(upstream)] == [
("tool_search", "server", None, None),
("function", None, "get_weather", True),
]
@pytest.mark.asyncio
async def test_a_deferred_typed_tool_keeps_the_tool_search_under_drop_params(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
upstream: Final = await _openai_request_for(
respx_mock, monkeypatch, [TOOL_SEARCH, {**TYPED_BASH, "defer_loading": True}], drop_params=True
)
assert [(tool["type"], tool.get("defer_loading")) for tool in _tools_sent(upstream)] == [
("tool_search", None),
("function", True),
]
@pytest.mark.asyncio
async def test_tool_search_for_a_responses_server_other_than_openai_answers_400_without_drop_params(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
with pytest.raises(litellm.UnsupportedParamsError) as error:
await _openai_request_for(
respx_mock, monkeypatch, [TOOL_SEARCH, DEFERRED_WEATHER], api_base=OTHER_RESPONSES_API_BASE, drop_params=False
)
assert error.value.status_code == 400
assert not respx_mock.calls
@pytest.mark.asyncio
async def test_drop_params_drops_tool_search_and_its_deferral_for_a_responses_server_other_than_openai(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
upstream: Final = await _openai_request_for(
respx_mock, monkeypatch, [TOOL_SEARCH, DEFERRED_WEATHER], api_base=OTHER_RESPONSES_API_BASE, drop_params=True
)
assert [(tool["type"], tool["name"], tool.get("defer_loading")) for tool in _tools_sent(upstream)] == [
("function", "get_weather", None)
]
MODEL_WITHOUT_TOOL_SEARCH: Final = "openai/a-model-the-cost-map-does-not-mark-for-tool-search"
@pytest.mark.asyncio
async def test_tool_search_on_an_openai_model_that_cannot_run_it_answers_400_naming_drop_params(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
with pytest.raises(litellm.UnsupportedParamsError) as error:
await _openai_request_for(
respx_mock, monkeypatch, [TOOL_SEARCH, DEFERRED_WEATHER], model=MODEL_WITHOUT_TOOL_SEARCH, drop_params=False
)
assert error.value.status_code == 400
assert "tool_search_tool_bm25_20251119" in error.value.message
assert "drop_params" in error.value.message
assert not respx_mock.calls
@pytest.mark.asyncio
async def test_drop_params_drops_tool_search_and_its_deferral_on_an_openai_model_that_cannot_run_it(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
upstream: Final = await _openai_request_for(
respx_mock, monkeypatch, [TOOL_SEARCH, DEFERRED_WEATHER], model=MODEL_WITHOUT_TOOL_SEARCH, drop_params=True
)
assert [(tool["type"], tool["name"], tool.get("defer_loading")) for tool in _tools_sent(upstream)] == [
("function", "get_weather", None)
]
UNDEFERRED_WEATHER: Final = {key: value for key, value in DEFERRED_WEATHER.items() if key != "defer_loading"}
@pytest.mark.asyncio
@pytest.mark.parametrize(
"weather",
[
pytest.param(UNDEFERRED_WEATHER, id="no-defer-loading"),
pytest.param({**DEFERRED_WEATHER, "defer_loading": "true"}, id="defer-loading-string"),
],
)
async def test_tool_search_with_no_deferred_tool_answers_400_naming_drop_params_before_any_openai_call(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, weather: Mapping[str, object]
):
with pytest.raises(litellm.UnsupportedParamsError) as error:
await _openai_request_for(respx_mock, monkeypatch, [TOOL_SEARCH, weather], drop_params=False)
assert error.value.status_code == 400
assert "tool_search_tool_bm25_20251119" in error.value.message
assert "drop_params" in error.value.message
assert not respx_mock.calls
@pytest.mark.asyncio
@pytest.mark.parametrize("tool_type", [pytest.param(["bash"], id="list"), pytest.param({"name": "bash"}, id="object")])
async def test_a_tool_whose_type_is_not_a_string_reaches_openai_as_a_function_beside_tool_search(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, tool_type: object
):
upstream: Final = await _openai_request_for(
respx_mock, monkeypatch, [TOOL_SEARCH, DEFERRED_WEATHER, {"type": tool_type, "name": "odd_tool"}]
)
assert [(tool["type"], tool.get("name")) for tool in _tools_sent(upstream)] == [
("tool_search", None),
("function", "get_weather"),
("function", "odd_tool"),
]
@pytest.mark.asyncio
async def test_additional_drop_params_tools_sends_no_tools_even_beside_one_openai_cannot_run(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
upstream: Final = await _openai_request_for(
respx_mock, monkeypatch, [WEB_FETCH, TYPED_BASH], drop_params=False, additional_drop_params=["tools"]
)
assert "tools" not in json.loads(upstream.calls.last.request.content)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"tool_choice",
[{"type": "tool", "name": "web_fetch"}, {"type": "tool", "name": "bash"}, {"type": "any"}],
ids=["forcing-a-tool-openai-cannot-run", "forcing-a-client-tool", "forcing-any-tool"],
)
async def test_additional_drop_params_tools_sends_no_tool_choice_forcing_a_dropped_tool(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, tool_choice: Mapping[str, object]
):
upstream: Final = await _openai_request_for(
respx_mock,
monkeypatch,
[WEB_FETCH, TYPED_BASH],
drop_params=True,
additional_drop_params=["tools"],
tool_choice=dict(tool_choice),
)
sent: Final = json.loads(upstream.calls.last.request.content)
assert "tools" not in sent
assert "tool_choice" not in sent
DEFERRED_STOCK: Final = {
"name": "get_stock",
"description": "Price for a ticker",
"input_schema": {"type": "object", "properties": {"ticker": {"type": "string"}}, "required": ["ticker"]},
"defer_loading": True,
}
FORCE_STOCK: Final = {"type": "tool", "name": "get_stock"}
def _tool_search_wire(upstream: respx.Route) -> list[tuple[object, ...]]:
return [(tool["type"], tool.get("name"), tool.get("defer_loading")) for tool in _tools_sent(upstream)]
@pytest.mark.asyncio
async def test_a_tool_choice_forcing_a_deferred_function_sends_that_function_loaded_beside_tool_search(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
upstream: Final = await _openai_request_for(
respx_mock,
monkeypatch,
[TOOL_SEARCH, DEFERRED_STOCK, DEFERRED_WEATHER],
drop_params=False,
tool_choice=dict(FORCE_STOCK),
)
assert _tool_search_wire(upstream) == [
("tool_search", None, None),
("function", "get_stock", None),
("function", "get_weather", True),
]
assert json.loads(upstream.calls.last.request.content)["tool_choice"] == {"type": "function", "name": "get_stock"}
@pytest.mark.asyncio
async def test_drop_params_drops_tool_search_when_the_forced_function_is_the_only_deferred_one(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
upstream: Final = await _openai_request_for(
respx_mock, monkeypatch, [TOOL_SEARCH, DEFERRED_STOCK], drop_params=True, tool_choice=dict(FORCE_STOCK)
)
assert _tool_search_wire(upstream) == [("function", "get_stock", None)]
assert json.loads(upstream.calls.last.request.content)["tool_choice"] == {"type": "function", "name": "get_stock"}
@pytest.mark.asyncio
async def test_additional_drop_params_tool_choice_sends_tool_search_without_the_choice_that_forces_it(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
upstream: Final = await _openai_request_for(
respx_mock,
monkeypatch,
[TOOL_SEARCH, DEFERRED_WEATHER],
drop_params=False,
additional_drop_params=["tool_choice"],
tool_choice={"type": "tool", "name": "tool_search_tool_bm25"},
)
assert "tool_choice" not in json.loads(upstream.calls.last.request.content)
assert _tool_search_wire(upstream) == [("tool_search", None, None), ("function", "get_weather", True)]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"additional_drop_params",
[
pytest.param(["tools[*].defer_loading"], id="every-tool"),
pytest.param(["tools[1].defer_loading"], id="one-tool-by-index"),
pytest.param(("tools[*].defer_loading",), id="sdk-tuple"),
],
)
async def test_additional_drop_params_of_a_defer_loading_path_answers_400_for_tool_search_without_drop_params(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, additional_drop_params: Sequence[str]
):
with pytest.raises(litellm.UnsupportedParamsError) as error:
await _openai_request_for(
respx_mock,
monkeypatch,
[TOOL_SEARCH, DEFERRED_WEATHER],
drop_params=False,
additional_drop_params=additional_drop_params,
)
assert "tool_search_tool_bm25_20251119" in error.value.message
assert not respx_mock.calls
@pytest.mark.asyncio
async def test_a_deferral_inside_the_custom_object_keeps_tool_search_with_the_tool_deferred(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
wrapped: Final = {**UNDEFERRED_WEATHER, "custom": {"defer_loading": True}}
upstream: Final = await _openai_request_for(respx_mock, monkeypatch, [TOOL_SEARCH, wrapped], drop_params=False)
assert _tool_search_wire(upstream) == [("tool_search", None, None), ("function", "get_weather", True)]
assert all("custom" not in tool for tool in _tools_sent(upstream))
@pytest.mark.asyncio
async def test_an_unversioned_typed_tool_name_reaches_openai_with_the_clients_own_schema(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
schema: Final = {"type": "object", "properties": {"script": {"type": "string"}}, "required": ["script"]}
upstream: Final = await _openai_request_for(
respx_mock, monkeypatch, [{"type": "bash", "name": "bash", "input_schema": schema}], drop_params=False
)
assert [(tool["name"], tool["parameters"]) for tool in _tools_sent(upstream)] == [("bash", schema)]
@pytest.mark.asyncio
async def test_a_tool_openai_cannot_run_answers_400_before_any_openai_call_without_drop_params(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
with pytest.raises(litellm.UnsupportedParamsError) as error:
await _openai_request_for(respx_mock, monkeypatch, [WEB_FETCH, TYPED_BASH], drop_params=False)
assert error.value.status_code == 400
assert not respx_mock.calls
@pytest.mark.asyncio
async def test_drop_params_drops_a_tool_openai_cannot_run_and_keeps_the_rest(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
upstream: Final = await _openai_request_for(respx_mock, monkeypatch, [WEB_FETCH, TYPED_BASH], drop_params=True)
tools: Final = _tools_sent(upstream)
assert [tool["name"] for tool in tools] == ["bash"]
assert "command" in cast(Mapping[str, Mapping[str, object]], tools[0]["parameters"])["properties"]
TEXT_RESPONSE: Final = {
"id": "resp_cleared",
"object": "response",

View file

@ -17,6 +17,7 @@ from litellm.constants import (
DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
)
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.prompt_templates.common_utils import (
TOOL_RESULT_IMAGE_BOUNDARY,
TOOL_RESULT_IMAGE_PLACEHOLDER,
@ -949,6 +950,220 @@ class TestTranslateToolChoiceToResponsesAPI:
assert result == {"type": "function", "name": "get_weather"}
_TOOL_SEARCH: Final = {"type": "tool_search_tool_bm25_20251119", "name": "tool_search_tool_bm25"}
_DEFERRED_WEATHER: Final = {
"name": "get_weather",
"description": "Weather for a city",
"input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
"defer_loading": True,
}
_DEFERRED_BASH: Final = {"type": "bash_20250124", "name": "bash", "defer_loading": True}
class TestAnthropicTypedToolsOnResponses:
"""Anthropic-defined tools carry no input_schema, so the adapter has to supply the schema Claude has built in."""
@staticmethod
def _only_tool(tool: Dict[str, Any]) -> Dict[str, Any]:
tools: Final = _ADAPTER.translate_request(_make_request(tools=[tool]), drop_params=False)["tools"]
assert len(tools) == 1
return tools[0]
@staticmethod
def _enum(tool: Dict[str, Any], prop: str) -> List[str]:
return tool["parameters"]["properties"][prop]["enum"]
def test_bash_gets_a_command_parameter(self):
tool: Final = self._only_tool({"type": "bash_20250124", "name": "bash"})
assert tool["type"] == "function"
assert tool["name"] == "bash"
assert tool["strict"] is False
assert set(tool["parameters"]["properties"]) == {"command", "restart"}
assert tool["description"]
def test_text_editor_from_20250429_inserts_insert_text_and_has_no_undo(self):
tool: Final = self._only_tool({"type": "text_editor_20250728", "name": "str_replace_based_edit_tool"})
assert tool["name"] == "str_replace_based_edit_tool"
assert self._enum(tool, "command") == ["view", "create", "str_replace", "insert"]
assert "insert_text" in tool["parameters"]["properties"]
assert tool["parameters"]["required"] == ["command", "path"]
def test_text_editor_before_20250429_inserts_new_str_and_can_undo(self):
tool: Final = self._only_tool({"type": "text_editor_20250124", "name": "str_replace_editor"})
assert "undo_edit" in self._enum(tool, "command")
assert "insert_text" not in tool["parameters"]["properties"]
assert "new_str" in tool["parameters"]["properties"]
def test_memory_gets_every_memory_command(self):
tool: Final = self._only_tool({"type": "memory_20250818", "name": "memory"})
assert set(self._enum(tool, "command")) == {"view", "create", "str_replace", "insert", "delete", "rename"}
assert {"old_path", "new_path"} <= set(tool["parameters"]["properties"])
@pytest.mark.parametrize(
("tool_type", "enable_zoom", "has_scroll", "has_zoom"),
[
("computer_20241022", True, False, False),
("computer_20250124", True, True, False),
("computer_20251124", False, True, False),
("computer_20251124", True, True, True),
],
)
def test_computer_actions_follow_the_tool_version(
self, tool_type: str, enable_zoom: bool, has_scroll: bool, has_zoom: bool
):
tool: Final = self._only_tool(
{
"type": tool_type,
"name": "computer",
"display_width_px": 1280,
"display_height_px": 800,
"enable_zoom": enable_zoom,
}
)
actions: Final = self._enum(tool, "action")
properties: Final = tool["parameters"]["properties"]
assert {"screenshot", "left_click", "type", "key"} <= set(actions)
assert ("scroll" in actions) is has_scroll
assert ("scroll_direction" in properties) is has_scroll
assert ("zoom" in actions) is has_zoom
assert ("region" in properties) is has_zoom
assert "1280x800" in tool["description"]
@pytest.mark.parametrize(
"tool_type", ["web_search_20250305", "web_search_preview", "web_search_preview_2025_03_11", "web_search_2025_08_26"]
)
def test_every_web_search_type_runs_the_hosted_web_search(self, tool_type: str):
assert self._only_tool({"type": tool_type, "name": "web_search"}) == {"type": "web_search_preview"}
def test_custom_tool_named_like_a_typed_tool_keeps_its_own_schema(self):
schema: Final = {"type": "object", "properties": {"cmd": {"type": "string"}}, "required": ["cmd"]}
tool: Final = self._only_tool({"name": "bash", "input_schema": schema})
assert tool["parameters"] == schema
assert "description" not in tool
@pytest.mark.parametrize(
"tool",
[
{"type": "web_fetch_20250910", "name": "web_fetch"},
{"type": "code_execution_20250825", "name": "code_execution"},
{"type": "mcp_toolset", "mcp_server_name": "docs"},
{"type": "computer_toolset_20260801"},
_TOOL_SEARCH,
],
)
def test_tool_with_no_responses_equivalent_is_a_400_without_drop_params(self, tool: Dict[str, Any]):
with pytest.raises(UnsupportedParamsError) as error:
_ADAPTER.translate_request(_make_request(tools=[tool]), drop_params=False)
assert error.value.status_code == 400
assert tool["type"] in str(error.value)
assert "litellm_params" in str(error.value)
def test_tool_with_no_responses_equivalent_drops_under_drop_params(self):
kwargs: Final = _ADAPTER.translate_request(
_make_request(
tools=[{"type": "web_fetch_20250910", "name": "web_fetch"}, {"type": "bash_20250124", "name": "bash"}]
),
drop_params=True,
)
assert [tool["name"] for tool in kwargs["tools"]] == ["bash"]
def test_tool_choice_of_a_hosted_tool_names_the_hosted_type(self):
kwargs: Final = _ADAPTER.translate_request(
_make_request(
tools=[
{"type": "bash_20250124", "name": "bash"},
{"type": "web_search_20250305", "name": "web_search"},
],
tool_choice={"type": "tool", "name": "web_search"},
)
)
assert kwargs["tool_choice"] == {"type": "web_search_preview"}
def test_tool_choice_of_a_dropped_tool_is_dropped_with_it(self):
kwargs: Final = _ADAPTER.translate_request(
_make_request(
tools=[{"type": "web_fetch_20250910", "name": "web_fetch"}, {"type": "bash_20250124", "name": "bash"}],
tool_choice={"type": "tool", "name": "web_fetch"},
),
drop_params=True,
)
assert [tool["name"] for tool in kwargs["tools"]] == ["bash"]
assert "tool_choice" not in kwargs
def test_tool_choice_is_dropped_when_every_tool_was_dropped(self):
kwargs: Final = _ADAPTER.translate_request(
_make_request(tools=[{"type": "web_fetch_20250910", "name": "web_fetch"}], tool_choice={"type": "any"}),
drop_params=True,
)
assert "tools" not in kwargs
assert "tool_choice" not in kwargs
def test_tool_choice_forcing_the_tool_search_is_a_400_without_drop_params(self):
with pytest.raises(UnsupportedParamsError) as error:
_ADAPTER.translate_request(
_make_request(
tools=[_TOOL_SEARCH, _DEFERRED_WEATHER],
tool_choice={"type": "tool", "name": "tool_search_tool_bm25"},
),
drop_params=False,
maps_tool_search=True,
)
assert error.value.status_code == 400
assert "tool_search_tool_bm25" in str(error.value)
assert "drop_params" in str(error.value)
def test_tool_choice_forcing_the_tool_search_is_dropped_under_drop_params(self):
kwargs: Final = _ADAPTER.translate_request(
_make_request(
tools=[_TOOL_SEARCH, _DEFERRED_WEATHER],
tool_choice={"type": "tool", "name": "tool_search_tool_bm25"},
),
drop_params=True,
maps_tool_search=True,
)
assert [tool["name"] for tool in kwargs["tools"]] == ["tool_search_tool_bm25", "get_weather"]
assert "tool_choice" not in kwargs
@pytest.mark.parametrize(
("tools", "maps_tool_search", "deferred"),
[
([_TOOL_SEARCH, _DEFERRED_WEATHER], True, True),
([_TOOL_SEARCH, _DEFERRED_BASH], True, True),
([_DEFERRED_WEATHER], True, None),
([_TOOL_SEARCH, _DEFERRED_WEATHER], False, None),
],
)
def test_a_tool_stays_deferred_only_beside_a_tool_search_the_target_runs(
self, tools: List[Dict[str, Any]], maps_tool_search: bool, deferred: Any
):
kwargs: Final = _ADAPTER.translate_request(
_make_request(tools=tools), drop_params=True, maps_tool_search=maps_tool_search
)
deferrable: Final = next(tool for tool in kwargs["tools"] if tool.get("type") == "function")
assert deferrable.get("defer_loading") is deferred
assert any(tool["type"] == _TOOL_SEARCH["type"] for tool in kwargs["tools"]) is (deferred is True)
def test_tool_choice_without_tools_still_reaches_the_provider(self):
kwargs: Final = _ADAPTER.translate_request(_make_request(tool_choice={"type": "any"}))
assert kwargs["tool_choice"] == "required"
# ---------------------------------------------------------------------------
# translate_thinking_to_reasoning
# ---------------------------------------------------------------------------

View file

@ -196,6 +196,95 @@ async def _sent_count_body(
return json.loads(route.calls.last.request.content)
TOOL_SEARCH_TOOLS: Final[list[dict[str, object]]] = [
{"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"},
{
"name": "get_weather",
"input_schema": {"type": "object", "properties": {"city": {"type": "string"}}},
"defer_loading": True,
},
]
UNDEFERRED_TOOL_SEARCH_TOOLS: Final[list[dict[str, object]]] = [
TOOL_SEARCH_TOOLS[0],
{key: value for key, value in TOOL_SEARCH_TOOLS[1].items() if key != "defer_loading"},
]
async def _counted_tools(
monkeypatch: pytest.MonkeyPatch,
model: str,
litellm_params: Mapping[str, object],
tools: list[dict[str, object]] = TOOL_SEARCH_TOOLS,
) -> list[tuple[object, object, object]]:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
with respx.mock:
route: Final = respx.post(f"{litellm_params.get('api_base', 'https://api.openai.com/v1')}/responses/input_tokens")
route.mock(return_value=httpx.Response(200, json={"input_tokens": 1}))
await OpenAITokenCounter().count_tokens(
model_to_use=model,
messages=[{"role": "user", "content": "What is the weather in Paris?"}],
contents=None,
deployment={"litellm_params": {"api_key": "sk-test", **litellm_params}},
request_model=f"openai/{model}",
tools=tools,
messages_format="anthropic",
)
counted: Final = json.loads(route.calls.last.request.content)["tools"]
return [(tool.get("type"), tool.get("name"), tool.get("defer_loading")) for tool in counted]
@pytest.mark.asyncio
async def test_anthropic_tool_search_is_counted_as_the_openai_tool_search_inference_sends(
monkeypatch: pytest.MonkeyPatch,
) -> None:
assert await _counted_tools(monkeypatch, "gpt-5.6-luna", {}) == [
("tool_search", None, None),
("function", "get_weather", True),
]
@pytest.mark.asyncio
@pytest.mark.parametrize(
("tools", "expected"),
[
pytest.param(
TOOL_SEARCH_TOOLS,
[("tool_search", None, None), ("function", "get_weather", True)],
id="a-tool-is-deferred",
),
pytest.param(UNDEFERRED_TOOL_SEARCH_TOOLS, [("function", "get_weather", None)], id="nothing-is-deferred"),
],
)
@pytest.mark.parametrize(
"litellm_params",
[pytest.param({}, id="without-drop-params"), pytest.param({"drop_params": True}, id="drop-params")],
)
async def test_anthropic_tool_search_is_counted_only_beside_a_deferred_tool(
monkeypatch: pytest.MonkeyPatch,
tools: list[dict[str, object]],
expected: list[tuple[object, object, object]],
litellm_params: Mapping[str, object],
) -> None:
assert await _counted_tools(monkeypatch, "gpt-5.6-luna", litellm_params, tools) == expected
@pytest.mark.asyncio
@pytest.mark.parametrize(
("model", "litellm_params"),
[
pytest.param("a-model-the-cost-map-does-not-mark-for-tool-search", {}, id="model-without-tool-search"),
pytest.param("gpt-5.6-luna", {"api_base": "https://responses.example.test/v1"}, id="other-responses-server"),
pytest.param(
"gpt-5.6-luna", {"additional_drop_params": ["tools[*].defer_loading"]}, id="deployment-drops-deferral"
),
],
)
async def test_anthropic_tool_search_is_left_out_of_the_count_where_inference_cannot_run_it(
monkeypatch: pytest.MonkeyPatch, model: str, litellm_params: Mapping[str, object]
) -> None:
assert await _counted_tools(monkeypatch, model, litellm_params) == [("function", "get_weather", None)]
@pytest.mark.asyncio
async def test_anthropic_format_sends_the_document_image_and_tool_turns_to_the_count_endpoint(
monkeypatch: pytest.MonkeyPatch,

View file

@ -603,6 +603,26 @@ def test_anthropic_messages_keep_every_block_the_chat_converter_drops() -> None:
)
def test_a_count_with_anthropic_tool_search_counts_the_tools_without_the_search() -> None:
_, _, tools, _ = OpenAICountTokensConfig.anthropic_messages_to_responses_input(
model="gpt-5.6-luna",
messages=[{"role": "user", "content": "Find a weather tool."}],
system=None,
tools=[
{"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"},
{
"name": "get_weather",
"input_schema": {"type": "object", "properties": {"city": {"type": "string"}}},
"defer_loading": True,
},
],
)
assert [(tool.get("type"), tool.get("name"), tool.get("defer_loading")) for tool in tools or ()] == [
("function", "get_weather", None)
]
def test_anthropic_list_system_becomes_joined_instruction_text() -> None:
_, instructions, _, _ = OpenAICountTokensConfig.anthropic_messages_to_responses_input(
model="gpt-4o",