mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-19 04:54:37 +00:00
fix(responses): gate tool search on the Anthropic request builder and validate deferred tool fields
Bedrock Converse builds its own request and has no tool search, so the bridge now drops the tool_search tool there with a warning instead of sending a tool the transport loses, while Bedrock invoke, Anthropic, and Vertex keep it. A replayed tool_search_call with no stored result is dropped from the Anthropic request so a previous_response_id follow-up no longer answers 400. Only srvtoolu_ ids whose name is one of Anthropic's search tools count as search calls, so a caller function named like one stays a function call. A non-boolean defer_loading or a non-list allowed_callers now answers 400 instead of 500
This commit is contained in:
mateo-berri
parent
b3b78d5d9d
commit
e7b738585b
9 changed files with 207 additions and 23 deletions
|
|
@ -1815,7 +1815,8 @@ def convert_function_to_anthropic_tool_invoke(
|
|||
raise e
|
||||
|
||||
|
||||
ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX: Final = "srvtoolu_"
|
||||
def is_anthropic_tool_search_use(tool_id: str, tool_name: str) -> bool:
|
||||
return tool_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX) and tool_name in ANTHROPIC_TOOL_SEARCH_TOOL_NAMES
|
||||
|
||||
|
||||
def find_anthropic_server_tool_result(
|
||||
|
|
@ -1910,6 +1911,8 @@ def convert_to_anthropic_tool_invoke(
|
|||
}
|
||||
)
|
||||
anthropic_tool_invoke.append(server_tool_result)
|
||||
elif is_anthropic_tool_search_use(tool_id, tool_name):
|
||||
continue
|
||||
else:
|
||||
sanitized_tool_id = _sanitize_anthropic_tool_use_id(tool_id)
|
||||
_anthropic_tool_use_param = AnthropicMessagesToolUseParam(
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ from typing import Final
|
|||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.llms.openai import (
|
||||
ChatCompletionToolParam,
|
||||
|
|
@ -159,14 +160,24 @@ class _CustomToolFormat(LiteLLMBaseModel):
|
|||
definition: str = ""
|
||||
|
||||
|
||||
_ALLOWED_CALLERS_ADAPTER: Final = TypeAdapter(list[str] | None)
|
||||
_ALLOWED_CALLERS_ADAPTER: Final = TypeAdapter[list[str] | None](list[str] | None)
|
||||
|
||||
|
||||
def validated_allowed_callers(value: object) -> list[str] | None:
|
||||
try:
|
||||
return _ALLOWED_CALLERS_ADAPTER.validate_python(value, strict=True)
|
||||
except ValidationError as exc:
|
||||
raise ValueError("allowed_callers must be a list of strings") from exc
|
||||
raise BadRequestError(message="allowed_callers must be a list of strings", model="", llm_provider="") from exc
|
||||
|
||||
|
||||
_DEFER_LOADING_ADAPTER: Final = TypeAdapter[bool | None](bool | None)
|
||||
|
||||
|
||||
def validated_defer_loading(value: object) -> bool:
|
||||
try:
|
||||
return bool(_DEFER_LOADING_ADAPTER.validate_python(value, strict=True))
|
||||
except ValidationError as exc:
|
||||
raise BadRequestError(message="defer_loading must be a boolean", model="", llm_provider="") from exc
|
||||
|
||||
|
||||
def custom_tool_grammar_suffix(fmt: object) -> str:
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ from litellm.responses.litellm_completion_transformation.reasoning_items import
|
|||
from litellm.responses.litellm_completion_transformation.tool_search import (
|
||||
build_tool_search_call,
|
||||
has_server_tool_search,
|
||||
is_tool_search_call_name,
|
||||
is_anthropic_tool_search_use,
|
||||
)
|
||||
from litellm.responses.litellm_completion_transformation.transformation import (
|
||||
LiteLLMCompletionResponsesConfig,
|
||||
|
|
@ -170,14 +170,14 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
self._web_search_calls: dict[str, object] = {} # mutable-ok: latest call by provider id
|
||||
self._queued_web_search_call_ids: set[str] = set() # mutable-ok: emitted call ids
|
||||
self._tool_search_requested: bool = has_server_tool_search(request_tools)
|
||||
self._tool_search_call_ids: set[str] = set() # mutable-ok: provider call ids seen as tool search calls
|
||||
self._tool_search_call_ids: frozenset[str] = frozenset()
|
||||
|
||||
def _is_tool_search_call(self, call_id: str, fn_name: str) -> bool:
|
||||
if call_id in self._tool_search_call_ids:
|
||||
return True
|
||||
if not self._tool_search_requested or not is_tool_search_call_name(fn_name):
|
||||
if not self._tool_search_requested or not is_anthropic_tool_search_use(call_id, fn_name):
|
||||
return False
|
||||
self._tool_search_call_ids.add(call_id)
|
||||
self._tool_search_call_ids = self._tool_search_call_ids | {call_id}
|
||||
return True
|
||||
|
||||
def _queue_output_item_added(self, output_index: int, item: BaseLiteLLMOpenAIResponseObject) -> None:
|
||||
|
|
|
|||
|
|
@ -3,12 +3,27 @@ from typing import Final, Literal, TypeAlias
|
|||
|
||||
from pydantic import ConfigDict, TypeAdapter, ValidationError
|
||||
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import is_anthropic_tool_search_use
|
||||
from litellm.types.llms.anthropic import AnthropicToolSearchToolRegex
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.responses.main import ResponseToolSearchCall
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
from .custom_tools import openai_shaped_tool_call_item_id
|
||||
|
||||
__all__ = [
|
||||
"ANTHROPIC_TOOL_SEARCH_TOOL_NAME",
|
||||
"ANTHROPIC_TOOL_SEARCH_TOOL_TYPE",
|
||||
"ToolSearchCallStatus",
|
||||
"anthropic_tool_search_chat_tool",
|
||||
"build_tool_search_call",
|
||||
"has_server_tool_search",
|
||||
"is_anthropic_tool_search_chat_tool",
|
||||
"is_anthropic_tool_search_use",
|
||||
"is_server_tool_search_tool",
|
||||
"provider_builds_anthropic_request",
|
||||
]
|
||||
|
||||
ANTHROPIC_TOOL_SEARCH_TOOL_TYPE: Final = "tool_search_tool_regex_20251119"
|
||||
ANTHROPIC_TOOL_SEARCH_TOOL_NAME: Final = "tool_search_tool_regex"
|
||||
ToolSearchCallStatus: TypeAlias = Literal["in_progress", "completed", "incomplete"]
|
||||
|
|
@ -43,8 +58,18 @@ def is_anthropic_tool_search_chat_tool(tool: Mapping[str, object]) -> bool:
|
|||
return tool.get("type") == ANTHROPIC_TOOL_SEARCH_TOOL_TYPE
|
||||
|
||||
|
||||
def is_tool_search_call_name(tool_name: str) -> bool:
|
||||
return tool_name == ANTHROPIC_TOOL_SEARCH_TOOL_NAME
|
||||
def provider_builds_anthropic_request(model: str, custom_llm_provider: str | None) -> bool:
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
_, provider, _, _ = get_llm_provider(model=model, custom_llm_provider=custom_llm_provider)
|
||||
try:
|
||||
llm_provider: Final = LlmProviders(provider)
|
||||
except ValueError:
|
||||
return False
|
||||
chat_config: Final = ProviderConfigManager.get_provider_chat_config(model=model, provider=llm_provider)
|
||||
return isinstance(chat_config, AnthropicConfig)
|
||||
|
||||
|
||||
def _parsed_arguments(arguments_json: str) -> Mapping[str, object] | None:
|
||||
|
|
|
|||
|
|
@ -111,6 +111,7 @@ from .custom_tools import (
|
|||
serialize_tool_call_arguments,
|
||||
unwrap_custom_tool_arguments,
|
||||
validated_allowed_callers,
|
||||
validated_defer_loading,
|
||||
)
|
||||
from .reasoning_items import decode_thinking_blocks, encode_thinking_blocks, mint_reasoning_item_id
|
||||
from .tool_search import (
|
||||
|
|
@ -118,8 +119,9 @@ from .tool_search import (
|
|||
build_tool_search_call,
|
||||
has_server_tool_search,
|
||||
is_anthropic_tool_search_chat_tool,
|
||||
is_anthropic_tool_search_use,
|
||||
is_server_tool_search_tool,
|
||||
is_tool_search_call_name,
|
||||
provider_builds_anthropic_request,
|
||||
)
|
||||
|
||||
NamespaceNameMap: TypeAlias = Mapping[str, tuple[str, str]]
|
||||
|
|
@ -388,11 +390,13 @@ class LiteLLMCompletionResponsesConfig:
|
|||
|
||||
if not any(is_anthropic_tool_search_chat_tool(tool) for tool in tools):
|
||||
return tools
|
||||
if supports_tool_search(model=model, custom_llm_provider=custom_llm_provider):
|
||||
if supports_tool_search(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
) and provider_builds_anthropic_request(model, custom_llm_provider):
|
||||
return tools
|
||||
verbose_logger.warning(
|
||||
"Dropping Responses API tool of type 'tool_search': model '%s' (provider '%s') does not support "
|
||||
"tool search, so the target provider would reject the request.",
|
||||
"Dropping Responses API tool of type 'tool_search': model '%s' (provider '%s') cannot run tool search "
|
||||
"on this route, so the target provider would reject or ignore it.",
|
||||
model,
|
||||
custom_llm_provider,
|
||||
)
|
||||
|
|
@ -1984,7 +1988,7 @@ class LiteLLMCompletionResponsesConfig:
|
|||
strict=bool(namespace_tool.get("strict", False)),
|
||||
)
|
||||
allowed_callers: Final = validated_allowed_callers(namespace_tool.get("allowed_callers"))
|
||||
deferred: Final = bool(namespace_tool.get("defer_loading"))
|
||||
deferred: Final = validated_defer_loading(namespace_tool.get("defer_loading"))
|
||||
if allowed_callers is None and not deferred:
|
||||
return ChatCompletionToolParam(type="function", function=function)
|
||||
if allowed_callers is None:
|
||||
|
|
@ -2076,8 +2080,8 @@ class LiteLLMCompletionResponsesConfig:
|
|||
}
|
||||
if tool.get("cache_control"):
|
||||
chat_completion_tool["cache_control"] = tool.get("cache_control")
|
||||
if tool.get("defer_loading"):
|
||||
chat_completion_tool["defer_loading"] = tool.get("defer_loading")
|
||||
if validated_defer_loading(tool.get("defer_loading")):
|
||||
chat_completion_tool["defer_loading"] = True
|
||||
if tool.get("allowed_callers"):
|
||||
chat_completion_tool["allowed_callers"] = tool.get("allowed_callers")
|
||||
if tool.get("input_examples"):
|
||||
|
|
@ -2325,7 +2329,7 @@ class LiteLLMCompletionResponsesConfig:
|
|||
web_search_call: Final = web_search_calls.get(tool_id)
|
||||
if web_search_call is not None:
|
||||
return web_search_call
|
||||
if tool_search_requested and is_tool_search_call_name(tool_name):
|
||||
if tool_search_requested and is_anthropic_tool_search_use(tool_id, tool_name):
|
||||
return build_tool_search_call(tool_id, tool_arguments, "completed")
|
||||
return None
|
||||
|
||||
|
|
|
|||
|
|
@ -772,6 +772,8 @@ ANTHROPIC_TOOL_SEARCH_BETA_HEADER: Final = "advanced-tool-use-2025-11-20"
|
|||
ANTHROPIC_TOOL_SEARCH_TOOL_TYPES: Final = frozenset(
|
||||
{"tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119"}
|
||||
)
|
||||
ANTHROPIC_TOOL_SEARCH_TOOL_NAMES: Final = frozenset({"tool_search_tool_regex", "tool_search_tool_bm25"})
|
||||
ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX: Final = "srvtoolu_"
|
||||
|
||||
# Effort beta header constant
|
||||
ANTHROPIC_EFFORT_BETA_HEADER: Final = "effort-2025-11-24"
|
||||
|
|
|
|||
|
|
@ -7077,3 +7077,34 @@ def test_completion_bedrock_invalid_role_exception(monkeypatch):
|
|||
str(exc_info.value)
|
||||
== "litellm.BadRequestError: Invalid Message passed in {'role': 'very-bad-role', 'content': 'hello'}"
|
||||
)
|
||||
|
||||
|
||||
def test_anthropic_messages_pt_drops_replayed_tool_search_call_without_its_result():
|
||||
messages: Final = [
|
||||
{"role": "user", "content": "What is the weather in Paris?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "srvtoolu_01Search",
|
||||
"type": "function",
|
||||
"function": {"name": "tool_search_tool_regex", "arguments": '{"pattern": "weather"}'},
|
||||
},
|
||||
{
|
||||
"id": "toolu_01Weather",
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "arguments": '{"city": "Paris"}'},
|
||||
},
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "toolu_01Weather", "content": "18C and cloudy"},
|
||||
{"role": "user", "content": "And tomorrow?"},
|
||||
]
|
||||
|
||||
result: Final = anthropic_messages_pt(messages=messages, model="claude-sonnet-5-5", llm_provider="anthropic")
|
||||
|
||||
assert [(block["type"], block["id"]) for block in result[1]["content"]] == [("tool_use", "toolu_01Weather")]
|
||||
assert [block["tool_use_id"] for block in result[2]["content"] if block["type"] == "tool_result"] == [
|
||||
"toolu_01Weather"
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1694,7 +1694,9 @@ def _deferred_tools_with_server_tool_search() -> list[dict[str, object]]:
|
|||
]
|
||||
|
||||
|
||||
def _chat_response_with_tool_search_call(search_arguments: str) -> ModelResponse:
|
||||
def _chat_response_with_tool_search_call(
|
||||
search_arguments: str, search_call_id: str = "srvtoolu_01Search"
|
||||
) -> ModelResponse:
|
||||
return ModelResponse(
|
||||
id="chatcmpl-tool-search",
|
||||
created=1748575031,
|
||||
|
|
@ -1709,7 +1711,7 @@ def _chat_response_with_tool_search_call(search_arguments: str) -> ModelResponse
|
|||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "srvtoolu_01Search",
|
||||
"id": search_call_id,
|
||||
"type": "function",
|
||||
"function": {"name": "tool_search_tool_regex", "arguments": search_arguments},
|
||||
},
|
||||
|
|
@ -1883,18 +1885,75 @@ class TestToolTransformation:
|
|||
assert [tool["defer_loading"] for tool in chat_tools[1:]] == [True, True]
|
||||
|
||||
def test_server_tool_search_dropped_for_models_without_tool_search(self) -> None:
|
||||
model: Final = "gemini/gemini-2.5-flash"
|
||||
model: Final = "openai/model-with-no-cost-map-entry"
|
||||
assert litellm.supports_tool_search(model) is False
|
||||
|
||||
result: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
|
||||
model=model,
|
||||
input="What is the weather in Paris right now?",
|
||||
responses_api_request={"tools": _deferred_tools_with_server_tool_search()},
|
||||
custom_llm_provider="gemini",
|
||||
custom_llm_provider="openai",
|
||||
)
|
||||
|
||||
assert [tool["type"] for tool in result["tools"]] == ["function", "function"]
|
||||
|
||||
def test_server_tool_search_follows_the_bedrock_route(self) -> None:
|
||||
transform: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request
|
||||
converse: Final = transform(
|
||||
model="bedrock/anthropic.claude-sonnet-5-5",
|
||||
input="What is the weather in Paris right now?",
|
||||
responses_api_request={"tools": _deferred_tools_with_server_tool_search()},
|
||||
custom_llm_provider="bedrock",
|
||||
)
|
||||
invoke: Final = transform(
|
||||
model="bedrock/invoke/anthropic.claude-sonnet-5-5",
|
||||
input="What is the weather in Paris right now?",
|
||||
responses_api_request={"tools": _deferred_tools_with_server_tool_search()},
|
||||
custom_llm_provider="bedrock",
|
||||
)
|
||||
|
||||
assert [tool["type"] for tool in converse["tools"]] == ["function", "function"]
|
||||
assert invoke["tools"][0] == {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"}
|
||||
|
||||
def test_caller_function_named_like_the_search_tool_stays_a_function_call(self) -> None:
|
||||
items: Final = LiteLLMCompletionResponsesConfig.transform_chat_completion_tools_to_responses_tools(
|
||||
chat_completion_response=_chat_response_with_tool_search_call(
|
||||
'{"pattern": "weather"}', search_call_id="call_01Mine"
|
||||
),
|
||||
responses_api_request={
|
||||
"tools": [
|
||||
*_deferred_tools_with_server_tool_search(),
|
||||
{"type": "function", "name": "tool_search_tool_regex", "parameters": {"type": "object"}},
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
assert [(item.type, item.name) for item in items] == [
|
||||
("function_call", "tool_search_tool_regex"),
|
||||
("function_call", "get_weather"),
|
||||
]
|
||||
|
||||
def test_defer_loading_must_be_a_boolean(self) -> None:
|
||||
transform: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools
|
||||
namespace_tool: Final = {
|
||||
"type": "namespace",
|
||||
"name": "finance",
|
||||
"tools": [{"type": "function", "name": "get_stock_price", "parameters": {"type": "object"}}],
|
||||
}
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match="defer_loading must be a boolean"):
|
||||
transform(tools=[{**namespace_tool, "tools": [{**namespace_tool["tools"][0], "defer_loading": "false"}]}])
|
||||
with pytest.raises(litellm.BadRequestError, match="defer_loading must be a boolean"):
|
||||
transform(tools=[{"type": "function", "name": "get_weather", "parameters": {}, "defer_loading": "false"}])
|
||||
not_deferred, _ = transform(
|
||||
tools=[
|
||||
{**namespace_tool, "tools": [{**namespace_tool["tools"][0], "defer_loading": False}]},
|
||||
{"type": "function", "name": "get_weather", "parameters": {}, "defer_loading": False},
|
||||
]
|
||||
)
|
||||
assert [tool["function"]["name"] for tool in not_deferred] == ["finance__get_stock_price", "get_weather"]
|
||||
assert all("defer_loading" not in tool for tool in not_deferred)
|
||||
|
||||
def test_anthropic_tool_search_call_becomes_tool_search_call_item(self) -> None:
|
||||
response: Final = _chat_response_with_tool_search_call('{"pattern": "weather|forecast"}')
|
||||
with_tool_search: Final = LiteLLMCompletionResponsesConfig.transform_chat_completion_tools_to_responses_tools(
|
||||
|
|
@ -2103,7 +2162,7 @@ class TestToolTransformation:
|
|||
|
||||
tools = [custom_tool]
|
||||
|
||||
with pytest.raises(ValueError, match="allowed_callers must be a list of strings"):
|
||||
with pytest.raises(litellm.BadRequestError, match="allowed_callers must be a list of strings"):
|
||||
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
|
||||
tools=tools
|
||||
)
|
||||
|
|
@ -2696,7 +2755,7 @@ class TestToolTransformation:
|
|||
else {**function_tool, "type": "namespace"}
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="allowed_callers must be a list of strings"):
|
||||
with pytest.raises(litellm.BadRequestError, match="allowed_callers must be a list of strings"):
|
||||
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
|
||||
tools=[namespace_tool]
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1418,3 +1418,52 @@ async def test_anthropic_tool_search_call_streams_as_tool_search_call_item(sync_
|
|||
assert {event["output_index"] for event in search_events}.isdisjoint(
|
||||
{event["output_index"] for event in function_events}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sync_mode", [True, False])
|
||||
@pytest.mark.asyncio
|
||||
async def test_caller_function_named_like_the_search_tool_streams_as_function_call(sync_mode: bool):
|
||||
chunk: Final = ModelResponseStream(
|
||||
id=CHAT_COMPLETION_ID,
|
||||
created=1748575031,
|
||||
model="claude-sonnet-5-5",
|
||||
object="chat.completion.chunk",
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
index=0,
|
||||
finish_reason="tool_calls",
|
||||
delta=Delta(
|
||||
content=None,
|
||||
tool_calls=[
|
||||
{
|
||||
"index": 0,
|
||||
"id": "call_01Mine",
|
||||
"type": "function",
|
||||
"function": {"name": "tool_search_tool_regex", "arguments": '{"pattern": "weather"}'},
|
||||
}
|
||||
],
|
||||
),
|
||||
)
|
||||
],
|
||||
)
|
||||
iterator: Final = LiteLLMCompletionStreamingIterator(
|
||||
model="claude-sonnet-5-5",
|
||||
litellm_custom_stream_wrapper=_FakeStreamWrapper((chunk,)),
|
||||
request_input="What is the weather in Paris right now?",
|
||||
responses_api_request={
|
||||
"tools": [
|
||||
{"type": "tool_search", "execution": "server"},
|
||||
{"type": "function", "name": "tool_search_tool_regex", "parameters": {"type": "object"}},
|
||||
]
|
||||
},
|
||||
custom_llm_provider="anthropic",
|
||||
)
|
||||
events: Final = (
|
||||
[event.model_dump(exclude_none=True) for event in iterator]
|
||||
if sync_mode
|
||||
else [event.model_dump(exclude_none=True) async for event in iterator]
|
||||
)
|
||||
output: Final = events[-1]["response"]["output"]
|
||||
|
||||
assert [(item["type"], item["name"]) for item in output] == [("function_call", "tool_search_tool_regex")]
|
||||
assert "response.function_call_arguments.done" in [event["type"] for event in events]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue