diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 95d54c9100d..2e632178272 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -197,15 +197,26 @@ def _namespace_members_without_deferral(members: object) -> object: return members +def _field_without_deferral(key: str, value: object) -> object: + match key: + case "function": + return _without_defer_loading(value) + case "tools": + return _namespace_members_without_deferral(value) + case _: + return value + + def _without_deferral(tool: object) -> object: - match _member_without_deferral(tool): - case Mapping() as stripped: + match tool: + case Mapping(): return { - key: _namespace_members_without_deferral(value) if key == "tools" else value - for key, value in _TOOL_FIELDS_ADAPTER.validate_python(stripped).items() + key: _field_without_deferral(key, value) + for key, value in _TOOL_FIELDS_ADAPTER.validate_python(tool).items() + if key != "defer_loading" } - case stripped: - return stripped + case _: + return tool def _validated_defer_loading(value: object) -> bool: @@ -370,8 +381,14 @@ DROP_UNSUPPORTED_TOOL_SEARCH_WARNING: Final = ( ) +TOOL_SEARCH_MODEL_INFO_REMEDY: Final = ( + " If this model does run tool search, set `supports_tool_search: true` in its `model_info`, or for the SDK " + "with `litellm.register_model`." +) + + def tools_without_unsupported_tool_search( - tools: Sequence[object], model: str, route: str, drop_params: bool + tools: Sequence[object], model: str, route: str, drop_params: bool, remedy: str ) -> Sequence[object]: if not any(_is_openai_tool_search_tool(tool) for tool in tools): return tools @@ -379,9 +396,9 @@ def tools_without_unsupported_tool_search( raise litellm.utils.UnsupportedParamsError( message=( f"{model} on {route} cannot run the OpenAI `tool_search` tool as sent. Only a server-executed " - "`tool_search` on a route that supports tool search becomes Anthropic's tool search. To drop it " - "and the `defer_loading` of the other tools, set `litellm.drop_params=True` or for proxy:\n\n" - "`litellm_settings:\n drop_params: true`\n" + f"`tool_search` on a route that supports tool search becomes Anthropic's tool search.{remedy} " + "To drop it and the `defer_loading` of the other tools, set `litellm.drop_params=True` or for " + "proxy:\n\n`litellm_settings:\n drop_params: true`\n" ), status_code=400, ) @@ -1107,12 +1124,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): return tools if any(_is_anthropic_tool_search_tool(tool) for tool in tools): return [tool for tool in tools if not _is_openai_tool_search_tool(tool)] - route_runs_the_search: Final = all( + model_runs_tool_search: Final = litellm.supports_tool_search(model, self._resolved_provider) + search_runs_on_the_server: Final = all( is_server_tool_search_tool(tool) for tool in tools if _is_openai_tool_search_tool(tool) - ) and litellm.supports_tool_search(model, self._resolved_provider) - if not route_runs_the_search: + ) + if not (model_runs_tool_search and search_runs_on_the_server): return tools_without_unsupported_tool_search( - tools=tools, model=model, route=self._resolved_provider, drop_params=drop_params + tools=tools, + model=model, + route=self._resolved_provider, + drop_params=drop_params, + remedy="" if model_runs_tool_search else TOOL_SEARCH_MODEL_INFO_REMEDY, ) return [ *tools[:first_search_tool_index], diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 89a8669d2a3..9079aa05b40 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -54,7 +54,7 @@ from litellm.llms.anthropic.chat.transformation import ( from litellm.llms.anthropic.common_utils import AnthropicModelInfo from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.llms.bedrock.chat.tool_result_images import place_tool_result_images -from litellm.llms.bedrock.common_utils import bedrock_model_supports_regex_lookaround +from litellm.llms.bedrock.common_utils import bedrock_model_supports_regex_lookaround, bedrock_supports_tool_search from litellm.llms.bedrock.request_metadata import ( bedrock_request_metadata_headers, bedrock_request_metadata_is_owned, @@ -138,6 +138,7 @@ UNSUPPORTED_BEDROCK_CONVERSE_BETA_PATTERNS: Final = [ _TOOLS_AS_SENT: Final = TypeAdapter(tuple[Mapping[str, object], ...]) _TOOLS_AS_RECEIVED: Final = TypeAdapter(list[object]) +CLAUDE_ON_BEDROCK_INVOKE_REMEDY: Final = " This model runs tool search on Bedrock through `bedrock/invoke/`." def _tools_the_model_accepts( @@ -1093,8 +1094,9 @@ class AmazonConverseConfig(BaseConfig): tools_without_unsupported_tool_search( tools=_TOOLS_AS_RECEIVED.validate_python(value), model=model, - route="bedrock converse", + route="bedrock", drop_params=drop_params, + remedy=CLAUDE_ON_BEDROCK_INVOKE_REMEDY if bedrock_supports_tool_search(model) else "", ) ), ), diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index c8894d03cee..bc164e17e5a 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -332,6 +332,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): model=model, route="databricks", drop_params=drop_params, + remedy="", ) ) ) # unclear how mcp tool calling on databricks works diff --git a/litellm/main.py b/litellm/main.py index 474d82e8ef1..1c1f36d1d3f 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1076,6 +1076,7 @@ from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the o ) _ANTHROPIC_ONLY_TOOL_KEYS: Final = frozenset({"input_examples", "eager_input_streaming"}) +_CHAT_HISTORY_ADAPTER: Final = TypeAdapter(Sequence[object]) def _is_claude_tool_target(custom_llm_provider: str | None, model: str) -> bool: @@ -1089,6 +1090,15 @@ def _is_claude_tool_target(custom_llm_provider: str | None, model: str) -> bool: return False +def _route_drops_tool_search_uses(messages: Sequence[object], provider_config: BaseConfig) -> bool: + from litellm.responses.litellm_completion_transformation.tool_search import ( + chat_config_builds_anthropic_request, + history_holds_tool_search_use, + ) + + return not chat_config_builds_anthropic_request(provider_config) and history_holds_tool_search_use(messages) + + def _without_anthropic_only_tool_keys(tool: dict[str, object]) -> dict[str, object]: kept: Final = {key: value for key, value in tool.items() if key not in _ANTHROPIC_ONLY_TOOL_KEYS} function: Final = tool.get("function") @@ -5557,6 +5567,15 @@ def completion( if provider_config is not None: messages = provider_config.translate_developer_role_to_system_role(messages=messages) + chat_history: Final = _CHAT_HISTORY_ADAPTER.validate_python(messages) + if _route_drops_tool_search_uses(chat_history, provider_config): + from litellm.responses.litellm_completion_transformation.tool_search import ( + chat_messages_without_tool_search_uses, + ) + + messages = list( # rebind-ok: completion() rewrites messages step by step for the provider + chat_messages_without_tool_search_uses(chat_history) + ) if ( supports_system_message is not None diff --git a/litellm/responses/litellm_completion_transformation/tool_search.py b/litellm/responses/litellm_completion_transformation/tool_search.py index b1e865f5f09..beffe371ddf 100644 --- a/litellm/responses/litellm_completion_transformation/tool_search.py +++ b/litellm/responses/litellm_completion_transformation/tool_search.py @@ -19,7 +19,10 @@ __all__ = [ "answers_tool_search", "anthropic_tool_search_chat_tool", "build_tool_search_call", + "chat_config_builds_anthropic_request", + "chat_messages_without_tool_search_uses", "has_server_tool_search", + "history_holds_tool_search_use", "holds_tool_search_use", "is_anthropic_tool_search_chat_tool", "is_anthropic_tool_search_use", @@ -35,6 +38,7 @@ ANTHROPIC_TOOL_SEARCH_TOOL_NAME: Final = "tool_search_tool_regex" ToolSearchCallStatus: TypeAlias = Literal["in_progress", "completed", "incomplete"] _ARGUMENTS_ADAPTER: Final = TypeAdapter(dict[str, object]) +_MESSAGE_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) class _ToolSearchFields(LiteLLMBaseModel): @@ -66,8 +70,6 @@ def is_anthropic_tool_search_chat_tool(tool: Mapping[str, object]) -> bool: def provider_builds_anthropic_request(model: str, custom_llm_provider: str | None) -> bool: from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider - from litellm.llms.anthropic.chat.transformation import AnthropicConfig - from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.utils import ProviderConfigManager _, provider, _, _ = get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) @@ -75,7 +77,15 @@ def provider_builds_anthropic_request(model: str, custom_llm_provider: str | Non llm_provider: Final = LlmProviders(provider) except ValueError: return False - chat_config: Final = ProviderConfigManager.get_provider_chat_config(model=model, provider=llm_provider) + return chat_config_builds_anthropic_request( + ProviderConfigManager.get_provider_chat_config(model=model, provider=llm_provider) + ) + + +def chat_config_builds_anthropic_request(chat_config: object) -> bool: + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig + return isinstance(chat_config, AnthropicConfig) and not isinstance(chat_config, OpenAIGPTConfig) @@ -167,6 +177,49 @@ def without_tool_search_uses(message: Message) -> Message | None: return message.model_copy(update={"tool_calls": list(kept) or None}) +def history_holds_tool_search_use(messages: Iterable[object]) -> bool: + return any(_is_assistant_message(message) and holds_tool_search_use(message) for message in messages) + + +def _is_assistant_message(message: object) -> bool: + match message: + case {"role": "assistant"}: + return True + case Mapping(): + return False + case _: + return _session_message_fields(message).role == "assistant" + + +def _assistant_mapping_without_tool_search_uses(message: Mapping[str, object]) -> Mapping[str, object] | None: + kept_calls: Final = without_tool_search_calls(message) + if not kept_calls and not message.get("content"): + return None + without_calls: Final = {key: value for key, value in message.items() if key != "tool_calls"} + return {**without_calls, "tool_calls": list(kept_calls)} if kept_calls else without_calls + + +def _without_tool_search_use(message: object, search_call_ids: Set[str]) -> object | None: + if answers_tool_search(message, search_call_ids): + return None + if not holds_tool_search_use(message): + return message + match message: + case Message(): + return without_tool_search_uses(message) + case Mapping(): + return _assistant_mapping_without_tool_search_uses(_MESSAGE_FIELDS_ADAPTER.validate_python(message)) + case _: + return message + + +def chat_messages_without_tool_search_uses(messages: Sequence[object]) -> tuple[object, ...]: + search_call_ids: Final = tool_search_call_ids(messages) + return tuple( + kept for message in messages if (kept := _without_tool_search_use(message, search_call_ids)) is not None + ) + + def _parsed_arguments(arguments_json: str) -> Mapping[str, object] | None: try: return _ARGUMENTS_ADAPTER.validate_json(arguments_json or "{}") diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 1b8cd059921..c85384b17de 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -29,6 +29,7 @@ from litellm.llms.anthropic.pass_through.messages.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.azure_ai.anthropic.transformation import AzureAnthropicConfig +from litellm.llms.bedrock.claude_platform.transformation import BedrockClaudePlatformConfig from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( AmazonAnthropicClaudeConfig, ) @@ -8809,6 +8810,9 @@ def _deferral_by_function_name(tools: list[dict[str, object]]) -> dict[object, o pytest.param(AmazonAnthropicClaudeConfig(), "us.anthropic.claude-sonnet-5-5", "bedrock", id="bedrock-invoke"), pytest.param(VertexAIAnthropicConfig(), "claude-sonnet-5-5", "vertex_ai", id="vertex_ai"), pytest.param(AzureAnthropicConfig(), "claude-sonnet-5-5", "azure_ai", id="azure_ai"), + pytest.param( + BedrockClaudePlatformConfig(), "claude_platform/claude-opus-5-5", "bedrock", id="bedrock-claude-platform" + ), ], ) @pytest.mark.parametrize( @@ -8916,6 +8920,25 @@ def test_map_openai_params_answers_400_for_an_openai_tool_search_tool_it_cannot_ assert raised.value.status_code == 400 assert "tool_search" in str(raised.value) assert "drop_params: true" in str(raised.value) + assert ("supports_tool_search: true" in str(raised.value)) is (execution == "server") + + +def test_map_openai_params_runs_tool_search_on_a_custom_named_deployment_whose_model_info_says_it_can(monkeypatch): + from litellm.responses.litellm_completion_transformation.tool_search import anthropic_tool_search_chat_tool + + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setattr(litellm, "model_cost", dict(litellm.model_cost)) + tools: Final = _openai_tool_search_with_deferred_functions() + with pytest.raises(litellm.utils.UnsupportedParamsError): + AzureAnthropicConfig().map_openai_params( + non_default_params={"tools": tools}, optional_params={}, model="my-claude-deployment", drop_params=False + ) + litellm.register_model({"azure_ai/my-claude-deployment": {"supports_tool_search": True, "litellm_provider": "azure_ai"}}) + mapped: Final = AzureAnthropicConfig().map_openai_params( + non_default_params={"tools": tools}, optional_params={}, model="my-claude-deployment", drop_params=False + ) + assert _search_tools_sent(mapped["tools"]) == [anthropic_tool_search_chat_tool()] + assert _deferral_by_function_name(mapped["tools"]) == dict.fromkeys(_DEFERRED_FUNCTION_NAMES, True) @pytest.mark.parametrize(("config", "model", "provider", "execution"), _UNRUNNABLE_TOOL_SEARCH_CASES) diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 249acfee7c3..4c78c966525 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -10166,17 +10166,29 @@ def _openai_tool_search_with_deferred_functions() -> list[dict[str, object]]: ] -def test_map_openai_params_answers_400_for_the_openai_tool_search_tool_without_drop_params(monkeypatch): +@pytest.mark.parametrize( + ("model", "names_invoke_route"), + [ + pytest.param("us.anthropic.claude-sonnet-5-5", True, id="claude-that-runs-tool-search-on-invoke"), + pytest.param("anthropic.claude-3-haiku-20240307-v1:0", False, id="claude-without-tool-search"), + pytest.param("us.amazon.nova-pro-v1:0", False, id="nova"), + ], +) +def test_map_openai_params_answers_400_for_the_openai_tool_search_tool_without_drop_params( + monkeypatch, model, names_invoke_route +): monkeypatch.setattr(litellm, "drop_params", False) with pytest.raises(litellm.utils.UnsupportedParamsError) as raised: AmazonConverseConfig().map_openai_params( non_default_params={"tools": _openai_tool_search_with_deferred_functions()}, optional_params={}, - model="us.anthropic.claude-sonnet-5-5", + model=model, drop_params=False, ) assert raised.value.status_code == 400 assert "tool_search" in str(raised.value) + assert "drop_params: true" in str(raised.value) + assert ("bedrock/invoke/" in str(raised.value)) is names_invoke_route def test_map_openai_params_drops_the_openai_tool_search_tool_and_every_deferral_under_drop_params(monkeypatch): diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 635ee98549e..8bef1cbe20b 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -415,6 +415,135 @@ def test_completion_strips_eager_input_streaming_before_openai(respx_mock: respx assert sent_tool["function"]["name"] == "write_file" +_CLAUDE_TOOL_SEARCH_CALL: Final = { + "id": "srvtoolu_01search", + "type": "function", + "function": {"name": "tool_search_tool_regex", "arguments": '{"pattern": "weather"}'}, +} +_LOADED_FUNCTION_CALL: Final = { + "id": "toolu_01weather", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, +} +_USER_TURN: Final = {"role": "user", "content": "What is the weather in Paris?"} +_WEATHER_RESULT: Final = {"role": "tool", "tool_call_id": "toolu_01weather", "content": "18C"} + + +@pytest.mark.parametrize( + ("claude_turn", "sent_after_user"), + [ + pytest.param( + [ + {"role": "assistant", "content": None, "tool_calls": [_CLAUDE_TOOL_SEARCH_CALL, _LOADED_FUNCTION_CALL]}, + _WEATHER_RESULT, + ], + [{"role": "assistant", "tool_calls": [_LOADED_FUNCTION_CALL]}, _WEATHER_RESULT], + id="search-beside-a-function-call", + ), + pytest.param( + [ + litellm.Message(content="Looking", tool_calls=[_CLAUDE_TOOL_SEARCH_CALL, _LOADED_FUNCTION_CALL]), + _WEATHER_RESULT, + ], + [{"role": "assistant", "content": "Looking", "tool_calls": [_LOADED_FUNCTION_CALL]}, _WEATHER_RESULT], + id="sdk-message-object", + ), + pytest.param( + [ + {"role": "assistant", "content": None, "tool_calls": [_CLAUDE_TOOL_SEARCH_CALL]}, + {"role": "tool", "tool_call_id": "srvtoolu_01search", "content": "searched"}, + ], + [], + id="answered-search-alone", + ), + ], +) +def test_completion_drops_claude_tool_search_calls_from_history_sent_to_openai( + respx_mock: respx.MockRouter, openai_api_response, claude_turn, sent_after_user +): + api_base: Final = "http://localhost:12346/v1" + respx_mock.post(url__regex=rf"{api_base}/chat/completions.*").mock( + return_value=httpx.Response(status_code=200, json=openai_api_response) + ) + + litellm.completion( + model="openai/gpt-5.6", + messages=[_USER_TURN, *claude_turn, {"role": "user", "content": "And tomorrow?"}], + api_base=api_base, + api_key="fake_openai_api_key", + ) + + sent: Final = json.loads(respx_mock.calls[0].request.content)["messages"] + assert [ + {key: message[key] for key in ("role", "content", "tool_calls", "tool_call_id") if key in message} + for message in sent + ] == [_USER_TURN, *sent_after_user, {"role": "user", "content": "And tomorrow?"}] + + +def test_completion_drops_claude_tool_search_calls_before_the_responses_bridge_sends_history_to_openai( + respx_mock: respx.MockRouter, +): + route: Final = respx_mock.post("https://api.openai.com/v1/responses").respond( + json={ + "id": "resp_turn2", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-5.6", + "output": [ + { + "type": "message", + "id": "msg_turn2", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "18C and clear", "annotations": []}], + } + ], + "usage": {"input_tokens": 40, "output_tokens": 5, "total_tokens": 45}, + } + ) + + litellm.completion( + model="openai/gpt-5.6", + api_key="sk-test", + api_base="https://api.openai.com/v1", + messages=[ + _USER_TURN, + {"role": "assistant", "content": None, "tool_calls": [_CLAUDE_TOOL_SEARCH_CALL, _LOADED_FUNCTION_CALL]}, + {"role": "tool", "tool_call_id": "srvtoolu_01search", "content": "searched"}, + _WEATHER_RESULT, + ], + tools=[{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}], + ) + + sent_input: Final = json.loads(route.calls.last.request.content)["input"] + assert [(item.get("type"), item.get("call_id")) for item in sent_input if "call_id" in item] == [ + ("function_call", "toolu_01weather"), + ("function_call_output", "toolu_01weather"), + ] + + +def test_completion_keeps_a_function_call_whose_id_only_looks_like_a_claude_server_tool( + respx_mock: respx.MockRouter, openai_api_response +): + api_base: Final = "http://localhost:12346/v1" + respx_mock.post(url__regex=rf"{api_base}/chat/completions.*").mock( + return_value=httpx.Response(status_code=200, json=openai_api_response) + ) + lookalike_call: Final = {**_LOADED_FUNCTION_CALL, "id": "srvtoolu_01weather"} + history: Final = [ + _USER_TURN, + {"role": "assistant", "content": None, "tool_calls": [lookalike_call]}, + {"role": "tool", "tool_call_id": "srvtoolu_01weather", "content": "18C"}, + ] + + litellm.completion(model="openai/gpt-5.6", messages=history, api_base=api_base, api_key="fake_openai_api_key") + + sent: Final = json.loads(respx_mock.calls[0].request.content)["messages"] + assert [message["role"] for message in sent] == ["user", "assistant", "tool"] + assert sent[1]["tool_calls"][0]["id"] == "srvtoolu_01weather" + + def test_embedding_keeps_an_internal_prefixed_kwarg_out_of_the_provider_request(respx_mock: respx.MockRouter) -> None: api_base: Final = "http://localhost:12346/v1" mock_route: Final = respx_mock.post(url__regex=rf"{api_base}/embeddings.*").mock(