diff --git a/litellm/constants.py b/litellm/constants.py index f9389d22dea..46b41fda13d 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -172,6 +172,11 @@ MCP_PER_USER_TOKEN_DEFAULT_TTL: Final = int( ) MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS: Final = int(os.getenv("MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS", "60")) +# Sentinel server name for the mcp_tool_search / mcp_tool_call virtual tools, +# stored in the Responses API tool_server_map to tell them apart from a real +# upstream tool of the same name. +MCP_VIRTUAL_TOOL_SEARCH_SERVER_NAME = "litellm_virtual_mcp_tool_search" + # MCP timeout defaults (seconds). Override via env vars for slow/custom MCP servers. MCP_CLIENT_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_CLIENT_TIMEOUT", "60.0")) MCP_TOOL_LISTING_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_TOOL_LISTING_TIMEOUT", "30.0")) diff --git a/litellm/proxy/_experimental/mcp_server/AGENTS.md b/litellm/proxy/_experimental/mcp_server/AGENTS.md index 6e1d121c3be..5c582f13c2c 100644 --- a/litellm/proxy/_experimental/mcp_server/AGENTS.md +++ b/litellm/proxy/_experimental/mcp_server/AGENTS.md @@ -85,6 +85,10 @@ module materially harder to understand. permissions, no-accessible-server rejection, per-request auth headers, server scope, error to `isError` conversion, and spend logging. Reuse `_list_mcp_tools` and `execute_mcp_tool` rather than reimplementing any of these checks. +- The same flag also drives the Responses API / chat completions `type: "mcp"` + tools that point at `litellm_proxy/`. That bridge lives in + `litellm/responses/mcp/tool_search_bridge.py` and calls `handle_mcp_tool_search` + / `handle_mcp_tool_call` here; keep those two entry points backwards compatible. ## Tests diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 7a210cdd970..209710e5174 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -59,6 +59,8 @@ from litellm.utils import ( if TYPE_CHECKING: from fastapi import WebSocket from mcp.types import Tool as MCPTool + + from litellm.responses.mcp.litellm_proxy_mcp_handler import MCPToolResult else: MCPTool = Any @@ -173,6 +175,7 @@ async def aresponses_api_with_mcp( from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, ) + from litellm.responses.mcp.tool_search_bridge import is_mcp_tool_search_enabled # Parse MCP tools and separate from other tools ( @@ -210,6 +213,12 @@ async def aresponses_api_with_mcp( ) openai_tools: Final = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai(original_mcp_tools) + virtual_tool_scope = ( + LiteLLM_Proxy_MCP_Handler._build_virtual_tool_scope(mcp_tools_with_litellm_proxy) + if is_mcp_tool_search_enabled(user_api_key_auth) + else None + ) + # Combine with other tools all_tools: Final = openai_tools + other_tools if (openai_tools or other_tools) else None @@ -299,105 +308,104 @@ async def aresponses_api_with_mcp( # Auto-Execute Tools Handling # If auto-execute tools is True, then we need to execute the tool calls ######################################################### - if should_auto_execute and isinstance(response, ResponsesAPIResponse): - tool_calls: Final = LiteLLM_Proxy_MCP_Handler._extract_tool_calls_from_response(response=response) + if not (should_auto_execute and isinstance(response, ResponsesAPIResponse)): + return response - if tool_calls: - user_api_key_auth = kwargs.get("litellm_metadata", {}).get("user_api_key_auth") + from litellm.responses.mcp.tool_search_bridge import max_tool_call_rounds - # Extract MCP auth headers from the request to pass to MCP server - secret_fields = kwargs.get("secret_fields") - ( - mcp_auth_header, - mcp_server_auth_headers, - oauth2_headers, - raw_headers_from_request, - ) = ResponsesAPIRequestUtils.extract_mcp_headers_from_request( - secret_fields=secret_fields, - tools=tools, + max_rounds = max_tool_call_rounds(virtual_tool_scope) + persistence_disabled: Final = LiteLLM_Proxy_MCP_Handler._is_persistence_disabled(call_params) + + # Extract MCP auth headers from the request to pass to MCP server + ( + mcp_auth_header, + mcp_server_auth_headers, + oauth2_headers, + raw_headers_from_request, + ) = ResponsesAPIRequestUtils.extract_mcp_headers_from_request( + secret_fields=kwargs.get("secret_fields"), + tools=tools, + ) + + current_response: ResponsesAPIResponse = response + conversation_input: Any = input + all_tool_results: Final[list[MCPToolResult]] = [] # mutable-ok: round accumulator + completed_rounds = 0 + + while completed_rounds < max_rounds: + tool_calls = LiteLLM_Proxy_MCP_Handler._extract_tool_calls_from_response(response=current_response) + if not tool_calls: + break + + tool_results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( + tool_server_map=tool_server_map, + tool_calls=tool_calls, + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers_from_request, + litellm_call_id=kwargs.get("litellm_call_id"), + litellm_trace_id=kwargs.get("litellm_trace_id"), + request_tags=LiteLLM_Proxy_MCP_Handler._get_parent_request_tags(kwargs), + virtual_tool_scope=virtual_tool_scope, + ) + if not tool_results: + break + + completed_rounds += 1 + all_tool_results.extend(tool_results) + + follow_up_input = LiteLLM_Proxy_MCP_Handler._create_follow_up_input( + response=current_response, + tool_results=tool_results, + original_input=conversation_input, + preserve_reasoning=persistence_disabled, + ) + + follow_up_call_params = LiteLLM_Proxy_MCP_Handler._prepare_follow_up_call_params( + call_params=call_params, original_stream_setting=stream or False + ) + + follow_up_tools = all_tools if completed_rounds < max_rounds else None + if follow_up_tools is None: + verbose_logger.warning( + "MCP auto-execute hit its %s-round cap; forcing a text-only follow-up.", + max_rounds, ) - tool_results: Final = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( - tool_server_map=tool_server_map, - tool_calls=tool_calls, - user_api_key_auth=user_api_key_auth, - mcp_auth_header=mcp_auth_header, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers_from_request, - litellm_call_id=kwargs.get("litellm_call_id"), - litellm_trace_id=kwargs.get("litellm_trace_id"), - request_tags=LiteLLM_Proxy_MCP_Handler._get_parent_request_tags(kwargs), - ) + follow_up_response = await LiteLLM_Proxy_MCP_Handler._make_follow_up_call( + follow_up_input=follow_up_input, + model=model, + all_tools=follow_up_tools, + response_id=previous_response_id if persistence_disabled else current_response.id, + **follow_up_call_params, + ) + if not isinstance(follow_up_response, ResponsesAPIResponse): + return follow_up_response - if tool_results: - persistence_disabled: Final = LiteLLM_Proxy_MCP_Handler._is_persistence_disabled(call_params) + conversation_input = follow_up_input + current_response = follow_up_response - follow_up_input: Final = LiteLLM_Proxy_MCP_Handler._create_follow_up_input( - response=response, - tool_results=tool_results, - original_input=input, - preserve_reasoning=persistence_disabled, - ) + if not all_tool_results: + return response - # Prepare parameters for follow-up call (restores original stream setting) - follow_up_call_params: Final = LiteLLM_Proxy_MCP_Handler._prepare_follow_up_call_params( - call_params=call_params, original_stream_setting=stream or False - ) - - # Create tool execution events for streaming if needed - tool_execution_events = [] - if stream: - tool_execution_events = LiteLLM_Proxy_MCP_Handler._create_tool_execution_events( - tool_calls=tool_calls, tool_results=tool_results - ) - - final_response = await LiteLLM_Proxy_MCP_Handler._make_follow_up_call( - follow_up_input=follow_up_input, - model=model, - all_tools=all_tools, - response_id=previous_response_id if persistence_disabled else response.id, - **follow_up_call_params, - ) - - # If streaming and we have tool execution events, wrap the response - if ( - stream - and tool_execution_events - and (hasattr(final_response, "__aiter__") or hasattr(final_response, "__iter__")) - ): - from litellm.responses.mcp.mcp_streaming_iterator import ( - MCPEnhancedStreamingIterator, - ) - - final_response = MCPEnhancedStreamingIterator( - tool_server_map=tool_server_map, - base_iterator=final_response, - mcp_events=tool_execution_events, - user_api_key_auth=user_api_key_auth, - ) - - # Add custom output elements to the final response (for non-streaming) - elif isinstance(final_response, ResponsesAPIResponse): - # Fetch MCP tools again for output elements (without OpenAI transformation) - ( - mcp_tools_for_output, - _, - ) = await LiteLLM_Proxy_MCP_Handler._process_mcp_tools_without_openai_transform( - user_api_key_auth=user_api_key_auth, - mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy, - mcp_auth_header=mcp_auth_header, - mcp_server_auth_headers=mcp_server_auth_headers, - request_tags=LiteLLM_Proxy_MCP_Handler._get_parent_request_tags(kwargs), - ) - final_response = LiteLLM_Proxy_MCP_Handler._add_mcp_output_elements_to_response( - response=final_response, - mcp_tools_fetched=mcp_tools_for_output, - tool_results=tool_results, - ) - return final_response - - return response + # Fetch MCP tools again for output elements (without OpenAI transformation) + ( + mcp_tools_for_output, + _, + ) = await LiteLLM_Proxy_MCP_Handler._process_mcp_tools_without_openai_transform( + user_api_key_auth=user_api_key_auth, + mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + request_tags=LiteLLM_Proxy_MCP_Handler._get_parent_request_tags(kwargs), + ) + return LiteLLM_Proxy_MCP_Handler._add_mcp_output_elements_to_response( + response=current_response, + mcp_tools_fetched=mcp_tools_for_output, + tool_results=all_tool_results, + ) def _bridges_to_chat_completions( diff --git a/litellm/responses/mcp/chat_completions_handler.py b/litellm/responses/mcp/chat_completions_handler.py index ae18d5f6f1b..b7c10c6c341 100644 --- a/litellm/responses/mcp/chat_completions_handler.py +++ b/litellm/responses/mcp/chat_completions_handler.py @@ -5,15 +5,21 @@ from typing import TYPE_CHECKING, Final, cast from typing_extensions import TypedDict, Unpack +from litellm._logging import verbose_logger from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, ) from litellm.responses.mcp.request_context import MCPRequestContext +from litellm.responses.mcp.tool_search_bridge import ( + is_mcp_tool_search_enabled, + max_tool_call_rounds, +) from litellm.types.utils import Message, ModelResponse from litellm.utils import CustomStreamWrapper if TYPE_CHECKING: from litellm.proxy._types import UserAPIKeyAuth + from litellm.responses.mcp.litellm_proxy_mcp_handler import MCPToolResult class _MCPCompletionKwargs(TypedDict, total=False, extra_items=object): @@ -126,6 +132,14 @@ async def acompletion_with_mcp( oauth2_headers: Final = context.oauth2_headers raw_headers: Final = context.raw_headers + stream: Final[object] = kwargs.get("stream", False) + + if stream and is_mcp_tool_search_enabled(user_api_key_auth): + verbose_logger.debug( + "mcp_tool_search_enabled is set but streaming /chat/completions only supports a single " + "tool-call round; falling back to the full catalog." + ) + # Process MCP tools (pass auth headers for dynamic auth) ( deduplicated_mcp_tools, @@ -137,6 +151,13 @@ async def acompletion_with_mcp( mcp_auth_header=mcp_auth_header, mcp_server_auth_headers=mcp_server_auth_headers, request_tags=request_tags, + allow_virtual_tools=not stream, + ) + + virtual_tool_scope = ( + LiteLLM_Proxy_MCP_Handler._build_virtual_tool_scope(mcp_tools_with_litellm_proxy) + if not stream and is_mcp_tool_search_enabled(user_api_key_auth) + else None ) openai_tools: Final = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai( @@ -175,7 +196,6 @@ async def acompletion_with_mcp( return response # For auto-execute: handle streaming vs non-streaming differently - stream: Final[object] = kwargs.get("stream", False) mock_tool_calls: Final = base_call_args.pop("mock_tool_calls", None) if stream: @@ -592,56 +612,73 @@ async def acompletion_with_mcp( if not isinstance(initial_response, ModelResponse): return initial_response - # Extract tool calls from response - tool_calls: Final = LiteLLM_Proxy_MCP_Handler._extract_tool_calls_from_chat_response(response=initial_response) + max_rounds: Final = max_tool_call_rounds(virtual_tool_scope) + current_response: ModelResponse = initial_response + current_messages: list[object] = messages # mutable-ok: chat follow-up takes list + all_tool_calls: Final[list[object]] = [] # mutable-ok: round accumulator + all_tool_results: Final[list[MCPToolResult]] = [] # mutable-ok: round accumulator + completed_rounds = 0 - if not tool_calls: - _add_mcp_metadata_to_response( - response=initial_response, - openai_tools=openai_tools, - ) - return initial_response + while completed_rounds < max_rounds: + tool_calls = LiteLLM_Proxy_MCP_Handler._extract_tool_calls_from_chat_response(response=current_response) + if not tool_calls: + break + all_tool_calls.extend(tool_calls) - # Execute tool calls - tool_results: Final = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( - tool_server_map=tool_server_map, - tool_calls=tool_calls, - user_api_key_auth=user_api_key_auth, - mcp_auth_header=mcp_auth_header, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - litellm_call_id=context.litellm_call_id, - litellm_trace_id=context.litellm_trace_id, - request_tags=request_tags, - ) - - if not tool_results: - _add_mcp_metadata_to_response( - response=initial_response, - openai_tools=openai_tools, + tool_results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( + tool_server_map=tool_server_map, tool_calls=tool_calls, + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + litellm_call_id=context.litellm_call_id, + litellm_trace_id=context.litellm_trace_id, + request_tags=request_tags, + virtual_tool_scope=virtual_tool_scope, ) - return initial_response + if not tool_results: + break - # Create follow-up messages with tool results - follow_up_messages: Final = LiteLLM_Proxy_MCP_Handler._create_follow_up_messages_for_chat( - original_messages=messages, - response=initial_response, - tool_results=tool_results, - ) + completed_rounds += 1 + all_tool_results.extend(tool_results) - # Make follow-up call with original stream setting - follow_up_call_args: Final = dict(base_call_args) - follow_up_call_args["messages"] = follow_up_messages - follow_up_call_args["stream"] = stream - - response = await litellm_acompletion(**follow_up_call_args) - if isinstance(response, (ModelResponse, CustomStreamWrapper)): - _add_mcp_metadata_to_response( - response=response, - openai_tools=openai_tools, - tool_calls=tool_calls, + follow_up_messages = LiteLLM_Proxy_MCP_Handler._create_follow_up_messages_for_chat( + original_messages=current_messages, + response=current_response, tool_results=tool_results, ) - return response + + follow_up_call_args = dict(base_call_args) + follow_up_call_args["messages"] = follow_up_messages + follow_up_call_args["stream"] = stream + if completed_rounds >= max_rounds: + follow_up_call_args["tools"] = None + verbose_logger.warning( + "MCP auto-execute hit its %s-round cap; forcing a text-only follow-up.", + max_rounds, + ) + + follow_up_response = await litellm_acompletion(**follow_up_call_args) + current_messages = follow_up_messages + + if not isinstance(follow_up_response, ModelResponse): + if isinstance(follow_up_response, CustomStreamWrapper): + _add_mcp_metadata_to_response( + response=follow_up_response, + openai_tools=openai_tools, + tool_calls=all_tool_calls, + tool_results=all_tool_results, + ) + return follow_up_response + + current_response = follow_up_response + + _add_mcp_metadata_to_response( + response=current_response, + openai_tools=openai_tools, + tool_calls=all_tool_calls or None, + tool_results=all_tool_results or None, + ) + return current_response diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index a5021e2f777..96ce081f604 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -6,9 +6,10 @@ from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypeAlias, Type from openai.types.chat import ChatCompletionToolParam from openai.types.responses.function_tool_param import FunctionToolParam +from typing_extensions import NotRequired, ReadOnly from litellm._logging import verbose_logger -from litellm.constants import MAXIMUM_TRACEBACK_LINES_TO_LOG +from litellm.constants import MAXIMUM_TRACEBACK_LINES_TO_LOG, MCP_VIRTUAL_TOOL_SEARCH_SERVER_NAME from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy._experimental.mcp_server.utils import ( iter_known_server_prefixes, @@ -17,6 +18,16 @@ from litellm.proxy._experimental.mcp_server.utils import ( strip_known_server_prefix, ) from litellm.responses.main import aresponses +from litellm.responses.mcp.tool_search_bridge import ( + VirtualToolCallPlan, + VirtualToolScope, + is_mcp_tool_search_enabled, + resolve_virtual_tool_call, + run_virtual_tool_call, + run_virtual_tool_search, + virtual_tool_definitions, + virtual_tool_server_map, +) from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator from litellm.types.llms.openai import ( ResponseInputParam, @@ -54,6 +65,7 @@ class MCPToolResult(TypedDict): tool_call_id: str | None result: str name: str | None + display_name: NotRequired[ReadOnly[str]] LITELLM_PROXY_MCP_SERVER_URL: Final = "litellm_proxy" @@ -258,12 +270,10 @@ class LiteLLM_Proxy_MCP_Handler: _get_tools_from_mcp_servers, ) - mcp_servers: Final = [ - server_url.split("/")[-1] - for _tool in (mcp_tools_with_litellm_proxy or ()) - for server_url in (_tool.get("server_url", "") if isinstance(_tool, dict) else "",) - if isinstance(server_url, str) and server_url.startswith(LITELLM_PROXY_MCP_SERVER_URL_PREFIX) - ] + # if user specifies servers as server_url: litellm_proxy/mcp/zapier,github then return zapier,github + mcp_servers: Final = list( + LiteLLM_Proxy_MCP_Handler._extract_mcp_server_names(mcp_tools_with_litellm_proxy or ()) + ) # Resolve toolset names: collect all toolset IDs first, then apply their # combined permissions in a single pass so multiple toolsets are unioned @@ -383,18 +393,47 @@ class LiteLLM_Proxy_MCP_Handler: return deduplicated_tools, tool_server_map + @staticmethod + def _collect_allowed_tool_names( + mcp_tools_with_litellm_proxy: Iterable[ToolParam], + ) -> frozenset[str]: + """Union of the ``allowed_tools`` names declared across the request's MCP tool configs.""" + return frozenset( + tool_name + for tool_config in mcp_tools_with_litellm_proxy + if isinstance(tool_config, dict) and isinstance(allowed := tool_config.get("allowed_tools"), list) + for tool_name in allowed + if isinstance(tool_name, str) + ) + + @staticmethod + def _extract_mcp_server_names( + mcp_tools_with_litellm_proxy: Iterable[ToolParam], + ) -> tuple[str, ...]: + """Server aliases named by ``litellm_proxy/mcp/`` server_urls, in request order.""" + return tuple( + server_url.split("/")[-1] + for tool_config in mcp_tools_with_litellm_proxy + if isinstance(tool_config, dict) + and isinstance(server_url := tool_config.get("server_url", ""), str) + and server_url.startswith(LITELLM_PROXY_MCP_SERVER_URL_PREFIX) + ) + + @staticmethod + def _build_virtual_tool_scope( + mcp_tools_with_litellm_proxy: Sequence[Mapping[str, object]], + ) -> VirtualToolScope: + return VirtualToolScope( + mcp_servers=LiteLLM_Proxy_MCP_Handler._extract_mcp_server_names(mcp_tools_with_litellm_proxy), + allowed_tools=LiteLLM_Proxy_MCP_Handler._collect_allowed_tool_names(mcp_tools_with_litellm_proxy), + ) + @staticmethod def _filter_mcp_tools_by_allowed_tools( mcp_tools: list[MCPTool], mcp_tools_with_litellm_proxy: Sequence[Mapping[str, object]] ) -> list[MCPTool]: """Filter MCP tools based on allowed_tools parameter from the original tool configs.""" - # Collect all allowed tool names from all MCP tool configs - allowed_tool_names: Final[set[str]] = set() - for tool_config in mcp_tools_with_litellm_proxy: - if isinstance(tool_config, dict) and "allowed_tools" in tool_config: - allowed_tools = tool_config.get("allowed_tools", []) - if isinstance(allowed_tools, list): - allowed_tool_names.update(allowed_tools) + allowed_tool_names: Final = LiteLLM_Proxy_MCP_Handler._collect_allowed_tool_names(mcp_tools_with_litellm_proxy) # If no allowed_tools specified, return all tools if not allowed_tool_names: @@ -452,6 +491,7 @@ class LiteLLM_Proxy_MCP_Handler: mcp_auth_header: str | None = None, mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, request_tags: list[str] | None = None, + allow_virtual_tools: bool = True, ) -> tuple[list[MCPTool], dict[str, str]]: """ Process MCP tools through filtering and deduplication pipeline without OpenAI transformation. @@ -469,6 +509,15 @@ class LiteLLM_Proxy_MCP_Handler: if not mcp_tools_with_litellm_proxy: return [], {} + if allow_virtual_tools and is_mcp_tool_search_enabled(user_api_key_auth): + if LiteLLM_Proxy_MCP_Handler._should_auto_execute_tools(mcp_tools_with_litellm_proxy): + # mutable-ok: the caller's contract is tuple[list[MCPTool], dict[str, str]] + return list(virtual_tool_definitions()), dict(virtual_tool_server_map()) + verbose_logger.debug( + "mcp_tool_search_enabled is set but no MCP tool has require_approval='never'; " + "the virtual tools cannot be auto-executed, falling back to the full catalog." + ) + typed_user_api_key_auth: Final[UserAPIKeyAuth | None] = user_api_key_auth # Step 1: Fetch MCP tools from manager @@ -679,6 +728,44 @@ class LiteLLM_Proxy_MCP_Handler: return result_text or "Tool executed successfully" + @staticmethod + async def _resolve_virtual_tool_call_step( + tool_name: str, + parsed_arguments: Mapping[str, object], + virtual_tool_scope: VirtualToolScope, + user_api_key_auth: "UserAPIKeyAuth", + mcp_auth_header: str | None, + mcp_server_auth_headers: dict[str, dict[str, str]] | None, # mutable-ok: handle_mcp_tool_* takes dict + oauth2_headers: dict[str, str] | None, # mutable-ok: handle_mcp_tool_* takes dict + raw_headers: dict[str, str] | None, # mutable-ok: handle_mcp_tool_* takes dict + ) -> VirtualToolCallPlan | str: + """Run one virtual tool hop. + + Returns finished result text for ``mcp_tool_search`` and for a rejected + ``mcp_tool_call``, or the plan naming the real tool that + ``mcp_tool_call`` unwraps to, so the caller spend-logs it as that tool. + """ + from litellm.proxy._experimental.mcp_server.tool_search import ( + MCP_TOOL_SEARCH_TOOL_NAME, + coerce_top_k, + ) + + if tool_name == MCP_TOOL_SEARCH_TOOL_NAME: + query = parsed_arguments.get("query", "") + search_result = await run_virtual_tool_search( + query=query if isinstance(query, str) else "", + top_k=coerce_top_k(parsed_arguments.get("top_k", 5)), + scope=virtual_tool_scope, + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + return LiteLLM_Proxy_MCP_Handler._parse_mcp_result(search_result) + + return resolve_virtual_tool_call(parsed_arguments, virtual_tool_scope) + @staticmethod async def _execute_tool_calls( tool_server_map: dict[str, str], @@ -691,6 +778,7 @@ class LiteLLM_Proxy_MCP_Handler: litellm_call_id: str | None = None, litellm_trace_id: str | None = None, request_tags: list[str] | None = None, + virtual_tool_scope: VirtualToolScope | None = None, ) -> list[MCPToolResult]: """Execute tool calls and return results.""" from fastapi import HTTPException @@ -729,13 +817,50 @@ class LiteLLM_Proxy_MCP_Handler: # Import here to avoid circular import from litellm.proxy.proxy_server import proxy_logging_obj - server_name = tool_server_map[tool_name] + virtual_call_plan: VirtualToolCallPlan | None = None + if ( + virtual_tool_scope is not None + and typed_user_api_key_auth is not None + and tool_server_map.get(tool_name) == MCP_VIRTUAL_TOOL_SEARCH_SERVER_NAME + ): + virtual_outcome = await LiteLLM_Proxy_MCP_Handler._resolve_virtual_tool_call_step( + tool_name=tool_name, + parsed_arguments=parsed_arguments, + virtual_tool_scope=virtual_tool_scope, + user_api_key_auth=typed_user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + if isinstance(virtual_outcome, str): + tool_results.append( + { + "tool_call_id": tool_call_id, + "result": virtual_outcome, + "name": tool_name, + "display_name": tool_name, + } + ) + continue + virtual_call_plan = virtual_outcome - mcp_server = global_mcp_server_manager.get_mcp_server_by_name( - server_name - ) or global_mcp_server_manager._get_mcp_server_from_tool_name(tool_name) + effective_tool_name = virtual_call_plan.tool_name if virtual_call_plan is not None else tool_name + effective_arguments: dict[str, object] = ( # mutable-ok: call_tool takes dict[str, Any] + dict(virtual_call_plan.arguments) # mutable-ok: same + if virtual_call_plan is not None + else parsed_arguments + ) + + mapped_server_name = tool_server_map[tool_name] if virtual_call_plan is None else None + mcp_server = ( + global_mcp_server_manager.get_mcp_server_by_name(mapped_server_name) if mapped_server_name else None + ) or global_mcp_server_manager._get_mcp_server_from_tool_name(effective_tool_name) + server_name = mapped_server_name or (mcp_server.name if mcp_server is not None else None) resolved_tool_name = ( - _resolve_display_name_to_original(tool_name, [mcp_server]) if mcp_server else tool_name + _resolve_display_name_to_original(effective_tool_name, [mcp_server]) + if mcp_server + else effective_tool_name ) sanitized_tool_name = strip_known_server_prefix(resolved_tool_name, mcp_server) @@ -745,7 +870,7 @@ class LiteLLM_Proxy_MCP_Handler: "role": "tool", "content": { "tool_name": sanitized_tool_name, - "arguments": parsed_arguments, + "arguments": effective_arguments, }, } ] @@ -757,7 +882,7 @@ class LiteLLM_Proxy_MCP_Handler: "headers": logging_safe_headers, } logging_request_data = { - "model": f"MCP: {tool_name}", + "model": f"MCP: {effective_tool_name}", "metadata": logging_metadata, "input": logging_input, "call_type": CallTypes.call_mcp_tool.value, @@ -769,7 +894,7 @@ class LiteLLM_Proxy_MCP_Handler: "headers": logging_safe_headers, "body": { "name": sanitized_tool_name, - "arguments": parsed_arguments, + "arguments": effective_arguments, }, }, } @@ -810,7 +935,7 @@ class LiteLLM_Proxy_MCP_Handler: litellm_logging_obj = None logging_request_data["litellm_logging_obj"] = litellm_logging_obj - logging_request_data["arguments"] = parsed_arguments + logging_request_data["arguments"] = effective_arguments if litellm_logging_obj: try: @@ -823,8 +948,8 @@ class LiteLLM_Proxy_MCP_Handler: standard_logging_mcp_tool_call: StandardLoggingMCPToolCall = { "name": sanitized_tool_name, - "arguments": parsed_arguments, - "namespaced_tool_name": tool_name, + "arguments": effective_arguments, + "namespaced_tool_name": effective_tool_name, } if mcp_server: mcp_info = mcp_server.mcp_info or {} @@ -840,21 +965,37 @@ class LiteLLM_Proxy_MCP_Handler: if litellm_logging_obj: litellm_logging_obj.model_call_details["mcp_tool_call_metadata"] = standard_logging_mcp_tool_call - litellm_logging_obj.model = f"MCP: {tool_name}" + litellm_logging_obj.model = f"MCP: {effective_tool_name}" litellm_logging_obj.call_type = CallTypes.call_mcp_tool.value - result = await global_mcp_server_manager.call_tool( - server_name=server_name, - name=sanitized_tool_name, - arguments=parsed_arguments, - user_api_key_auth=typed_user_api_key_auth, - mcp_auth_header=mcp_auth_header, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - proxy_logging_obj=proxy_logging_obj, - litellm_logging_obj=litellm_logging_obj, - ) + if ( + virtual_call_plan is not None + and virtual_tool_scope is not None + and typed_user_api_key_auth is not None + ): + result = await run_virtual_tool_call( + plan=virtual_call_plan, + scope=virtual_tool_scope, + user_api_key_auth=typed_user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + litellm_logging_obj=litellm_logging_obj, + ) + else: + result = await global_mcp_server_manager.call_tool( + server_name=tool_server_map[tool_name], + name=sanitized_tool_name, + arguments=effective_arguments, + user_api_key_auth=typed_user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + proxy_logging_obj=proxy_logging_obj, + litellm_logging_obj=litellm_logging_obj, + ) if proxy_logging_obj: result = await proxy_logging_obj.post_mcp_call_hook( @@ -894,6 +1035,7 @@ class LiteLLM_Proxy_MCP_Handler: "tool_call_id": tool_call_id, "result": result_text, "name": tool_name, + "display_name": effective_tool_name, } ) @@ -1240,7 +1382,7 @@ class LiteLLM_Proxy_MCP_Handler: result_text = tool_result.get("result", "") # Extract tool name and arguments from tool calls - tool_name = "unknown" + tool_name = tool_result.get("display_name") or "unknown" tool_arguments = "{}" for tool_call in tool_calls: ( @@ -1249,7 +1391,7 @@ class LiteLLM_Proxy_MCP_Handler: call_id, ) = LiteLLM_Proxy_MCP_Handler._extract_tool_call_details(tool_call) if call_id == tool_call_id: - tool_name = name or "unknown" + tool_name = tool_result.get("display_name") or name or "unknown" tool_arguments = args or "{}" break diff --git a/litellm/responses/mcp/mcp_streaming_iterator.py b/litellm/responses/mcp/mcp_streaming_iterator.py index ca12b3e7cc3..92fede51a81 100644 --- a/litellm/responses/mcp/mcp_streaming_iterator.py +++ b/litellm/responses/mcp/mcp_streaming_iterator.py @@ -28,6 +28,7 @@ if TYPE_CHECKING: from litellm.proxy._types import UserAPIKeyAuth from litellm.responses.mcp.litellm_proxy_mcp_handler import MCPToolResult + from litellm.responses.mcp.tool_search_bridge import VirtualToolScope else: MCPTool = Any @@ -272,6 +273,8 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): self.user_api_key_auth = user_api_key_auth self.original_request_params = original_request_params or {} self.should_auto_execute = self._should_auto_execute_tools() + self.virtual_tool_scope = self._build_virtual_tool_scope() + self.max_tool_call_rounds = self._max_tool_call_rounds() # Streaming state management self.phase = "initial_response" # initial_response -> mcp_discovery -> (continue_initial_response <-> tool_execution) -> finished @@ -395,6 +398,22 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): return LiteLLM_Proxy_MCP_Handler._should_auto_execute_tools(self.mcp_tools_with_litellm_proxy) + def _build_virtual_tool_scope(self) -> "VirtualToolScope | None": + """Scope for the mcp_tool_search / mcp_tool_call virtual tools, or None when they are not in play.""" + from litellm.responses.mcp.litellm_proxy_mcp_handler import ( + LiteLLM_Proxy_MCP_Handler, + ) + from litellm.responses.mcp.tool_search_bridge import is_mcp_tool_search_enabled + + if not self.should_auto_execute or not is_mcp_tool_search_enabled(self.user_api_key_auth): + return None + return LiteLLM_Proxy_MCP_Handler._build_virtual_tool_scope(self.mcp_tools_with_litellm_proxy) + + def _max_tool_call_rounds(self) -> int: + from litellm.responses.mcp.tool_search_bridge import max_tool_call_rounds + + return max_tool_call_rounds(self.virtual_tool_scope) + def _make_stream_error_event(self) -> ResponsesAPIStreamingResponse: err: Final = self._stream_error status_code: Final[object] = getattr(err, "status_code", None) @@ -698,6 +717,7 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): litellm_call_id=self.litellm_call_id, litellm_trace_id=self.litellm_trace_id, request_tags=LiteLLM_Proxy_MCP_Handler._get_parent_request_tags(self.original_request_params), + virtual_tool_scope=self.virtual_tool_scope, ) # Create completion events and output_item.done events for tool execution @@ -706,7 +726,7 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): result_text = tool_result.get("result", "") # Find matching tool name and arguments - tool_name = "unknown" + tool_name = tool_result.get("display_name") or "unknown" tool_arguments = "{}" for tool_call in tool_calls: ( @@ -715,7 +735,7 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): call_id, ) = LiteLLM_Proxy_MCP_Handler._extract_tool_call_details(tool_call) if call_id == tool_call_id: - tool_name = name or "unknown" + tool_name = tool_result.get("display_name") or name or "unknown" tool_arguments = args or "{}" break @@ -805,14 +825,14 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): # Remove tool_choice to avoid forcing more tool calls follow_up_params.pop("tool_choice", None) - if self.tool_call_round >= MAX_MCP_TOOL_CALL_ROUNDS: + if self.tool_call_round >= self.max_tool_call_rounds: # Hit the round cap: drop tools entirely so the model must # answer in text instead of emitting another (unexecuted) # tool call that would otherwise end the stream in silence. follow_up_params.pop("tools", None) verbose_logger.warning( - "MCP auto-execute hit MAX_MCP_TOOL_CALL_ROUNDS=%s; forcing a text-only follow-up.", - MAX_MCP_TOOL_CALL_ROUNDS, + "MCP auto-execute hit its %s-round cap; forcing a text-only follow-up.", + self.max_tool_call_rounds, ) follow_up_response: Final = await aresponses(**follow_up_params) diff --git a/litellm/responses/mcp/tool_search_bridge.py b/litellm/responses/mcp/tool_search_bridge.py new file mode 100644 index 00000000000..8566dd73bd0 --- /dev/null +++ b/litellm/responses/mcp/tool_search_bridge.py @@ -0,0 +1,206 @@ +"""Responses-side adapter for the proxy's mcp_tool_search virtual tools. + +``server.py`` owns the virtual tools; this module only supplies what the +Responses path needs on top of them: the request's ``allowed_tools`` (a +Responses-level concept ``execute_mcp_tool`` knows nothing about) and the inner +tool name, so the caller can spend-log the real tool rather than the wrapper. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType +from typing import TYPE_CHECKING, Final + +from pydantic import BaseModel, Field, TypeAdapter, ValidationError + +from litellm.constants import MCP_VIRTUAL_TOOL_SEARCH_SERVER_NAME +from litellm.proxy._experimental.mcp_server.utils import split_server_prefix_from_name +from litellm.responses.mcp.mcp_streaming_iterator import MAX_MCP_TOOL_CALL_ROUNDS + +if TYPE_CHECKING: + from mcp.types import CallToolResult + from mcp.types import Tool as MCPTool + + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.proxy._types import UserAPIKeyAuth + +_SEARCH_RESULTS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...]) + + +@dataclass(frozen=True, slots=True) +class VirtualToolScope: + """Request-level scoping the virtual tools must reproduce, so a two-hop + search + call sees exactly what a direct catalog listing would have.""" + + mcp_servers: tuple[str, ...] + allowed_tools: frozenset[str] + + @property + def server_filter(self) -> list[str] | None: # mutable-ok: handle_mcp_tool_* takes Optional[list[str]] + return list(self.mcp_servers) or None # mutable-ok: same + + +@dataclass(frozen=True, slots=True) +class VirtualToolCallPlan: + """The real tool an ``mcp_tool_call`` unwraps to.""" + + tool_name: str + arguments: Mapping[str, object] + + +class _VirtualToolCallArguments(BaseModel): + tool_name: str + arguments: Mapping[str, object] = Field(default_factory=dict) + + +def max_tool_call_rounds(scope: VirtualToolScope | None) -> int: + """Round budget for one auto-executed request. Doubled for the virtual + tools, which spend two rounds per real tool use, so the flag does not halve + a key's effective tool budget.""" + if scope is None: + return MAX_MCP_TOOL_CALL_ROUNDS + return MAX_MCP_TOOL_CALL_ROUNDS * 2 + + +def is_mcp_tool_search_enabled(user_api_key_auth: object) -> bool: + object_permission: Final = getattr(user_api_key_auth, "object_permission", None) + return bool(getattr(object_permission, "mcp_tool_search_enabled", False)) + + +def virtual_tool_definitions() -> tuple[MCPTool, ...]: + """The two virtual tools, as MCP ``Tool`` objects. Any other definition the + ``/mcp`` path may grow is excluded: this path only supports search + call.""" + from mcp.types import Tool + + from litellm.proxy._experimental.mcp_server.tool_search import ( + MCP_TOOL_CALL_TOOL_NAME, + MCP_TOOL_SEARCH_TOOL_NAME, + get_virtual_tool_definitions, + ) + + wanted: Final = (MCP_TOOL_SEARCH_TOOL_NAME, MCP_TOOL_CALL_TOOL_NAME) + return tuple( + Tool.model_validate(definition) + for definition in get_virtual_tool_definitions() + if definition.get("name") in wanted + ) + + +def virtual_tool_server_map() -> Mapping[str, str]: + """``tool_server_map`` entries marking the two names as virtual, so a real + upstream tool of the same name still dispatches normally.""" + return MappingProxyType({tool.name: MCP_VIRTUAL_TOOL_SEARCH_SERVER_NAME for tool in virtual_tool_definitions()}) + + +def is_tool_name_allowed(tool_name: str, allowed_tools: frozenset[str]) -> bool: + if not allowed_tools or tool_name in allowed_tools: + return True + unprefixed_name, _ = split_server_prefix_from_name(tool_name) + return unprefixed_name in allowed_tools + + +def resolve_virtual_tool_call( + arguments: Mapping[str, object], + scope: VirtualToolScope, +) -> VirtualToolCallPlan | str: + """The plan naming the real tool, or the rejection text to hand back as the tool result.""" + try: + parsed: Final = _VirtualToolCallArguments.model_validate(arguments) + except ValidationError: + return "mcp_tool_call requires a 'tool_name' string argument" + + if not parsed.tool_name: + return "mcp_tool_call requires a non-empty 'tool_name'" + + if not is_tool_name_allowed(parsed.tool_name, scope.allowed_tools): + return f"Tool '{parsed.tool_name}' is not in allowed_tools for this request" + + return VirtualToolCallPlan(tool_name=parsed.tool_name, arguments=parsed.arguments) + + +async def run_virtual_tool_search( + query: str, + top_k: int, + scope: VirtualToolScope, + user_api_key_auth: UserAPIKeyAuth, + mcp_auth_header: str | None = None, + mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, # mutable-ok: handle_mcp_tool_* takes dict + oauth2_headers: dict[str, str] | None = None, # mutable-ok: handle_mcp_tool_* takes dict + raw_headers: dict[str, str] | None = None, # mutable-ok: handle_mcp_tool_* takes dict +) -> CallToolResult: + from litellm.proxy._experimental.mcp_server.tool_search import handle_mcp_tool_search + + result: Final = await handle_mcp_tool_search( + query=query, + top_k=top_k, + user_api_key_dict=user_api_key_auth, + mcp_servers=scope.server_filter, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + return _apply_allowed_tools_to_search_result(result, scope.allowed_tools) + + +async def run_virtual_tool_call( + plan: VirtualToolCallPlan, + scope: VirtualToolScope, + user_api_key_auth: UserAPIKeyAuth, + mcp_auth_header: str | None = None, + mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, # mutable-ok: handle_mcp_tool_* takes dict + oauth2_headers: dict[str, str] | None = None, # mutable-ok: handle_mcp_tool_* takes dict + raw_headers: dict[str, str] | None = None, # mutable-ok: handle_mcp_tool_* takes dict + litellm_logging_obj: LiteLLMLoggingObj | None = None, +) -> CallToolResult: + from litellm.proxy._experimental.mcp_server.tool_search import handle_mcp_tool_call + + return await handle_mcp_tool_call( + tool_name=plan.tool_name, + arguments=dict(plan.arguments), # mutable-ok: handle_mcp_tool_call takes dict[str, Any] + user_api_key_dict=user_api_key_auth, + mcp_servers=scope.server_filter, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + litellm_logging_obj=litellm_logging_obj, + ) + + +def _apply_allowed_tools_to_search_result( + result: CallToolResult, + allowed_tools: frozenset[str], +) -> CallToolResult: + if not allowed_tools or result.isError: + return result + + from mcp.types import CallToolResult, TextContent + + return CallToolResult( + content=[ # mutable-ok: CallToolResult.content is a list field + TextContent(type="text", text=_filtered_search_text(item.text, allowed_tools)) + if isinstance(item, TextContent) + else item + for item in result.content + ], + isError=result.isError, + ) + + +def _filtered_search_text(text: str, allowed_tools: frozenset[str]) -> str: + try: + entries: Final[Sequence[Mapping[str, object]]] = _SEARCH_RESULTS_ADAPTER.validate_json(text) + except ValidationError: + return text + + return json.dumps( + tuple( + entry + for entry in entries + if isinstance(name := entry.get("name"), str) and is_tool_name_allowed(name, allowed_tools) + ) + ) diff --git a/tests/test_litellm/responses/mcp/test_chat_completions_handler.py b/tests/test_litellm/responses/mcp/test_chat_completions_handler.py index 2c1845f7b92..e813d0343a9 100644 --- a/tests/test_litellm/responses/mcp/test_chat_completions_handler.py +++ b/tests/test_litellm/responses/mcp/test_chat_completions_handler.py @@ -1387,3 +1387,178 @@ async def test_acompletion_with_mcp_forwards_unserved_external_mcp_tool_to_the_p assert isinstance(result, ModelResponse) assert result.id == "chatcmpl-zapier" assert json.loads(provider.calls.last.request.content)["tools"] == [zapier_tool] +# --------------------------------------------------------------------------- +# mcp_tool_search virtual tools on the /chat/completions MCP bridge +# --------------------------------------------------------------------------- + +VIRTUAL_TOOL_MCP_CONFIG = { + "type": "mcp", + "server_label": "x", + "server_url": "litellm_proxy/mcp/deepwiki", + "require_approval": "never", +} + + +def _tool_search_auth(): + from litellm.models.object_permission import LiteLLM_ObjectPermissionTable + from litellm.proxy._types import UserAPIKeyAuth + + return UserAPIKeyAuth( + api_key="k", + user_id="u", + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id="test", mcp_tool_search_enabled=True + ), + ) + + +def _chat_tool_call_response(call_id: str, name: str, arguments: str) -> ModelResponse: + return ModelResponse( + id=call_id, + model="gpt-5.5", + created=0, + object="chat.completion", + choices=[ + { + "index": 0, + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": call_id, + "type": "function", + "function": {"name": name, "arguments": arguments}, + } + ], + }, + "finish_reason": "tool_calls", + } + ], + ) + + +def _chat_text_response(text: str) -> ModelResponse: + return ModelResponse( + id="final", + model="gpt-5.5", + created=0, + object="chat.completion", + choices=[{"index": 0, "message": {"role": "assistant", "content": text}, "finish_reason": "stop"}], + ) + + +def _patch_chat_mcp_globals(monkeypatch): + import sys + import types as _types + from unittest.mock import MagicMock + + from mcp.types import Tool + + monkeypatch.setitem( + sys.modules, + "litellm.proxy.proxy_server", + _types.SimpleNamespace(proxy_logging_obj=MagicMock()), + ) + from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing + + mock_get_tools = AsyncMock( + return_value=AggregateToolListing( + tools=[Tool(name="deepwiki-read_wiki_contents", description="d", inputSchema={"type": "object"})], + outcomes={}, + ) + ) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.server._get_tools_from_mcp_servers", + mock_get_tools, + ) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + _types.SimpleNamespace( + get_registry=MagicMock(return_value={}), + call_tool=AsyncMock(), + _get_mcp_server_from_tool_name=MagicMock(return_value=None), + get_mcp_server_by_name=MagicMock(return_value=None), + get_allowed_mcp_servers=AsyncMock(return_value=[]), + get_mcp_servers_from_ids=MagicMock(return_value=[]), + ), + ) + return mock_get_tools + + +@pytest.mark.asyncio +async def test_acompletion_with_mcp_virtual_tools_search_then_call(monkeypatch): + """Non-streaming /chat/completions must complete the two-hop virtual tool + flow in a single request and never list the catalog.""" + import litellm + from mcp.types import CallToolResult, TextContent + + mock_get_tools = _patch_chat_mcp_globals(monkeypatch) + + search_mock = AsyncMock( + return_value=CallToolResult( + content=[TextContent(type="text", text='[{"name": "deepwiki-read_wiki_contents"}]')], isError=False + ) + ) + call_mock = AsyncMock( + return_value=CallToolResult(content=[TextContent(type="text", text="wiki body")], isError=False) + ) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_search", search_mock + ) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_call", call_mock + ) + + mock_acompletion = AsyncMock( + side_effect=[ + _chat_tool_call_response("call-1", "mcp_tool_search", '{"query": "wiki"}'), + _chat_tool_call_response( + "call-2", + "mcp_tool_call", + '{"tool_name": "deepwiki-read_wiki_contents", "arguments": {"repo": "x"}}', + ), + _chat_text_response("The wiki says hello."), + ] + ) + monkeypatch.setattr(litellm, "acompletion", mock_acompletion) + + response = await acompletion_with_mcp( + model="gpt-5.5", + messages=[{"role": "user", "content": "describe the wiki"}], + tools=[VIRTUAL_TOOL_MCP_CONFIG], + user_api_key_auth=_tool_search_auth(), + ) + + assert mock_get_tools.await_count == 0 + assert mock_acompletion.await_count == 3 + assert search_mock.await_count == 1 + assert call_mock.await_count == 1 + assert call_mock.await_args.kwargs["tool_name"] == "deepwiki-read_wiki_contents" + + injected_tools = mock_acompletion.await_args_list[0].kwargs["tools"] + assert [tool["function"]["name"] for tool in injected_tools] == ["mcp_tool_search", "mcp_tool_call"] + assert response.choices[0].message.content == "The wiki says hello." + + +@pytest.mark.asyncio +async def test_acompletion_with_mcp_streaming_keeps_catalog_when_tool_search_enabled(monkeypatch): + """The streaming chat bridge only does one tool-call round, so the virtual + tools stay off there and the full catalog is injected as before.""" + import litellm + + mock_get_tools = _patch_chat_mcp_globals(monkeypatch) + mock_acompletion = AsyncMock(return_value=_chat_text_response("hi")) + monkeypatch.setattr(litellm, "acompletion", mock_acompletion) + + await acompletion_with_mcp( + model="gpt-5.5", + messages=[{"role": "user", "content": "hi"}], + tools=[VIRTUAL_TOOL_MCP_CONFIG], + user_api_key_auth=_tool_search_auth(), + stream=True, + ) + + assert mock_get_tools.await_count == 1 + injected_tools = mock_acompletion.await_args_list[0].kwargs["tools"] + assert [tool["function"]["name"] for tool in injected_tools] == ["deepwiki-read_wiki_contents"] diff --git a/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py b/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py index 9745a0af970..16ced299d7d 100644 --- a/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py +++ b/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py @@ -1,3 +1,4 @@ +import json import subprocess import sys import textwrap @@ -15,6 +16,8 @@ from litellm.responses.mcp import litellm_proxy_mcp_handler as mcp_handler_modul from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, ) +from litellm.constants import MCP_VIRTUAL_TOOL_SEARCH_SERVER_NAME +from litellm.responses.mcp.tool_search_bridge import virtual_tool_server_map from typing import Any, cast from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import ModelResponse @@ -1102,3 +1105,292 @@ async def test_mcp_follow_up_call_is_stateless_when_store_is_false( item for item in follow_up_call["input"] if isinstance(item, dict) and item.get("type") == "reasoning" ] assert bool(reasoning_items) is (store is False) +# mcp_tool_search virtual tools on the Responses API type: "mcp" path + + +def _mcp_tool_config(server: str = "deepwiki", allowed_tools=None): + config = { + "type": "mcp", + "server_label": "x", + "server_url": f"litellm_proxy/mcp/{server}", + "require_approval": "never", + } + if allowed_tools is not None: + config["allowed_tools"] = allowed_tools + return config + + +def _auth(tool_search_enabled=None): + """Real UserAPIKeyAuth; tool_search_enabled=None means no object_permission at all.""" + from litellm.models.object_permission import LiteLLM_ObjectPermissionTable + from litellm.proxy._types import UserAPIKeyAuth + + if tool_search_enabled is None: + return UserAPIKeyAuth(api_key="k", user_id="u") + return UserAPIKeyAuth( + api_key="k", + user_id="u", + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id="test", + mcp_tool_search_enabled=tool_search_enabled, + ), + ) + + +def _setup_async_proxy_logging(monkeypatch: pytest.MonkeyPatch) -> None: + """proxy_logging_obj with awaitable hooks, needed once a tool actually dispatches.""" + proxy_logging_obj = MagicMock() + proxy_logging_obj.post_call_failure_hook = AsyncMock() + proxy_logging_obj.post_mcp_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["response"]) + monkeypatch.setitem( + sys.modules, + "litellm.proxy.proxy_server", + types.SimpleNamespace(proxy_logging_obj=proxy_logging_obj), + ) + + +def _text_result(text: str, is_error: bool = False): + from mcp.types import CallToolResult, TextContent + + return CallToolResult(content=[TextContent(type="text", text=text)], isError=is_error) + + +def _patch_catalog_listing(monkeypatch): + """Patch the manager listing seam and return the mock, so a test can assert it was never used.""" + from mcp.types import Tool + + mock_get_tools = AsyncMock( + return_value=AggregateToolListing( + tools=[ + Tool( + name="deepwiki-read_wiki_structure", + description="read the wiki structure", + inputSchema={"type": "object", "properties": {}}, + ) + ], + outcomes={}, + ) + ) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.server._get_tools_from_mcp_servers", + mock_get_tools, + ) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + types.SimpleNamespace( + get_registry=MagicMock(return_value={}), + get_allowed_mcp_servers=AsyncMock(return_value=[]), + get_mcp_servers_from_ids=MagicMock(return_value=[]), + get_mcp_server_by_name=MagicMock(return_value=None), + ), + ) + return mock_get_tools + + +@pytest.mark.asyncio +async def test_process_mcp_tools_returns_only_virtual_tools_when_flag_enabled(monkeypatch): + """With mcp_tool_search_enabled the catalog must never be listed; the model + sees only mcp_tool_search + mcp_tool_call.""" + mock_get_tools = _patch_catalog_listing(monkeypatch) + + openai_tools, tool_server_map = await LiteLLM_Proxy_MCP_Handler._process_mcp_tools_to_openai_format( + user_api_key_auth=_auth(True), + mcp_tools_with_litellm_proxy=[_mcp_tool_config()], + ) + + assert [tool["name"] for tool in openai_tools] == ["mcp_tool_search", "mcp_tool_call"] + assert all(tool["type"] == "function" for tool in openai_tools) + assert tool_server_map == { + "mcp_tool_search": MCP_VIRTUAL_TOOL_SEARCH_SERVER_NAME, + "mcp_tool_call": MCP_VIRTUAL_TOOL_SEARCH_SERVER_NAME, + } + assert mock_get_tools.await_count == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "tool_search_enabled,require_approval", + [ + (False, "never"), + (None, "never"), + (True, None), + ], + ids=["flag_false", "no_object_permission", "flag_on_but_approval_required"], +) +async def test_process_mcp_tools_injects_catalog_unchanged(monkeypatch, tool_search_enabled, require_approval): + """Regression: the catalog path is untouched for keys without the flag, and + the virtual tools are skipped when they could not be auto-executed.""" + mock_get_tools = _patch_catalog_listing(monkeypatch) + tool_config = _mcp_tool_config() + if require_approval is None: + tool_config.pop("require_approval") + + openai_tools, tool_server_map = await LiteLLM_Proxy_MCP_Handler._process_mcp_tools_to_openai_format( + user_api_key_auth=_auth(tool_search_enabled), + mcp_tools_with_litellm_proxy=[tool_config], + ) + + assert [tool["name"] for tool in openai_tools] == ["deepwiki-read_wiki_structure"] + assert tool_server_map == {"deepwiki-read_wiki_structure": "deepwiki"} + assert mock_get_tools.await_count == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "allowed_tools,expected_names", + [ + (["read_wiki_structure"], ["deepwiki-read_wiki_structure"]), + (["something_else"], []), + ], + ids=["keeps_allowed", "filters_everything_out"], +) +async def test_execute_tool_calls_scopes_mcp_tool_search(monkeypatch, allowed_tools, expected_names): + """mcp_tool_search is scoped to the request's server aliases, and its results + are filtered to allowed_tools; filtering everything out is an empty list, not an error.""" + _setup_proxy_logging(monkeypatch) + _setup_mcp_call_environment(monkeypatch) + + search_mock = AsyncMock( + return_value=_text_result( + json.dumps( + [ + {"name": "deepwiki-read_wiki_structure", "description": "d", "inputSchema": {}}, + {"name": "deepwiki-ask_question", "description": "d", "inputSchema": {}}, + ] + ) + ) + ) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_search", + search_mock, + ) + + results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( + tool_server_map=virtual_tool_server_map(), + tool_calls=[ + { + "id": "call-search", + "function": {"name": "mcp_tool_search", "arguments": json.dumps({"query": "wiki", "top_k": 3})}, + } + ], + user_api_key_auth=_auth(True), + virtual_tool_scope=LiteLLM_Proxy_MCP_Handler._build_virtual_tool_scope( + [_mcp_tool_config(allowed_tools=allowed_tools)] + ), + ) + + assert search_mock.await_args.kwargs["mcp_servers"] == ["deepwiki"] + assert search_mock.await_args.kwargs["query"] == "wiki" + assert search_mock.await_args.kwargs["top_k"] == 3 + assert [entry["name"] for entry in json.loads(results[0]["result"])] == expected_names + + +@pytest.mark.asyncio +async def test_execute_tool_calls_mcp_tool_call_dispatches_inner_tool_with_auth_headers(monkeypatch): + """mcp_tool_call unwraps to the real tool, forwards the per-request auth + headers and spend-logs the inner tool name rather than mcp_tool_call.""" + _setup_mcp_call_environment(monkeypatch) + _setup_async_proxy_logging(monkeypatch) + + call_mock = AsyncMock(return_value=_text_result("wiki structure")) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_call", + call_mock, + ) + + captured = {} + + def fake_function_setup(*_args, **kwargs): + captured.update(kwargs) + return None, None + + handler_module = importlib.import_module("litellm.responses.mcp.litellm_proxy_mcp_handler") + monkeypatch.setattr(handler_module, "function_setup", fake_function_setup) + + results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( + tool_server_map=virtual_tool_server_map(), + tool_calls=[ + { + "id": "call-inner", + "function": { + "name": "mcp_tool_call", + "arguments": json.dumps( + {"tool_name": "deepwiki-read_wiki_structure", "arguments": {"repo": "litellm"}} + ), + }, + } + ], + user_api_key_auth=_auth(True), + mcp_auth_header="legacy-token", + mcp_server_auth_headers={"deepwiki": {"authorization": "Bearer per-server"}}, + oauth2_headers={"x-oauth": "1"}, + raw_headers={"x-raw": "1"}, + virtual_tool_scope=LiteLLM_Proxy_MCP_Handler._build_virtual_tool_scope([_mcp_tool_config()]), + ) + + call_kwargs = call_mock.await_args.kwargs + assert call_kwargs["tool_name"] == "deepwiki-read_wiki_structure" + assert call_kwargs["arguments"] == {"repo": "litellm"} + assert call_kwargs["mcp_servers"] == ["deepwiki"] + assert call_kwargs["mcp_auth_header"] == "legacy-token" + assert call_kwargs["mcp_server_auth_headers"] == {"deepwiki": {"authorization": "Bearer per-server"}} + assert call_kwargs["oauth2_headers"] == {"x-oauth": "1"} + assert call_kwargs["raw_headers"] == {"x-raw": "1"} + + assert captured["model"] == "MCP: deepwiki-read_wiki_structure" + assert captured["metadata"]["tool_name"] == "read_wiki_structure" + assert captured["input"][0]["content"]["arguments"] == {"repo": "litellm"} + assert results[0]["result"] == "wiki structure" + assert results[0]["display_name"] == "deepwiki-read_wiki_structure" + + +@pytest.mark.asyncio +async def test_execute_tool_calls_mcp_tool_call_rejects_tool_outside_allowed_tools(monkeypatch): + _setup_proxy_logging(monkeypatch) + _setup_mcp_call_environment(monkeypatch) + call_mock = AsyncMock(return_value=_text_result("should not run")) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_call", + call_mock, + ) + + results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( + tool_server_map=virtual_tool_server_map(), + tool_calls=[ + { + "id": "call-inner", + "function": { + "name": "mcp_tool_call", + "arguments": json.dumps({"tool_name": "deepwiki-delete_everything", "arguments": {}}), + }, + } + ], + user_api_key_auth=_auth(True), + virtual_tool_scope=LiteLLM_Proxy_MCP_Handler._build_virtual_tool_scope( + [_mcp_tool_config(allowed_tools=["read_wiki_structure"])] + ), + ) + + assert call_mock.await_count == 0 + assert "not in allowed_tools" in results[0]["result"] + + +@pytest.mark.asyncio +async def test_execute_tool_calls_ignores_virtual_names_without_scope(monkeypatch): + """A real upstream tool literally named mcp_tool_search must still dispatch + normally when the virtual tools are not in play.""" + call_tool_mock = _setup_mcp_call_environment(monkeypatch) + search_mock = AsyncMock(return_value=_text_result("[]")) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_search", + search_mock, + ) + + await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( + tool_server_map={"mcp_tool_search": "deepwiki"}, + tool_calls=[{"id": "c", "function": {"name": "mcp_tool_search", "arguments": "{}"}}], + user_api_key_auth=None, + ) + + assert search_mock.await_count == 0 + assert call_tool_mock.await_count == 1 diff --git a/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py b/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py index 5001589ce54..5045bac176e 100644 --- a/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py +++ b/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py @@ -336,3 +336,100 @@ async def test_streaming_follow_up_keeps_previous_response_id_when_stored(monkey follow_up_kwargs = aresponses_mock.call_args_list[0].kwargs assert follow_up_kwargs["previous_response_id"] == "resp_prev" assert not [item for item in follow_up_kwargs["input"] if item.get("type") == "reasoning"] +def _virtual_tool_iterator(initial_chunks) -> MCPEnhancedStreamingIterator: + from litellm.models.object_permission import LiteLLM_ObjectPermissionTable + from litellm.proxy._types import UserAPIKeyAuth + from litellm.responses.mcp.tool_search_bridge import virtual_tool_server_map + + return MCPEnhancedStreamingIterator( + base_iterator=_FakeAsyncStream(initial_chunks), + mcp_events=[], + tool_server_map=virtual_tool_server_map(), + mcp_tools_with_litellm_proxy=[ + { + "type": "mcp", + "server_url": "litellm_proxy/mcp/deepwiki", + "require_approval": "never", + } + ], + user_api_key_auth=UserAPIKeyAuth( + api_key="k", + user_id="u", + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id="test", mcp_tool_search_enabled=True + ), + ), + original_request_params={ + "model": "gpt-5.5", + "input": "what is berriai/litellm?", + "tools": [{"type": "mcp"}], + }, + ) + + +@pytest.mark.asyncio +async def test_streaming_virtual_tool_search_then_call_shows_inner_tool_name(monkeypatch): + """With mcp_tool_search_enabled the stream must run search -> call -> text, + dispatch through the virtual tool handlers (never the manager directly) and + label the mcp_call item with the real tool the model reached.""" + call_tool = _mock_mcp_environment(monkeypatch) + proxy_logging_obj = sys.modules["litellm.proxy.proxy_server"].proxy_logging_obj + proxy_logging_obj.post_mcp_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["response"]) + + search_mock = AsyncMock( + return_value=CallToolResult( + content=[TextContent(type="text", text='[{"name": "deepwiki-read_wiki_contents"}]')], isError=False + ) + ) + call_mock = AsyncMock( + return_value=CallToolResult(content=[TextContent(type="text", text="wiki body")], isError=False) + ) + monkeypatch.setattr("litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_search", search_mock) + monkeypatch.setattr("litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_call", call_mock) + + aresponses_mock = AsyncMock( + side_effect=[ + _FakeAsyncStream( + [ + _completed_chunk( + [ + _function_call( + "call_2", + "mcp_tool_call", + '{"tool_name": "deepwiki-read_wiki_contents", "arguments": {"repo": "x"}}', + ) + ] + ) + ] + ), + _text_only_stream("The wiki says hello."), + ] + ) + monkeypatch.setattr(responses_main_module, "aresponses", aresponses_mock) + + iterator = _virtual_tool_iterator( + [ + _output_item_added_chunk(), + _completed_chunk([_function_call("call_1", "mcp_tool_search", '{"query": "wiki"}')]), + ] + ) + + chunks = [chunk async for chunk in iterator] + + assert iterator.max_tool_call_rounds == MAX_MCP_TOOL_CALL_ROUNDS * 2 + assert search_mock.await_count == 1 + assert search_mock.await_args.kwargs["mcp_servers"] == ["deepwiki"] + assert call_mock.await_count == 1 + assert call_mock.await_args.kwargs["tool_name"] == "deepwiki-read_wiki_contents" + assert call_tool.call_count == 0 + + mcp_call_names = [ + chunk.item.name + for chunk in chunks + if getattr(chunk, "type", None) == ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE + and getattr(getattr(chunk, "item", None), "type", None) == "mcp_call" + ] + assert mcp_call_names == ["mcp_tool_search", "deepwiki-read_wiki_contents"] + + completed_chunks = [c for c in chunks if getattr(c, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED] + assert completed_chunks[-1].response.output[0]["content"][0]["text"] == "The wiki says hello." diff --git a/tests/test_litellm/responses/test_main.py b/tests/test_litellm/responses/test_main.py new file mode 100644 index 00000000000..e01bd16623f --- /dev/null +++ b/tests/test_litellm/responses/test_main.py @@ -0,0 +1,232 @@ +"""Tests for litellm/responses/main.py MCP auto-execution.""" + +import importlib +import json +import types +from unittest.mock import AsyncMock + +import pytest + +from litellm.models.object_permission import LiteLLM_ObjectPermissionTable +from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing +from litellm.proxy._types import UserAPIKeyAuth +from litellm.responses.main import aresponses_api_with_mcp +from litellm.responses.mcp.mcp_streaming_iterator import MAX_MCP_TOOL_CALL_ROUNDS +from litellm.types.llms.openai import ResponsesAPIResponse + +MCP_TOOL_CONFIG = { + "type": "mcp", + "server_label": "x", + "server_url": "litellm_proxy/mcp/deepwiki", + "require_approval": "never", +} + + +def _auth(tool_search_enabled: bool) -> UserAPIKeyAuth: + return UserAPIKeyAuth( + api_key="k", + user_id="u", + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id="test", + mcp_tool_search_enabled=tool_search_enabled, + ), + ) + + +def _function_call_response(response_id: str, call_id: str, name: str, arguments: dict) -> ResponsesAPIResponse: + return ResponsesAPIResponse( + id=response_id, + created_at=0, + model="gpt-5.5", + object="response", + status="completed", + output=[ + { + "type": "function_call", + "id": call_id, + "call_id": call_id, + "name": name, + "arguments": json.dumps(arguments), + } + ], + ) + + +def _text_response(response_id: str, text: str) -> ResponsesAPIResponse: + return ResponsesAPIResponse( + id=response_id, + created_at=0, + model="gpt-5.5", + object="response", + status="completed", + output=[ + { + "type": "message", + "id": "msg-1", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": text, "annotations": []}], + } + ], + ) + + +def _text_result(text: str): + from mcp.types import CallToolResult, TextContent + + return CallToolResult(content=[TextContent(type="text", text=text)], isError=False) + + +def _patch_aresponses(monkeypatch, responses): + """Patch both references to aresponses (main.py's own and the handler's follow-up import).""" + mock = AsyncMock(side_effect=list(responses)) + main_module = importlib.import_module("litellm.responses.main") + handler_module = importlib.import_module("litellm.responses.mcp.litellm_proxy_mcp_handler") + monkeypatch.setattr(main_module, "aresponses", mock) + monkeypatch.setattr(handler_module, "aresponses", mock) + return mock + + +def _patch_proxy_globals(monkeypatch): + import sys + + from unittest.mock import MagicMock + + proxy_logging_obj = MagicMock() + proxy_logging_obj.post_call_failure_hook = AsyncMock() + proxy_logging_obj.post_mcp_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["response"]) + monkeypatch.setitem( + sys.modules, + "litellm.proxy.proxy_server", + types.SimpleNamespace(proxy_logging_obj=proxy_logging_obj), + ) + fake_manager = types.SimpleNamespace( + get_registry=MagicMock(return_value={}), + call_tool=AsyncMock(), + _get_mcp_server_from_tool_name=MagicMock(return_value=None), + get_mcp_server_by_name=MagicMock(return_value=None), + get_allowed_mcp_servers=AsyncMock(return_value=[]), + get_mcp_servers_from_ids=MagicMock(return_value=[]), + ) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + fake_manager, + ) + + +@pytest.mark.asyncio +async def test_search_then_call_then_answer_completes_in_one_request(monkeypatch): + """The two-hop virtual tool flow (search -> call -> text) must finish inside a + single client request; a single-round implementation would return the + unexecuted mcp_tool_call instead.""" + _patch_proxy_globals(monkeypatch) + + search_mock = AsyncMock( + return_value=_text_result( + json.dumps([{"name": "deepwiki-read_wiki_structure", "description": "d", "inputSchema": {}}]) + ) + ) + call_mock = AsyncMock(return_value=_text_result("wiki structure")) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_search", search_mock + ) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_call", call_mock + ) + + aresponses_mock = _patch_aresponses( + monkeypatch, + [ + _function_call_response("resp-1", "call-1", "mcp_tool_search", {"query": "wiki"}), + _function_call_response( + "resp-2", + "call-2", + "mcp_tool_call", + {"tool_name": "deepwiki-read_wiki_structure", "arguments": {"repo": "litellm"}}, + ), + _text_response("resp-3", "The wiki has 3 sections."), + ], + ) + + final = await aresponses_api_with_mcp( + input="describe the wiki", + model="gpt-5.5", + tools=[MCP_TOOL_CONFIG], + litellm_metadata={"user_api_key_auth": _auth(True)}, + user_api_key_auth=_auth(True), + ) + + assert aresponses_mock.await_count == 3 + assert search_mock.await_count == 1 + assert call_mock.await_count == 1 + assert call_mock.await_args.kwargs["tool_name"] == "deepwiki-read_wiki_structure" + + # Only the two virtual tools ever reach the model. + first_call_tools = aresponses_mock.await_args_list[0].kwargs["tools"] + assert [tool["name"] for tool in first_call_tools] == ["mcp_tool_search", "mcp_tool_call"] + + assert isinstance(final, ResponsesAPIResponse) + output_types = [item.get("type") if isinstance(item, dict) else item.type for item in final.output] + assert "tool_execution_results" in output_types + tool_results_item = next( + item for item in final.output if isinstance(item, dict) and item.get("type") == "tool_execution_results" + ) + logged_results = json.loads(tool_results_item["content"][0]["text"]) + assert [entry["name"] for entry in logged_results] == ["mcp_tool_search", "mcp_tool_call"] + assert [entry["display_name"] for entry in logged_results] == [ + "mcp_tool_search", + "deepwiki-read_wiki_structure", + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("tool_search_enabled", [True, False], ids=["virtual_tools", "plain_catalog"]) +async def test_auto_execute_round_cap(monkeypatch, tool_search_enabled): + """A model that keeps calling tools is capped and the final follow-up drops + tools so it has to answer in text. The virtual tools spend two rounds per + real tool use, so the flagged path gets twice the budget.""" + from mcp.types import Tool + + _patch_proxy_globals(monkeypatch) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.server._get_tools_from_mcp_servers", + AsyncMock( + return_value=AggregateToolListing( + tools=[Tool(name="deepwiki-read_wiki_contents", description="d", inputSchema={})], outcomes={} + ) + ), + ) + executed = AsyncMock(return_value=_text_result("[]")) + if tool_search_enabled: + called_tool = "mcp_tool_search" + expected_cap = MAX_MCP_TOOL_CALL_ROUNDS * 2 + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_search", executed + ) + else: + called_tool = "deepwiki-read_wiki_contents" + expected_cap = MAX_MCP_TOOL_CALL_ROUNDS + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager.call_tool", + executed, + ) + + aresponses_mock = _patch_aresponses( + monkeypatch, + [ + _function_call_response(f"resp-{i}", f"call-{i}", called_tool, {"query": "wiki"}) + for i in range(MAX_MCP_TOOL_CALL_ROUNDS * 2 + 5) + ], + ) + + await aresponses_api_with_mcp( + input="describe the wiki", + model="gpt-5.5", + tools=[MCP_TOOL_CONFIG], + litellm_metadata={"user_api_key_auth": _auth(tool_search_enabled)}, + user_api_key_auth=_auth(tool_search_enabled), + ) + + assert executed.await_count == expected_cap + assert aresponses_mock.await_count == expected_cap + 1 + assert aresponses_mock.await_args_list[-1].kwargs["tools"] is None