From 9f0cfce792955838c6043cc12c4e409cb143b36b Mon Sep 17 00:00:00 2001 From: Thibault Serot Date: Fri, 4 Sep 2026 14:12:12 +1000 Subject: [PATCH] feat(responses): honour mcp_tool_search_enabled on the Responses API mcp path A key with object_permission.mcp_tool_search_enabled already gets the two-hop discovery flow (mcp_tool_search then mcp_tool_call) on the /mcp protocol and REST paths, but not when it sends tools: [{"type": "mcp", "server_url": "litellm_proxy/mcp/"}] to /v1/responses; that path always listed the full catalog into the prompt. The Responses MCP handler now injects the two virtual tools instead of the catalog when the flag is set and require_approval is "never", and routes those tool calls back through handle_mcp_tool_search / handle_mcp_tool_call so server scope, per-key permissions, per-request auth headers, guardrails and error-to-isError conversion behave exactly as on /mcp. The mcp_servers derived from the litellm_proxy/mcp/ server_urls and the request's allowed_tools scope both hops. A mcp_tool_call spend log records the inner tool (metadata.tool_name and model = "MCP: "), not the wrapper. The non-streaming Responses path only ran one execute-then-follow-up round, which cannot complete a search followed by a call, so it now loops and drops tools from the follow-up made at the cap so the model has to answer in text. The virtual tools spend two rounds per real tool use, so max_tool_call_rounds() doubles the budget for that path; otherwise the flag would halve a key's effective tool budget. The /chat/completions MCP bridge gets the same loop for its non-streaming path; its streaming path still makes a single follow-up call, so the virtual tools stay off there and it keeps injecting the full catalog. Keys without the flag are unaffected: the catalog is injected exactly as before. --- litellm/constants.py | 5 + .../proxy/_experimental/mcp_server/AGENTS.md | 4 + litellm/responses/main.py | 192 ++++++------ .../responses/mcp/chat_completions_handler.py | 129 +++++--- .../mcp/litellm_proxy_mcp_handler.py | 222 ++++++++++--- .../responses/mcp/mcp_streaming_iterator.py | 30 +- litellm/responses/mcp/tool_search_bridge.py | 206 ++++++++++++ .../mcp/test_chat_completions_handler.py | 175 +++++++++++ .../mcp/test_litellm_proxy_mcp_handler.py | 292 ++++++++++++++++++ .../mcp/test_mcp_streaming_iterator.py | 97 ++++++ tests/test_litellm/responses/test_main.py | 232 ++++++++++++++ 11 files changed, 1401 insertions(+), 183 deletions(-) create mode 100644 litellm/responses/mcp/tool_search_bridge.py create mode 100644 tests/test_litellm/responses/test_main.py diff --git a/litellm/constants.py b/litellm/constants.py index f9389d22dea..46b41fda13d 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -172,6 +172,11 @@ MCP_PER_USER_TOKEN_DEFAULT_TTL: Final = int( ) MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS: Final = int(os.getenv("MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS", "60")) +# Sentinel server name for the mcp_tool_search / mcp_tool_call virtual tools, +# stored in the Responses API tool_server_map to tell them apart from a real +# upstream tool of the same name. +MCP_VIRTUAL_TOOL_SEARCH_SERVER_NAME = "litellm_virtual_mcp_tool_search" + # MCP timeout defaults (seconds). Override via env vars for slow/custom MCP servers. MCP_CLIENT_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_CLIENT_TIMEOUT", "60.0")) MCP_TOOL_LISTING_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_TOOL_LISTING_TIMEOUT", "30.0")) diff --git a/litellm/proxy/_experimental/mcp_server/AGENTS.md b/litellm/proxy/_experimental/mcp_server/AGENTS.md index 6e1d121c3be..5c582f13c2c 100644 --- a/litellm/proxy/_experimental/mcp_server/AGENTS.md +++ b/litellm/proxy/_experimental/mcp_server/AGENTS.md @@ -85,6 +85,10 @@ module materially harder to understand. permissions, no-accessible-server rejection, per-request auth headers, server scope, error to `isError` conversion, and spend logging. Reuse `_list_mcp_tools` and `execute_mcp_tool` rather than reimplementing any of these checks. +- The same flag also drives the Responses API / chat completions `type: "mcp"` + tools that point at `litellm_proxy/`. That bridge lives in + `litellm/responses/mcp/tool_search_bridge.py` and calls `handle_mcp_tool_search` + / `handle_mcp_tool_call` here; keep those two entry points backwards compatible. ## Tests diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 7a210cdd970..209710e5174 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -59,6 +59,8 @@ from litellm.utils import ( if TYPE_CHECKING: from fastapi import WebSocket from mcp.types import Tool as MCPTool + + from litellm.responses.mcp.litellm_proxy_mcp_handler import MCPToolResult else: MCPTool = Any @@ -173,6 +175,7 @@ async def aresponses_api_with_mcp( from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, ) + from litellm.responses.mcp.tool_search_bridge import is_mcp_tool_search_enabled # Parse MCP tools and separate from other tools ( @@ -210,6 +213,12 @@ async def aresponses_api_with_mcp( ) openai_tools: Final = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai(original_mcp_tools) + virtual_tool_scope = ( + LiteLLM_Proxy_MCP_Handler._build_virtual_tool_scope(mcp_tools_with_litellm_proxy) + if is_mcp_tool_search_enabled(user_api_key_auth) + else None + ) + # Combine with other tools all_tools: Final = openai_tools + other_tools if (openai_tools or other_tools) else None @@ -299,105 +308,104 @@ async def aresponses_api_with_mcp( # Auto-Execute Tools Handling # If auto-execute tools is True, then we need to execute the tool calls ######################################################### - if should_auto_execute and isinstance(response, ResponsesAPIResponse): - tool_calls: Final = LiteLLM_Proxy_MCP_Handler._extract_tool_calls_from_response(response=response) + if not (should_auto_execute and isinstance(response, ResponsesAPIResponse)): + return response - if tool_calls: - user_api_key_auth = kwargs.get("litellm_metadata", {}).get("user_api_key_auth") + from litellm.responses.mcp.tool_search_bridge import max_tool_call_rounds - # Extract MCP auth headers from the request to pass to MCP server - secret_fields = kwargs.get("secret_fields") - ( - mcp_auth_header, - mcp_server_auth_headers, - oauth2_headers, - raw_headers_from_request, - ) = ResponsesAPIRequestUtils.extract_mcp_headers_from_request( - secret_fields=secret_fields, - tools=tools, + max_rounds = max_tool_call_rounds(virtual_tool_scope) + persistence_disabled: Final = LiteLLM_Proxy_MCP_Handler._is_persistence_disabled(call_params) + + # Extract MCP auth headers from the request to pass to MCP server + ( + mcp_auth_header, + mcp_server_auth_headers, + oauth2_headers, + raw_headers_from_request, + ) = ResponsesAPIRequestUtils.extract_mcp_headers_from_request( + secret_fields=kwargs.get("secret_fields"), + tools=tools, + ) + + current_response: ResponsesAPIResponse = response + conversation_input: Any = input + all_tool_results: Final[list[MCPToolResult]] = [] # mutable-ok: round accumulator + completed_rounds = 0 + + while completed_rounds < max_rounds: + tool_calls = LiteLLM_Proxy_MCP_Handler._extract_tool_calls_from_response(response=current_response) + if not tool_calls: + break + + tool_results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( + tool_server_map=tool_server_map, + tool_calls=tool_calls, + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers_from_request, + litellm_call_id=kwargs.get("litellm_call_id"), + litellm_trace_id=kwargs.get("litellm_trace_id"), + request_tags=LiteLLM_Proxy_MCP_Handler._get_parent_request_tags(kwargs), + virtual_tool_scope=virtual_tool_scope, + ) + if not tool_results: + break + + completed_rounds += 1 + all_tool_results.extend(tool_results) + + follow_up_input = LiteLLM_Proxy_MCP_Handler._create_follow_up_input( + response=current_response, + tool_results=tool_results, + original_input=conversation_input, + preserve_reasoning=persistence_disabled, + ) + + follow_up_call_params = LiteLLM_Proxy_MCP_Handler._prepare_follow_up_call_params( + call_params=call_params, original_stream_setting=stream or False + ) + + follow_up_tools = all_tools if completed_rounds < max_rounds else None + if follow_up_tools is None: + verbose_logger.warning( + "MCP auto-execute hit its %s-round cap; forcing a text-only follow-up.", + max_rounds, ) - tool_results: Final = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( - tool_server_map=tool_server_map, - tool_calls=tool_calls, - user_api_key_auth=user_api_key_auth, - mcp_auth_header=mcp_auth_header, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers_from_request, - litellm_call_id=kwargs.get("litellm_call_id"), - litellm_trace_id=kwargs.get("litellm_trace_id"), - request_tags=LiteLLM_Proxy_MCP_Handler._get_parent_request_tags(kwargs), - ) + follow_up_response = await LiteLLM_Proxy_MCP_Handler._make_follow_up_call( + follow_up_input=follow_up_input, + model=model, + all_tools=follow_up_tools, + response_id=previous_response_id if persistence_disabled else current_response.id, + **follow_up_call_params, + ) + if not isinstance(follow_up_response, ResponsesAPIResponse): + return follow_up_response - if tool_results: - persistence_disabled: Final = LiteLLM_Proxy_MCP_Handler._is_persistence_disabled(call_params) + conversation_input = follow_up_input + current_response = follow_up_response - follow_up_input: Final = LiteLLM_Proxy_MCP_Handler._create_follow_up_input( - response=response, - tool_results=tool_results, - original_input=input, - preserve_reasoning=persistence_disabled, - ) + if not all_tool_results: + return response - # Prepare parameters for follow-up call (restores original stream setting) - follow_up_call_params: Final = LiteLLM_Proxy_MCP_Handler._prepare_follow_up_call_params( - call_params=call_params, original_stream_setting=stream or False - ) - - # Create tool execution events for streaming if needed - tool_execution_events = [] - if stream: - tool_execution_events = LiteLLM_Proxy_MCP_Handler._create_tool_execution_events( - tool_calls=tool_calls, tool_results=tool_results - ) - - final_response = await LiteLLM_Proxy_MCP_Handler._make_follow_up_call( - follow_up_input=follow_up_input, - model=model, - all_tools=all_tools, - response_id=previous_response_id if persistence_disabled else response.id, - **follow_up_call_params, - ) - - # If streaming and we have tool execution events, wrap the response - if ( - stream - and tool_execution_events - and (hasattr(final_response, "__aiter__") or hasattr(final_response, "__iter__")) - ): - from litellm.responses.mcp.mcp_streaming_iterator import ( - MCPEnhancedStreamingIterator, - ) - - final_response = MCPEnhancedStreamingIterator( - tool_server_map=tool_server_map, - base_iterator=final_response, - mcp_events=tool_execution_events, - user_api_key_auth=user_api_key_auth, - ) - - # Add custom output elements to the final response (for non-streaming) - elif isinstance(final_response, ResponsesAPIResponse): - # Fetch MCP tools again for output elements (without OpenAI transformation) - ( - mcp_tools_for_output, - _, - ) = await LiteLLM_Proxy_MCP_Handler._process_mcp_tools_without_openai_transform( - user_api_key_auth=user_api_key_auth, - mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy, - mcp_auth_header=mcp_auth_header, - mcp_server_auth_headers=mcp_server_auth_headers, - request_tags=LiteLLM_Proxy_MCP_Handler._get_parent_request_tags(kwargs), - ) - final_response = LiteLLM_Proxy_MCP_Handler._add_mcp_output_elements_to_response( - response=final_response, - mcp_tools_fetched=mcp_tools_for_output, - tool_results=tool_results, - ) - return final_response - - return response + # Fetch MCP tools again for output elements (without OpenAI transformation) + ( + mcp_tools_for_output, + _, + ) = await LiteLLM_Proxy_MCP_Handler._process_mcp_tools_without_openai_transform( + user_api_key_auth=user_api_key_auth, + mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + request_tags=LiteLLM_Proxy_MCP_Handler._get_parent_request_tags(kwargs), + ) + return LiteLLM_Proxy_MCP_Handler._add_mcp_output_elements_to_response( + response=current_response, + mcp_tools_fetched=mcp_tools_for_output, + tool_results=all_tool_results, + ) def _bridges_to_chat_completions( diff --git a/litellm/responses/mcp/chat_completions_handler.py b/litellm/responses/mcp/chat_completions_handler.py index ae18d5f6f1b..b7c10c6c341 100644 --- a/litellm/responses/mcp/chat_completions_handler.py +++ b/litellm/responses/mcp/chat_completions_handler.py @@ -5,15 +5,21 @@ from typing import TYPE_CHECKING, Final, cast from typing_extensions import TypedDict, Unpack +from litellm._logging import verbose_logger from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, ) from litellm.responses.mcp.request_context import MCPRequestContext +from litellm.responses.mcp.tool_search_bridge import ( + is_mcp_tool_search_enabled, + max_tool_call_rounds, +) from litellm.types.utils import Message, ModelResponse from litellm.utils import CustomStreamWrapper if TYPE_CHECKING: from litellm.proxy._types import UserAPIKeyAuth + from litellm.responses.mcp.litellm_proxy_mcp_handler import MCPToolResult class _MCPCompletionKwargs(TypedDict, total=False, extra_items=object): @@ -126,6 +132,14 @@ async def acompletion_with_mcp( oauth2_headers: Final = context.oauth2_headers raw_headers: Final = context.raw_headers + stream: Final[object] = kwargs.get("stream", False) + + if stream and is_mcp_tool_search_enabled(user_api_key_auth): + verbose_logger.debug( + "mcp_tool_search_enabled is set but streaming /chat/completions only supports a single " + "tool-call round; falling back to the full catalog." + ) + # Process MCP tools (pass auth headers for dynamic auth) ( deduplicated_mcp_tools, @@ -137,6 +151,13 @@ async def acompletion_with_mcp( mcp_auth_header=mcp_auth_header, mcp_server_auth_headers=mcp_server_auth_headers, request_tags=request_tags, + allow_virtual_tools=not stream, + ) + + virtual_tool_scope = ( + LiteLLM_Proxy_MCP_Handler._build_virtual_tool_scope(mcp_tools_with_litellm_proxy) + if not stream and is_mcp_tool_search_enabled(user_api_key_auth) + else None ) openai_tools: Final = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai( @@ -175,7 +196,6 @@ async def acompletion_with_mcp( return response # For auto-execute: handle streaming vs non-streaming differently - stream: Final[object] = kwargs.get("stream", False) mock_tool_calls: Final = base_call_args.pop("mock_tool_calls", None) if stream: @@ -592,56 +612,73 @@ async def acompletion_with_mcp( if not isinstance(initial_response, ModelResponse): return initial_response - # Extract tool calls from response - tool_calls: Final = LiteLLM_Proxy_MCP_Handler._extract_tool_calls_from_chat_response(response=initial_response) + max_rounds: Final = max_tool_call_rounds(virtual_tool_scope) + current_response: ModelResponse = initial_response + current_messages: list[object] = messages # mutable-ok: chat follow-up takes list + all_tool_calls: Final[list[object]] = [] # mutable-ok: round accumulator + all_tool_results: Final[list[MCPToolResult]] = [] # mutable-ok: round accumulator + completed_rounds = 0 - if not tool_calls: - _add_mcp_metadata_to_response( - response=initial_response, - openai_tools=openai_tools, - ) - return initial_response + while completed_rounds < max_rounds: + tool_calls = LiteLLM_Proxy_MCP_Handler._extract_tool_calls_from_chat_response(response=current_response) + if not tool_calls: + break + all_tool_calls.extend(tool_calls) - # Execute tool calls - tool_results: Final = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( - tool_server_map=tool_server_map, - tool_calls=tool_calls, - user_api_key_auth=user_api_key_auth, - mcp_auth_header=mcp_auth_header, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - litellm_call_id=context.litellm_call_id, - litellm_trace_id=context.litellm_trace_id, - request_tags=request_tags, - ) - - if not tool_results: - _add_mcp_metadata_to_response( - response=initial_response, - openai_tools=openai_tools, + tool_results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( + tool_server_map=tool_server_map, tool_calls=tool_calls, + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + litellm_call_id=context.litellm_call_id, + litellm_trace_id=context.litellm_trace_id, + request_tags=request_tags, + virtual_tool_scope=virtual_tool_scope, ) - return initial_response + if not tool_results: + break - # Create follow-up messages with tool results - follow_up_messages: Final = LiteLLM_Proxy_MCP_Handler._create_follow_up_messages_for_chat( - original_messages=messages, - response=initial_response, - tool_results=tool_results, - ) + completed_rounds += 1 + all_tool_results.extend(tool_results) - # Make follow-up call with original stream setting - follow_up_call_args: Final = dict(base_call_args) - follow_up_call_args["messages"] = follow_up_messages - follow_up_call_args["stream"] = stream - - response = await litellm_acompletion(**follow_up_call_args) - if isinstance(response, (ModelResponse, CustomStreamWrapper)): - _add_mcp_metadata_to_response( - response=response, - openai_tools=openai_tools, - tool_calls=tool_calls, + follow_up_messages = LiteLLM_Proxy_MCP_Handler._create_follow_up_messages_for_chat( + original_messages=current_messages, + response=current_response, tool_results=tool_results, ) - return response + + follow_up_call_args = dict(base_call_args) + follow_up_call_args["messages"] = follow_up_messages + follow_up_call_args["stream"] = stream + if completed_rounds >= max_rounds: + follow_up_call_args["tools"] = None + verbose_logger.warning( + "MCP auto-execute hit its %s-round cap; forcing a text-only follow-up.", + max_rounds, + ) + + follow_up_response = await litellm_acompletion(**follow_up_call_args) + current_messages = follow_up_messages + + if not isinstance(follow_up_response, ModelResponse): + if isinstance(follow_up_response, CustomStreamWrapper): + _add_mcp_metadata_to_response( + response=follow_up_response, + openai_tools=openai_tools, + tool_calls=all_tool_calls, + tool_results=all_tool_results, + ) + return follow_up_response + + current_response = follow_up_response + + _add_mcp_metadata_to_response( + response=current_response, + openai_tools=openai_tools, + tool_calls=all_tool_calls or None, + tool_results=all_tool_results or None, + ) + return current_response diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index a5021e2f777..96ce081f604 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -6,9 +6,10 @@ from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypeAlias, Type from openai.types.chat import ChatCompletionToolParam from openai.types.responses.function_tool_param import FunctionToolParam +from typing_extensions import NotRequired, ReadOnly from litellm._logging import verbose_logger -from litellm.constants import MAXIMUM_TRACEBACK_LINES_TO_LOG +from litellm.constants import MAXIMUM_TRACEBACK_LINES_TO_LOG, MCP_VIRTUAL_TOOL_SEARCH_SERVER_NAME from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy._experimental.mcp_server.utils import ( iter_known_server_prefixes, @@ -17,6 +18,16 @@ from litellm.proxy._experimental.mcp_server.utils import ( strip_known_server_prefix, ) from litellm.responses.main import aresponses +from litellm.responses.mcp.tool_search_bridge import ( + VirtualToolCallPlan, + VirtualToolScope, + is_mcp_tool_search_enabled, + resolve_virtual_tool_call, + run_virtual_tool_call, + run_virtual_tool_search, + virtual_tool_definitions, + virtual_tool_server_map, +) from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator from litellm.types.llms.openai import ( ResponseInputParam, @@ -54,6 +65,7 @@ class MCPToolResult(TypedDict): tool_call_id: str | None result: str name: str | None + display_name: NotRequired[ReadOnly[str]] LITELLM_PROXY_MCP_SERVER_URL: Final = "litellm_proxy" @@ -258,12 +270,10 @@ class LiteLLM_Proxy_MCP_Handler: _get_tools_from_mcp_servers, ) - mcp_servers: Final = [ - server_url.split("/")[-1] - for _tool in (mcp_tools_with_litellm_proxy or ()) - for server_url in (_tool.get("server_url", "") if isinstance(_tool, dict) else "",) - if isinstance(server_url, str) and server_url.startswith(LITELLM_PROXY_MCP_SERVER_URL_PREFIX) - ] + # if user specifies servers as server_url: litellm_proxy/mcp/zapier,github then return zapier,github + mcp_servers: Final = list( + LiteLLM_Proxy_MCP_Handler._extract_mcp_server_names(mcp_tools_with_litellm_proxy or ()) + ) # Resolve toolset names: collect all toolset IDs first, then apply their # combined permissions in a single pass so multiple toolsets are unioned @@ -383,18 +393,47 @@ class LiteLLM_Proxy_MCP_Handler: return deduplicated_tools, tool_server_map + @staticmethod + def _collect_allowed_tool_names( + mcp_tools_with_litellm_proxy: Iterable[ToolParam], + ) -> frozenset[str]: + """Union of the ``allowed_tools`` names declared across the request's MCP tool configs.""" + return frozenset( + tool_name + for tool_config in mcp_tools_with_litellm_proxy + if isinstance(tool_config, dict) and isinstance(allowed := tool_config.get("allowed_tools"), list) + for tool_name in allowed + if isinstance(tool_name, str) + ) + + @staticmethod + def _extract_mcp_server_names( + mcp_tools_with_litellm_proxy: Iterable[ToolParam], + ) -> tuple[str, ...]: + """Server aliases named by ``litellm_proxy/mcp/`` server_urls, in request order.""" + return tuple( + server_url.split("/")[-1] + for tool_config in mcp_tools_with_litellm_proxy + if isinstance(tool_config, dict) + and isinstance(server_url := tool_config.get("server_url", ""), str) + and server_url.startswith(LITELLM_PROXY_MCP_SERVER_URL_PREFIX) + ) + + @staticmethod + def _build_virtual_tool_scope( + mcp_tools_with_litellm_proxy: Sequence[Mapping[str, object]], + ) -> VirtualToolScope: + return VirtualToolScope( + mcp_servers=LiteLLM_Proxy_MCP_Handler._extract_mcp_server_names(mcp_tools_with_litellm_proxy), + allowed_tools=LiteLLM_Proxy_MCP_Handler._collect_allowed_tool_names(mcp_tools_with_litellm_proxy), + ) + @staticmethod def _filter_mcp_tools_by_allowed_tools( mcp_tools: list[MCPTool], mcp_tools_with_litellm_proxy: Sequence[Mapping[str, object]] ) -> list[MCPTool]: """Filter MCP tools based on allowed_tools parameter from the original tool configs.""" - # Collect all allowed tool names from all MCP tool configs - allowed_tool_names: Final[set[str]] = set() - for tool_config in mcp_tools_with_litellm_proxy: - if isinstance(tool_config, dict) and "allowed_tools" in tool_config: - allowed_tools = tool_config.get("allowed_tools", []) - if isinstance(allowed_tools, list): - allowed_tool_names.update(allowed_tools) + allowed_tool_names: Final = LiteLLM_Proxy_MCP_Handler._collect_allowed_tool_names(mcp_tools_with_litellm_proxy) # If no allowed_tools specified, return all tools if not allowed_tool_names: @@ -452,6 +491,7 @@ class LiteLLM_Proxy_MCP_Handler: mcp_auth_header: str | None = None, mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, request_tags: list[str] | None = None, + allow_virtual_tools: bool = True, ) -> tuple[list[MCPTool], dict[str, str]]: """ Process MCP tools through filtering and deduplication pipeline without OpenAI transformation. @@ -469,6 +509,15 @@ class LiteLLM_Proxy_MCP_Handler: if not mcp_tools_with_litellm_proxy: return [], {} + if allow_virtual_tools and is_mcp_tool_search_enabled(user_api_key_auth): + if LiteLLM_Proxy_MCP_Handler._should_auto_execute_tools(mcp_tools_with_litellm_proxy): + # mutable-ok: the caller's contract is tuple[list[MCPTool], dict[str, str]] + return list(virtual_tool_definitions()), dict(virtual_tool_server_map()) + verbose_logger.debug( + "mcp_tool_search_enabled is set but no MCP tool has require_approval='never'; " + "the virtual tools cannot be auto-executed, falling back to the full catalog." + ) + typed_user_api_key_auth: Final[UserAPIKeyAuth | None] = user_api_key_auth # Step 1: Fetch MCP tools from manager @@ -679,6 +728,44 @@ class LiteLLM_Proxy_MCP_Handler: return result_text or "Tool executed successfully" + @staticmethod + async def _resolve_virtual_tool_call_step( + tool_name: str, + parsed_arguments: Mapping[str, object], + virtual_tool_scope: VirtualToolScope, + user_api_key_auth: "UserAPIKeyAuth", + mcp_auth_header: str | None, + mcp_server_auth_headers: dict[str, dict[str, str]] | None, # mutable-ok: handle_mcp_tool_* takes dict + oauth2_headers: dict[str, str] | None, # mutable-ok: handle_mcp_tool_* takes dict + raw_headers: dict[str, str] | None, # mutable-ok: handle_mcp_tool_* takes dict + ) -> VirtualToolCallPlan | str: + """Run one virtual tool hop. + + Returns finished result text for ``mcp_tool_search`` and for a rejected + ``mcp_tool_call``, or the plan naming the real tool that + ``mcp_tool_call`` unwraps to, so the caller spend-logs it as that tool. + """ + from litellm.proxy._experimental.mcp_server.tool_search import ( + MCP_TOOL_SEARCH_TOOL_NAME, + coerce_top_k, + ) + + if tool_name == MCP_TOOL_SEARCH_TOOL_NAME: + query = parsed_arguments.get("query", "") + search_result = await run_virtual_tool_search( + query=query if isinstance(query, str) else "", + top_k=coerce_top_k(parsed_arguments.get("top_k", 5)), + scope=virtual_tool_scope, + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + return LiteLLM_Proxy_MCP_Handler._parse_mcp_result(search_result) + + return resolve_virtual_tool_call(parsed_arguments, virtual_tool_scope) + @staticmethod async def _execute_tool_calls( tool_server_map: dict[str, str], @@ -691,6 +778,7 @@ class LiteLLM_Proxy_MCP_Handler: litellm_call_id: str | None = None, litellm_trace_id: str | None = None, request_tags: list[str] | None = None, + virtual_tool_scope: VirtualToolScope | None = None, ) -> list[MCPToolResult]: """Execute tool calls and return results.""" from fastapi import HTTPException @@ -729,13 +817,50 @@ class LiteLLM_Proxy_MCP_Handler: # Import here to avoid circular import from litellm.proxy.proxy_server import proxy_logging_obj - server_name = tool_server_map[tool_name] + virtual_call_plan: VirtualToolCallPlan | None = None + if ( + virtual_tool_scope is not None + and typed_user_api_key_auth is not None + and tool_server_map.get(tool_name) == MCP_VIRTUAL_TOOL_SEARCH_SERVER_NAME + ): + virtual_outcome = await LiteLLM_Proxy_MCP_Handler._resolve_virtual_tool_call_step( + tool_name=tool_name, + parsed_arguments=parsed_arguments, + virtual_tool_scope=virtual_tool_scope, + user_api_key_auth=typed_user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + if isinstance(virtual_outcome, str): + tool_results.append( + { + "tool_call_id": tool_call_id, + "result": virtual_outcome, + "name": tool_name, + "display_name": tool_name, + } + ) + continue + virtual_call_plan = virtual_outcome - mcp_server = global_mcp_server_manager.get_mcp_server_by_name( - server_name - ) or global_mcp_server_manager._get_mcp_server_from_tool_name(tool_name) + effective_tool_name = virtual_call_plan.tool_name if virtual_call_plan is not None else tool_name + effective_arguments: dict[str, object] = ( # mutable-ok: call_tool takes dict[str, Any] + dict(virtual_call_plan.arguments) # mutable-ok: same + if virtual_call_plan is not None + else parsed_arguments + ) + + mapped_server_name = tool_server_map[tool_name] if virtual_call_plan is None else None + mcp_server = ( + global_mcp_server_manager.get_mcp_server_by_name(mapped_server_name) if mapped_server_name else None + ) or global_mcp_server_manager._get_mcp_server_from_tool_name(effective_tool_name) + server_name = mapped_server_name or (mcp_server.name if mcp_server is not None else None) resolved_tool_name = ( - _resolve_display_name_to_original(tool_name, [mcp_server]) if mcp_server else tool_name + _resolve_display_name_to_original(effective_tool_name, [mcp_server]) + if mcp_server + else effective_tool_name ) sanitized_tool_name = strip_known_server_prefix(resolved_tool_name, mcp_server) @@ -745,7 +870,7 @@ class LiteLLM_Proxy_MCP_Handler: "role": "tool", "content": { "tool_name": sanitized_tool_name, - "arguments": parsed_arguments, + "arguments": effective_arguments, }, } ] @@ -757,7 +882,7 @@ class LiteLLM_Proxy_MCP_Handler: "headers": logging_safe_headers, } logging_request_data = { - "model": f"MCP: {tool_name}", + "model": f"MCP: {effective_tool_name}", "metadata": logging_metadata, "input": logging_input, "call_type": CallTypes.call_mcp_tool.value, @@ -769,7 +894,7 @@ class LiteLLM_Proxy_MCP_Handler: "headers": logging_safe_headers, "body": { "name": sanitized_tool_name, - "arguments": parsed_arguments, + "arguments": effective_arguments, }, }, } @@ -810,7 +935,7 @@ class LiteLLM_Proxy_MCP_Handler: litellm_logging_obj = None logging_request_data["litellm_logging_obj"] = litellm_logging_obj - logging_request_data["arguments"] = parsed_arguments + logging_request_data["arguments"] = effective_arguments if litellm_logging_obj: try: @@ -823,8 +948,8 @@ class LiteLLM_Proxy_MCP_Handler: standard_logging_mcp_tool_call: StandardLoggingMCPToolCall = { "name": sanitized_tool_name, - "arguments": parsed_arguments, - "namespaced_tool_name": tool_name, + "arguments": effective_arguments, + "namespaced_tool_name": effective_tool_name, } if mcp_server: mcp_info = mcp_server.mcp_info or {} @@ -840,21 +965,37 @@ class LiteLLM_Proxy_MCP_Handler: if litellm_logging_obj: litellm_logging_obj.model_call_details["mcp_tool_call_metadata"] = standard_logging_mcp_tool_call - litellm_logging_obj.model = f"MCP: {tool_name}" + litellm_logging_obj.model = f"MCP: {effective_tool_name}" litellm_logging_obj.call_type = CallTypes.call_mcp_tool.value - result = await global_mcp_server_manager.call_tool( - server_name=server_name, - name=sanitized_tool_name, - arguments=parsed_arguments, - user_api_key_auth=typed_user_api_key_auth, - mcp_auth_header=mcp_auth_header, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - proxy_logging_obj=proxy_logging_obj, - litellm_logging_obj=litellm_logging_obj, - ) + if ( + virtual_call_plan is not None + and virtual_tool_scope is not None + and typed_user_api_key_auth is not None + ): + result = await run_virtual_tool_call( + plan=virtual_call_plan, + scope=virtual_tool_scope, + user_api_key_auth=typed_user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + litellm_logging_obj=litellm_logging_obj, + ) + else: + result = await global_mcp_server_manager.call_tool( + server_name=tool_server_map[tool_name], + name=sanitized_tool_name, + arguments=effective_arguments, + user_api_key_auth=typed_user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + proxy_logging_obj=proxy_logging_obj, + litellm_logging_obj=litellm_logging_obj, + ) if proxy_logging_obj: result = await proxy_logging_obj.post_mcp_call_hook( @@ -894,6 +1035,7 @@ class LiteLLM_Proxy_MCP_Handler: "tool_call_id": tool_call_id, "result": result_text, "name": tool_name, + "display_name": effective_tool_name, } ) @@ -1240,7 +1382,7 @@ class LiteLLM_Proxy_MCP_Handler: result_text = tool_result.get("result", "") # Extract tool name and arguments from tool calls - tool_name = "unknown" + tool_name = tool_result.get("display_name") or "unknown" tool_arguments = "{}" for tool_call in tool_calls: ( @@ -1249,7 +1391,7 @@ class LiteLLM_Proxy_MCP_Handler: call_id, ) = LiteLLM_Proxy_MCP_Handler._extract_tool_call_details(tool_call) if call_id == tool_call_id: - tool_name = name or "unknown" + tool_name = tool_result.get("display_name") or name or "unknown" tool_arguments = args or "{}" break diff --git a/litellm/responses/mcp/mcp_streaming_iterator.py b/litellm/responses/mcp/mcp_streaming_iterator.py index ca12b3e7cc3..92fede51a81 100644 --- a/litellm/responses/mcp/mcp_streaming_iterator.py +++ b/litellm/responses/mcp/mcp_streaming_iterator.py @@ -28,6 +28,7 @@ if TYPE_CHECKING: from litellm.proxy._types import UserAPIKeyAuth from litellm.responses.mcp.litellm_proxy_mcp_handler import MCPToolResult + from litellm.responses.mcp.tool_search_bridge import VirtualToolScope else: MCPTool = Any @@ -272,6 +273,8 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): self.user_api_key_auth = user_api_key_auth self.original_request_params = original_request_params or {} self.should_auto_execute = self._should_auto_execute_tools() + self.virtual_tool_scope = self._build_virtual_tool_scope() + self.max_tool_call_rounds = self._max_tool_call_rounds() # Streaming state management self.phase = "initial_response" # initial_response -> mcp_discovery -> (continue_initial_response <-> tool_execution) -> finished @@ -395,6 +398,22 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): return LiteLLM_Proxy_MCP_Handler._should_auto_execute_tools(self.mcp_tools_with_litellm_proxy) + def _build_virtual_tool_scope(self) -> "VirtualToolScope | None": + """Scope for the mcp_tool_search / mcp_tool_call virtual tools, or None when they are not in play.""" + from litellm.responses.mcp.litellm_proxy_mcp_handler import ( + LiteLLM_Proxy_MCP_Handler, + ) + from litellm.responses.mcp.tool_search_bridge import is_mcp_tool_search_enabled + + if not self.should_auto_execute or not is_mcp_tool_search_enabled(self.user_api_key_auth): + return None + return LiteLLM_Proxy_MCP_Handler._build_virtual_tool_scope(self.mcp_tools_with_litellm_proxy) + + def _max_tool_call_rounds(self) -> int: + from litellm.responses.mcp.tool_search_bridge import max_tool_call_rounds + + return max_tool_call_rounds(self.virtual_tool_scope) + def _make_stream_error_event(self) -> ResponsesAPIStreamingResponse: err: Final = self._stream_error status_code: Final[object] = getattr(err, "status_code", None) @@ -698,6 +717,7 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): litellm_call_id=self.litellm_call_id, litellm_trace_id=self.litellm_trace_id, request_tags=LiteLLM_Proxy_MCP_Handler._get_parent_request_tags(self.original_request_params), + virtual_tool_scope=self.virtual_tool_scope, ) # Create completion events and output_item.done events for tool execution @@ -706,7 +726,7 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): result_text = tool_result.get("result", "") # Find matching tool name and arguments - tool_name = "unknown" + tool_name = tool_result.get("display_name") or "unknown" tool_arguments = "{}" for tool_call in tool_calls: ( @@ -715,7 +735,7 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): call_id, ) = LiteLLM_Proxy_MCP_Handler._extract_tool_call_details(tool_call) if call_id == tool_call_id: - tool_name = name or "unknown" + tool_name = tool_result.get("display_name") or name or "unknown" tool_arguments = args or "{}" break @@ -805,14 +825,14 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): # Remove tool_choice to avoid forcing more tool calls follow_up_params.pop("tool_choice", None) - if self.tool_call_round >= MAX_MCP_TOOL_CALL_ROUNDS: + if self.tool_call_round >= self.max_tool_call_rounds: # Hit the round cap: drop tools entirely so the model must # answer in text instead of emitting another (unexecuted) # tool call that would otherwise end the stream in silence. follow_up_params.pop("tools", None) verbose_logger.warning( - "MCP auto-execute hit MAX_MCP_TOOL_CALL_ROUNDS=%s; forcing a text-only follow-up.", - MAX_MCP_TOOL_CALL_ROUNDS, + "MCP auto-execute hit its %s-round cap; forcing a text-only follow-up.", + self.max_tool_call_rounds, ) follow_up_response: Final = await aresponses(**follow_up_params) diff --git a/litellm/responses/mcp/tool_search_bridge.py b/litellm/responses/mcp/tool_search_bridge.py new file mode 100644 index 00000000000..8566dd73bd0 --- /dev/null +++ b/litellm/responses/mcp/tool_search_bridge.py @@ -0,0 +1,206 @@ +"""Responses-side adapter for the proxy's mcp_tool_search virtual tools. + +``server.py`` owns the virtual tools; this module only supplies what the +Responses path needs on top of them: the request's ``allowed_tools`` (a +Responses-level concept ``execute_mcp_tool`` knows nothing about) and the inner +tool name, so the caller can spend-log the real tool rather than the wrapper. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType +from typing import TYPE_CHECKING, Final + +from pydantic import BaseModel, Field, TypeAdapter, ValidationError + +from litellm.constants import MCP_VIRTUAL_TOOL_SEARCH_SERVER_NAME +from litellm.proxy._experimental.mcp_server.utils import split_server_prefix_from_name +from litellm.responses.mcp.mcp_streaming_iterator import MAX_MCP_TOOL_CALL_ROUNDS + +if TYPE_CHECKING: + from mcp.types import CallToolResult + from mcp.types import Tool as MCPTool + + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.proxy._types import UserAPIKeyAuth + +_SEARCH_RESULTS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...]) + + +@dataclass(frozen=True, slots=True) +class VirtualToolScope: + """Request-level scoping the virtual tools must reproduce, so a two-hop + search + call sees exactly what a direct catalog listing would have.""" + + mcp_servers: tuple[str, ...] + allowed_tools: frozenset[str] + + @property + def server_filter(self) -> list[str] | None: # mutable-ok: handle_mcp_tool_* takes Optional[list[str]] + return list(self.mcp_servers) or None # mutable-ok: same + + +@dataclass(frozen=True, slots=True) +class VirtualToolCallPlan: + """The real tool an ``mcp_tool_call`` unwraps to.""" + + tool_name: str + arguments: Mapping[str, object] + + +class _VirtualToolCallArguments(BaseModel): + tool_name: str + arguments: Mapping[str, object] = Field(default_factory=dict) + + +def max_tool_call_rounds(scope: VirtualToolScope | None) -> int: + """Round budget for one auto-executed request. Doubled for the virtual + tools, which spend two rounds per real tool use, so the flag does not halve + a key's effective tool budget.""" + if scope is None: + return MAX_MCP_TOOL_CALL_ROUNDS + return MAX_MCP_TOOL_CALL_ROUNDS * 2 + + +def is_mcp_tool_search_enabled(user_api_key_auth: object) -> bool: + object_permission: Final = getattr(user_api_key_auth, "object_permission", None) + return bool(getattr(object_permission, "mcp_tool_search_enabled", False)) + + +def virtual_tool_definitions() -> tuple[MCPTool, ...]: + """The two virtual tools, as MCP ``Tool`` objects. Any other definition the + ``/mcp`` path may grow is excluded: this path only supports search + call.""" + from mcp.types import Tool + + from litellm.proxy._experimental.mcp_server.tool_search import ( + MCP_TOOL_CALL_TOOL_NAME, + MCP_TOOL_SEARCH_TOOL_NAME, + get_virtual_tool_definitions, + ) + + wanted: Final = (MCP_TOOL_SEARCH_TOOL_NAME, MCP_TOOL_CALL_TOOL_NAME) + return tuple( + Tool.model_validate(definition) + for definition in get_virtual_tool_definitions() + if definition.get("name") in wanted + ) + + +def virtual_tool_server_map() -> Mapping[str, str]: + """``tool_server_map`` entries marking the two names as virtual, so a real + upstream tool of the same name still dispatches normally.""" + return MappingProxyType({tool.name: MCP_VIRTUAL_TOOL_SEARCH_SERVER_NAME for tool in virtual_tool_definitions()}) + + +def is_tool_name_allowed(tool_name: str, allowed_tools: frozenset[str]) -> bool: + if not allowed_tools or tool_name in allowed_tools: + return True + unprefixed_name, _ = split_server_prefix_from_name(tool_name) + return unprefixed_name in allowed_tools + + +def resolve_virtual_tool_call( + arguments: Mapping[str, object], + scope: VirtualToolScope, +) -> VirtualToolCallPlan | str: + """The plan naming the real tool, or the rejection text to hand back as the tool result.""" + try: + parsed: Final = _VirtualToolCallArguments.model_validate(arguments) + except ValidationError: + return "mcp_tool_call requires a 'tool_name' string argument" + + if not parsed.tool_name: + return "mcp_tool_call requires a non-empty 'tool_name'" + + if not is_tool_name_allowed(parsed.tool_name, scope.allowed_tools): + return f"Tool '{parsed.tool_name}' is not in allowed_tools for this request" + + return VirtualToolCallPlan(tool_name=parsed.tool_name, arguments=parsed.arguments) + + +async def run_virtual_tool_search( + query: str, + top_k: int, + scope: VirtualToolScope, + user_api_key_auth: UserAPIKeyAuth, + mcp_auth_header: str | None = None, + mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, # mutable-ok: handle_mcp_tool_* takes dict + oauth2_headers: dict[str, str] | None = None, # mutable-ok: handle_mcp_tool_* takes dict + raw_headers: dict[str, str] | None = None, # mutable-ok: handle_mcp_tool_* takes dict +) -> CallToolResult: + from litellm.proxy._experimental.mcp_server.tool_search import handle_mcp_tool_search + + result: Final = await handle_mcp_tool_search( + query=query, + top_k=top_k, + user_api_key_dict=user_api_key_auth, + mcp_servers=scope.server_filter, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + return _apply_allowed_tools_to_search_result(result, scope.allowed_tools) + + +async def run_virtual_tool_call( + plan: VirtualToolCallPlan, + scope: VirtualToolScope, + user_api_key_auth: UserAPIKeyAuth, + mcp_auth_header: str | None = None, + mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, # mutable-ok: handle_mcp_tool_* takes dict + oauth2_headers: dict[str, str] | None = None, # mutable-ok: handle_mcp_tool_* takes dict + raw_headers: dict[str, str] | None = None, # mutable-ok: handle_mcp_tool_* takes dict + litellm_logging_obj: LiteLLMLoggingObj | None = None, +) -> CallToolResult: + from litellm.proxy._experimental.mcp_server.tool_search import handle_mcp_tool_call + + return await handle_mcp_tool_call( + tool_name=plan.tool_name, + arguments=dict(plan.arguments), # mutable-ok: handle_mcp_tool_call takes dict[str, Any] + user_api_key_dict=user_api_key_auth, + mcp_servers=scope.server_filter, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + litellm_logging_obj=litellm_logging_obj, + ) + + +def _apply_allowed_tools_to_search_result( + result: CallToolResult, + allowed_tools: frozenset[str], +) -> CallToolResult: + if not allowed_tools or result.isError: + return result + + from mcp.types import CallToolResult, TextContent + + return CallToolResult( + content=[ # mutable-ok: CallToolResult.content is a list field + TextContent(type="text", text=_filtered_search_text(item.text, allowed_tools)) + if isinstance(item, TextContent) + else item + for item in result.content + ], + isError=result.isError, + ) + + +def _filtered_search_text(text: str, allowed_tools: frozenset[str]) -> str: + try: + entries: Final[Sequence[Mapping[str, object]]] = _SEARCH_RESULTS_ADAPTER.validate_json(text) + except ValidationError: + return text + + return json.dumps( + tuple( + entry + for entry in entries + if isinstance(name := entry.get("name"), str) and is_tool_name_allowed(name, allowed_tools) + ) + ) diff --git a/tests/test_litellm/responses/mcp/test_chat_completions_handler.py b/tests/test_litellm/responses/mcp/test_chat_completions_handler.py index 2c1845f7b92..e813d0343a9 100644 --- a/tests/test_litellm/responses/mcp/test_chat_completions_handler.py +++ b/tests/test_litellm/responses/mcp/test_chat_completions_handler.py @@ -1387,3 +1387,178 @@ async def test_acompletion_with_mcp_forwards_unserved_external_mcp_tool_to_the_p assert isinstance(result, ModelResponse) assert result.id == "chatcmpl-zapier" assert json.loads(provider.calls.last.request.content)["tools"] == [zapier_tool] +# --------------------------------------------------------------------------- +# mcp_tool_search virtual tools on the /chat/completions MCP bridge +# --------------------------------------------------------------------------- + +VIRTUAL_TOOL_MCP_CONFIG = { + "type": "mcp", + "server_label": "x", + "server_url": "litellm_proxy/mcp/deepwiki", + "require_approval": "never", +} + + +def _tool_search_auth(): + from litellm.models.object_permission import LiteLLM_ObjectPermissionTable + from litellm.proxy._types import UserAPIKeyAuth + + return UserAPIKeyAuth( + api_key="k", + user_id="u", + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id="test", mcp_tool_search_enabled=True + ), + ) + + +def _chat_tool_call_response(call_id: str, name: str, arguments: str) -> ModelResponse: + return ModelResponse( + id=call_id, + model="gpt-5.5", + created=0, + object="chat.completion", + choices=[ + { + "index": 0, + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": call_id, + "type": "function", + "function": {"name": name, "arguments": arguments}, + } + ], + }, + "finish_reason": "tool_calls", + } + ], + ) + + +def _chat_text_response(text: str) -> ModelResponse: + return ModelResponse( + id="final", + model="gpt-5.5", + created=0, + object="chat.completion", + choices=[{"index": 0, "message": {"role": "assistant", "content": text}, "finish_reason": "stop"}], + ) + + +def _patch_chat_mcp_globals(monkeypatch): + import sys + import types as _types + from unittest.mock import MagicMock + + from mcp.types import Tool + + monkeypatch.setitem( + sys.modules, + "litellm.proxy.proxy_server", + _types.SimpleNamespace(proxy_logging_obj=MagicMock()), + ) + from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing + + mock_get_tools = AsyncMock( + return_value=AggregateToolListing( + tools=[Tool(name="deepwiki-read_wiki_contents", description="d", inputSchema={"type": "object"})], + outcomes={}, + ) + ) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.server._get_tools_from_mcp_servers", + mock_get_tools, + ) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + _types.SimpleNamespace( + get_registry=MagicMock(return_value={}), + call_tool=AsyncMock(), + _get_mcp_server_from_tool_name=MagicMock(return_value=None), + get_mcp_server_by_name=MagicMock(return_value=None), + get_allowed_mcp_servers=AsyncMock(return_value=[]), + get_mcp_servers_from_ids=MagicMock(return_value=[]), + ), + ) + return mock_get_tools + + +@pytest.mark.asyncio +async def test_acompletion_with_mcp_virtual_tools_search_then_call(monkeypatch): + """Non-streaming /chat/completions must complete the two-hop virtual tool + flow in a single request and never list the catalog.""" + import litellm + from mcp.types import CallToolResult, TextContent + + mock_get_tools = _patch_chat_mcp_globals(monkeypatch) + + search_mock = AsyncMock( + return_value=CallToolResult( + content=[TextContent(type="text", text='[{"name": "deepwiki-read_wiki_contents"}]')], isError=False + ) + ) + call_mock = AsyncMock( + return_value=CallToolResult(content=[TextContent(type="text", text="wiki body")], isError=False) + ) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_search", search_mock + ) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_call", call_mock + ) + + mock_acompletion = AsyncMock( + side_effect=[ + _chat_tool_call_response("call-1", "mcp_tool_search", '{"query": "wiki"}'), + _chat_tool_call_response( + "call-2", + "mcp_tool_call", + '{"tool_name": "deepwiki-read_wiki_contents", "arguments": {"repo": "x"}}', + ), + _chat_text_response("The wiki says hello."), + ] + ) + monkeypatch.setattr(litellm, "acompletion", mock_acompletion) + + response = await acompletion_with_mcp( + model="gpt-5.5", + messages=[{"role": "user", "content": "describe the wiki"}], + tools=[VIRTUAL_TOOL_MCP_CONFIG], + user_api_key_auth=_tool_search_auth(), + ) + + assert mock_get_tools.await_count == 0 + assert mock_acompletion.await_count == 3 + assert search_mock.await_count == 1 + assert call_mock.await_count == 1 + assert call_mock.await_args.kwargs["tool_name"] == "deepwiki-read_wiki_contents" + + injected_tools = mock_acompletion.await_args_list[0].kwargs["tools"] + assert [tool["function"]["name"] for tool in injected_tools] == ["mcp_tool_search", "mcp_tool_call"] + assert response.choices[0].message.content == "The wiki says hello." + + +@pytest.mark.asyncio +async def test_acompletion_with_mcp_streaming_keeps_catalog_when_tool_search_enabled(monkeypatch): + """The streaming chat bridge only does one tool-call round, so the virtual + tools stay off there and the full catalog is injected as before.""" + import litellm + + mock_get_tools = _patch_chat_mcp_globals(monkeypatch) + mock_acompletion = AsyncMock(return_value=_chat_text_response("hi")) + monkeypatch.setattr(litellm, "acompletion", mock_acompletion) + + await acompletion_with_mcp( + model="gpt-5.5", + messages=[{"role": "user", "content": "hi"}], + tools=[VIRTUAL_TOOL_MCP_CONFIG], + user_api_key_auth=_tool_search_auth(), + stream=True, + ) + + assert mock_get_tools.await_count == 1 + injected_tools = mock_acompletion.await_args_list[0].kwargs["tools"] + assert [tool["function"]["name"] for tool in injected_tools] == ["deepwiki-read_wiki_contents"] diff --git a/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py b/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py index 9745a0af970..16ced299d7d 100644 --- a/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py +++ b/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py @@ -1,3 +1,4 @@ +import json import subprocess import sys import textwrap @@ -15,6 +16,8 @@ from litellm.responses.mcp import litellm_proxy_mcp_handler as mcp_handler_modul from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, ) +from litellm.constants import MCP_VIRTUAL_TOOL_SEARCH_SERVER_NAME +from litellm.responses.mcp.tool_search_bridge import virtual_tool_server_map from typing import Any, cast from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import ModelResponse @@ -1102,3 +1105,292 @@ async def test_mcp_follow_up_call_is_stateless_when_store_is_false( item for item in follow_up_call["input"] if isinstance(item, dict) and item.get("type") == "reasoning" ] assert bool(reasoning_items) is (store is False) +# mcp_tool_search virtual tools on the Responses API type: "mcp" path + + +def _mcp_tool_config(server: str = "deepwiki", allowed_tools=None): + config = { + "type": "mcp", + "server_label": "x", + "server_url": f"litellm_proxy/mcp/{server}", + "require_approval": "never", + } + if allowed_tools is not None: + config["allowed_tools"] = allowed_tools + return config + + +def _auth(tool_search_enabled=None): + """Real UserAPIKeyAuth; tool_search_enabled=None means no object_permission at all.""" + from litellm.models.object_permission import LiteLLM_ObjectPermissionTable + from litellm.proxy._types import UserAPIKeyAuth + + if tool_search_enabled is None: + return UserAPIKeyAuth(api_key="k", user_id="u") + return UserAPIKeyAuth( + api_key="k", + user_id="u", + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id="test", + mcp_tool_search_enabled=tool_search_enabled, + ), + ) + + +def _setup_async_proxy_logging(monkeypatch: pytest.MonkeyPatch) -> None: + """proxy_logging_obj with awaitable hooks, needed once a tool actually dispatches.""" + proxy_logging_obj = MagicMock() + proxy_logging_obj.post_call_failure_hook = AsyncMock() + proxy_logging_obj.post_mcp_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["response"]) + monkeypatch.setitem( + sys.modules, + "litellm.proxy.proxy_server", + types.SimpleNamespace(proxy_logging_obj=proxy_logging_obj), + ) + + +def _text_result(text: str, is_error: bool = False): + from mcp.types import CallToolResult, TextContent + + return CallToolResult(content=[TextContent(type="text", text=text)], isError=is_error) + + +def _patch_catalog_listing(monkeypatch): + """Patch the manager listing seam and return the mock, so a test can assert it was never used.""" + from mcp.types import Tool + + mock_get_tools = AsyncMock( + return_value=AggregateToolListing( + tools=[ + Tool( + name="deepwiki-read_wiki_structure", + description="read the wiki structure", + inputSchema={"type": "object", "properties": {}}, + ) + ], + outcomes={}, + ) + ) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.server._get_tools_from_mcp_servers", + mock_get_tools, + ) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + types.SimpleNamespace( + get_registry=MagicMock(return_value={}), + get_allowed_mcp_servers=AsyncMock(return_value=[]), + get_mcp_servers_from_ids=MagicMock(return_value=[]), + get_mcp_server_by_name=MagicMock(return_value=None), + ), + ) + return mock_get_tools + + +@pytest.mark.asyncio +async def test_process_mcp_tools_returns_only_virtual_tools_when_flag_enabled(monkeypatch): + """With mcp_tool_search_enabled the catalog must never be listed; the model + sees only mcp_tool_search + mcp_tool_call.""" + mock_get_tools = _patch_catalog_listing(monkeypatch) + + openai_tools, tool_server_map = await LiteLLM_Proxy_MCP_Handler._process_mcp_tools_to_openai_format( + user_api_key_auth=_auth(True), + mcp_tools_with_litellm_proxy=[_mcp_tool_config()], + ) + + assert [tool["name"] for tool in openai_tools] == ["mcp_tool_search", "mcp_tool_call"] + assert all(tool["type"] == "function" for tool in openai_tools) + assert tool_server_map == { + "mcp_tool_search": MCP_VIRTUAL_TOOL_SEARCH_SERVER_NAME, + "mcp_tool_call": MCP_VIRTUAL_TOOL_SEARCH_SERVER_NAME, + } + assert mock_get_tools.await_count == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "tool_search_enabled,require_approval", + [ + (False, "never"), + (None, "never"), + (True, None), + ], + ids=["flag_false", "no_object_permission", "flag_on_but_approval_required"], +) +async def test_process_mcp_tools_injects_catalog_unchanged(monkeypatch, tool_search_enabled, require_approval): + """Regression: the catalog path is untouched for keys without the flag, and + the virtual tools are skipped when they could not be auto-executed.""" + mock_get_tools = _patch_catalog_listing(monkeypatch) + tool_config = _mcp_tool_config() + if require_approval is None: + tool_config.pop("require_approval") + + openai_tools, tool_server_map = await LiteLLM_Proxy_MCP_Handler._process_mcp_tools_to_openai_format( + user_api_key_auth=_auth(tool_search_enabled), + mcp_tools_with_litellm_proxy=[tool_config], + ) + + assert [tool["name"] for tool in openai_tools] == ["deepwiki-read_wiki_structure"] + assert tool_server_map == {"deepwiki-read_wiki_structure": "deepwiki"} + assert mock_get_tools.await_count == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "allowed_tools,expected_names", + [ + (["read_wiki_structure"], ["deepwiki-read_wiki_structure"]), + (["something_else"], []), + ], + ids=["keeps_allowed", "filters_everything_out"], +) +async def test_execute_tool_calls_scopes_mcp_tool_search(monkeypatch, allowed_tools, expected_names): + """mcp_tool_search is scoped to the request's server aliases, and its results + are filtered to allowed_tools; filtering everything out is an empty list, not an error.""" + _setup_proxy_logging(monkeypatch) + _setup_mcp_call_environment(monkeypatch) + + search_mock = AsyncMock( + return_value=_text_result( + json.dumps( + [ + {"name": "deepwiki-read_wiki_structure", "description": "d", "inputSchema": {}}, + {"name": "deepwiki-ask_question", "description": "d", "inputSchema": {}}, + ] + ) + ) + ) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_search", + search_mock, + ) + + results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( + tool_server_map=virtual_tool_server_map(), + tool_calls=[ + { + "id": "call-search", + "function": {"name": "mcp_tool_search", "arguments": json.dumps({"query": "wiki", "top_k": 3})}, + } + ], + user_api_key_auth=_auth(True), + virtual_tool_scope=LiteLLM_Proxy_MCP_Handler._build_virtual_tool_scope( + [_mcp_tool_config(allowed_tools=allowed_tools)] + ), + ) + + assert search_mock.await_args.kwargs["mcp_servers"] == ["deepwiki"] + assert search_mock.await_args.kwargs["query"] == "wiki" + assert search_mock.await_args.kwargs["top_k"] == 3 + assert [entry["name"] for entry in json.loads(results[0]["result"])] == expected_names + + +@pytest.mark.asyncio +async def test_execute_tool_calls_mcp_tool_call_dispatches_inner_tool_with_auth_headers(monkeypatch): + """mcp_tool_call unwraps to the real tool, forwards the per-request auth + headers and spend-logs the inner tool name rather than mcp_tool_call.""" + _setup_mcp_call_environment(monkeypatch) + _setup_async_proxy_logging(monkeypatch) + + call_mock = AsyncMock(return_value=_text_result("wiki structure")) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_call", + call_mock, + ) + + captured = {} + + def fake_function_setup(*_args, **kwargs): + captured.update(kwargs) + return None, None + + handler_module = importlib.import_module("litellm.responses.mcp.litellm_proxy_mcp_handler") + monkeypatch.setattr(handler_module, "function_setup", fake_function_setup) + + results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( + tool_server_map=virtual_tool_server_map(), + tool_calls=[ + { + "id": "call-inner", + "function": { + "name": "mcp_tool_call", + "arguments": json.dumps( + {"tool_name": "deepwiki-read_wiki_structure", "arguments": {"repo": "litellm"}} + ), + }, + } + ], + user_api_key_auth=_auth(True), + mcp_auth_header="legacy-token", + mcp_server_auth_headers={"deepwiki": {"authorization": "Bearer per-server"}}, + oauth2_headers={"x-oauth": "1"}, + raw_headers={"x-raw": "1"}, + virtual_tool_scope=LiteLLM_Proxy_MCP_Handler._build_virtual_tool_scope([_mcp_tool_config()]), + ) + + call_kwargs = call_mock.await_args.kwargs + assert call_kwargs["tool_name"] == "deepwiki-read_wiki_structure" + assert call_kwargs["arguments"] == {"repo": "litellm"} + assert call_kwargs["mcp_servers"] == ["deepwiki"] + assert call_kwargs["mcp_auth_header"] == "legacy-token" + assert call_kwargs["mcp_server_auth_headers"] == {"deepwiki": {"authorization": "Bearer per-server"}} + assert call_kwargs["oauth2_headers"] == {"x-oauth": "1"} + assert call_kwargs["raw_headers"] == {"x-raw": "1"} + + assert captured["model"] == "MCP: deepwiki-read_wiki_structure" + assert captured["metadata"]["tool_name"] == "read_wiki_structure" + assert captured["input"][0]["content"]["arguments"] == {"repo": "litellm"} + assert results[0]["result"] == "wiki structure" + assert results[0]["display_name"] == "deepwiki-read_wiki_structure" + + +@pytest.mark.asyncio +async def test_execute_tool_calls_mcp_tool_call_rejects_tool_outside_allowed_tools(monkeypatch): + _setup_proxy_logging(monkeypatch) + _setup_mcp_call_environment(monkeypatch) + call_mock = AsyncMock(return_value=_text_result("should not run")) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_call", + call_mock, + ) + + results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( + tool_server_map=virtual_tool_server_map(), + tool_calls=[ + { + "id": "call-inner", + "function": { + "name": "mcp_tool_call", + "arguments": json.dumps({"tool_name": "deepwiki-delete_everything", "arguments": {}}), + }, + } + ], + user_api_key_auth=_auth(True), + virtual_tool_scope=LiteLLM_Proxy_MCP_Handler._build_virtual_tool_scope( + [_mcp_tool_config(allowed_tools=["read_wiki_structure"])] + ), + ) + + assert call_mock.await_count == 0 + assert "not in allowed_tools" in results[0]["result"] + + +@pytest.mark.asyncio +async def test_execute_tool_calls_ignores_virtual_names_without_scope(monkeypatch): + """A real upstream tool literally named mcp_tool_search must still dispatch + normally when the virtual tools are not in play.""" + call_tool_mock = _setup_mcp_call_environment(monkeypatch) + search_mock = AsyncMock(return_value=_text_result("[]")) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_search", + search_mock, + ) + + await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( + tool_server_map={"mcp_tool_search": "deepwiki"}, + tool_calls=[{"id": "c", "function": {"name": "mcp_tool_search", "arguments": "{}"}}], + user_api_key_auth=None, + ) + + assert search_mock.await_count == 0 + assert call_tool_mock.await_count == 1 diff --git a/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py b/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py index 5001589ce54..5045bac176e 100644 --- a/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py +++ b/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py @@ -336,3 +336,100 @@ async def test_streaming_follow_up_keeps_previous_response_id_when_stored(monkey follow_up_kwargs = aresponses_mock.call_args_list[0].kwargs assert follow_up_kwargs["previous_response_id"] == "resp_prev" assert not [item for item in follow_up_kwargs["input"] if item.get("type") == "reasoning"] +def _virtual_tool_iterator(initial_chunks) -> MCPEnhancedStreamingIterator: + from litellm.models.object_permission import LiteLLM_ObjectPermissionTable + from litellm.proxy._types import UserAPIKeyAuth + from litellm.responses.mcp.tool_search_bridge import virtual_tool_server_map + + return MCPEnhancedStreamingIterator( + base_iterator=_FakeAsyncStream(initial_chunks), + mcp_events=[], + tool_server_map=virtual_tool_server_map(), + mcp_tools_with_litellm_proxy=[ + { + "type": "mcp", + "server_url": "litellm_proxy/mcp/deepwiki", + "require_approval": "never", + } + ], + user_api_key_auth=UserAPIKeyAuth( + api_key="k", + user_id="u", + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id="test", mcp_tool_search_enabled=True + ), + ), + original_request_params={ + "model": "gpt-5.5", + "input": "what is berriai/litellm?", + "tools": [{"type": "mcp"}], + }, + ) + + +@pytest.mark.asyncio +async def test_streaming_virtual_tool_search_then_call_shows_inner_tool_name(monkeypatch): + """With mcp_tool_search_enabled the stream must run search -> call -> text, + dispatch through the virtual tool handlers (never the manager directly) and + label the mcp_call item with the real tool the model reached.""" + call_tool = _mock_mcp_environment(monkeypatch) + proxy_logging_obj = sys.modules["litellm.proxy.proxy_server"].proxy_logging_obj + proxy_logging_obj.post_mcp_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["response"]) + + search_mock = AsyncMock( + return_value=CallToolResult( + content=[TextContent(type="text", text='[{"name": "deepwiki-read_wiki_contents"}]')], isError=False + ) + ) + call_mock = AsyncMock( + return_value=CallToolResult(content=[TextContent(type="text", text="wiki body")], isError=False) + ) + monkeypatch.setattr("litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_search", search_mock) + monkeypatch.setattr("litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_call", call_mock) + + aresponses_mock = AsyncMock( + side_effect=[ + _FakeAsyncStream( + [ + _completed_chunk( + [ + _function_call( + "call_2", + "mcp_tool_call", + '{"tool_name": "deepwiki-read_wiki_contents", "arguments": {"repo": "x"}}', + ) + ] + ) + ] + ), + _text_only_stream("The wiki says hello."), + ] + ) + monkeypatch.setattr(responses_main_module, "aresponses", aresponses_mock) + + iterator = _virtual_tool_iterator( + [ + _output_item_added_chunk(), + _completed_chunk([_function_call("call_1", "mcp_tool_search", '{"query": "wiki"}')]), + ] + ) + + chunks = [chunk async for chunk in iterator] + + assert iterator.max_tool_call_rounds == MAX_MCP_TOOL_CALL_ROUNDS * 2 + assert search_mock.await_count == 1 + assert search_mock.await_args.kwargs["mcp_servers"] == ["deepwiki"] + assert call_mock.await_count == 1 + assert call_mock.await_args.kwargs["tool_name"] == "deepwiki-read_wiki_contents" + assert call_tool.call_count == 0 + + mcp_call_names = [ + chunk.item.name + for chunk in chunks + if getattr(chunk, "type", None) == ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE + and getattr(getattr(chunk, "item", None), "type", None) == "mcp_call" + ] + assert mcp_call_names == ["mcp_tool_search", "deepwiki-read_wiki_contents"] + + completed_chunks = [c for c in chunks if getattr(c, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED] + assert completed_chunks[-1].response.output[0]["content"][0]["text"] == "The wiki says hello." diff --git a/tests/test_litellm/responses/test_main.py b/tests/test_litellm/responses/test_main.py new file mode 100644 index 00000000000..e01bd16623f --- /dev/null +++ b/tests/test_litellm/responses/test_main.py @@ -0,0 +1,232 @@ +"""Tests for litellm/responses/main.py MCP auto-execution.""" + +import importlib +import json +import types +from unittest.mock import AsyncMock + +import pytest + +from litellm.models.object_permission import LiteLLM_ObjectPermissionTable +from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing +from litellm.proxy._types import UserAPIKeyAuth +from litellm.responses.main import aresponses_api_with_mcp +from litellm.responses.mcp.mcp_streaming_iterator import MAX_MCP_TOOL_CALL_ROUNDS +from litellm.types.llms.openai import ResponsesAPIResponse + +MCP_TOOL_CONFIG = { + "type": "mcp", + "server_label": "x", + "server_url": "litellm_proxy/mcp/deepwiki", + "require_approval": "never", +} + + +def _auth(tool_search_enabled: bool) -> UserAPIKeyAuth: + return UserAPIKeyAuth( + api_key="k", + user_id="u", + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id="test", + mcp_tool_search_enabled=tool_search_enabled, + ), + ) + + +def _function_call_response(response_id: str, call_id: str, name: str, arguments: dict) -> ResponsesAPIResponse: + return ResponsesAPIResponse( + id=response_id, + created_at=0, + model="gpt-5.5", + object="response", + status="completed", + output=[ + { + "type": "function_call", + "id": call_id, + "call_id": call_id, + "name": name, + "arguments": json.dumps(arguments), + } + ], + ) + + +def _text_response(response_id: str, text: str) -> ResponsesAPIResponse: + return ResponsesAPIResponse( + id=response_id, + created_at=0, + model="gpt-5.5", + object="response", + status="completed", + output=[ + { + "type": "message", + "id": "msg-1", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": text, "annotations": []}], + } + ], + ) + + +def _text_result(text: str): + from mcp.types import CallToolResult, TextContent + + return CallToolResult(content=[TextContent(type="text", text=text)], isError=False) + + +def _patch_aresponses(monkeypatch, responses): + """Patch both references to aresponses (main.py's own and the handler's follow-up import).""" + mock = AsyncMock(side_effect=list(responses)) + main_module = importlib.import_module("litellm.responses.main") + handler_module = importlib.import_module("litellm.responses.mcp.litellm_proxy_mcp_handler") + monkeypatch.setattr(main_module, "aresponses", mock) + monkeypatch.setattr(handler_module, "aresponses", mock) + return mock + + +def _patch_proxy_globals(monkeypatch): + import sys + + from unittest.mock import MagicMock + + proxy_logging_obj = MagicMock() + proxy_logging_obj.post_call_failure_hook = AsyncMock() + proxy_logging_obj.post_mcp_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["response"]) + monkeypatch.setitem( + sys.modules, + "litellm.proxy.proxy_server", + types.SimpleNamespace(proxy_logging_obj=proxy_logging_obj), + ) + fake_manager = types.SimpleNamespace( + get_registry=MagicMock(return_value={}), + call_tool=AsyncMock(), + _get_mcp_server_from_tool_name=MagicMock(return_value=None), + get_mcp_server_by_name=MagicMock(return_value=None), + get_allowed_mcp_servers=AsyncMock(return_value=[]), + get_mcp_servers_from_ids=MagicMock(return_value=[]), + ) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + fake_manager, + ) + + +@pytest.mark.asyncio +async def test_search_then_call_then_answer_completes_in_one_request(monkeypatch): + """The two-hop virtual tool flow (search -> call -> text) must finish inside a + single client request; a single-round implementation would return the + unexecuted mcp_tool_call instead.""" + _patch_proxy_globals(monkeypatch) + + search_mock = AsyncMock( + return_value=_text_result( + json.dumps([{"name": "deepwiki-read_wiki_structure", "description": "d", "inputSchema": {}}]) + ) + ) + call_mock = AsyncMock(return_value=_text_result("wiki structure")) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_search", search_mock + ) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_call", call_mock + ) + + aresponses_mock = _patch_aresponses( + monkeypatch, + [ + _function_call_response("resp-1", "call-1", "mcp_tool_search", {"query": "wiki"}), + _function_call_response( + "resp-2", + "call-2", + "mcp_tool_call", + {"tool_name": "deepwiki-read_wiki_structure", "arguments": {"repo": "litellm"}}, + ), + _text_response("resp-3", "The wiki has 3 sections."), + ], + ) + + final = await aresponses_api_with_mcp( + input="describe the wiki", + model="gpt-5.5", + tools=[MCP_TOOL_CONFIG], + litellm_metadata={"user_api_key_auth": _auth(True)}, + user_api_key_auth=_auth(True), + ) + + assert aresponses_mock.await_count == 3 + assert search_mock.await_count == 1 + assert call_mock.await_count == 1 + assert call_mock.await_args.kwargs["tool_name"] == "deepwiki-read_wiki_structure" + + # Only the two virtual tools ever reach the model. + first_call_tools = aresponses_mock.await_args_list[0].kwargs["tools"] + assert [tool["name"] for tool in first_call_tools] == ["mcp_tool_search", "mcp_tool_call"] + + assert isinstance(final, ResponsesAPIResponse) + output_types = [item.get("type") if isinstance(item, dict) else item.type for item in final.output] + assert "tool_execution_results" in output_types + tool_results_item = next( + item for item in final.output if isinstance(item, dict) and item.get("type") == "tool_execution_results" + ) + logged_results = json.loads(tool_results_item["content"][0]["text"]) + assert [entry["name"] for entry in logged_results] == ["mcp_tool_search", "mcp_tool_call"] + assert [entry["display_name"] for entry in logged_results] == [ + "mcp_tool_search", + "deepwiki-read_wiki_structure", + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("tool_search_enabled", [True, False], ids=["virtual_tools", "plain_catalog"]) +async def test_auto_execute_round_cap(monkeypatch, tool_search_enabled): + """A model that keeps calling tools is capped and the final follow-up drops + tools so it has to answer in text. The virtual tools spend two rounds per + real tool use, so the flagged path gets twice the budget.""" + from mcp.types import Tool + + _patch_proxy_globals(monkeypatch) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.server._get_tools_from_mcp_servers", + AsyncMock( + return_value=AggregateToolListing( + tools=[Tool(name="deepwiki-read_wiki_contents", description="d", inputSchema={})], outcomes={} + ) + ), + ) + executed = AsyncMock(return_value=_text_result("[]")) + if tool_search_enabled: + called_tool = "mcp_tool_search" + expected_cap = MAX_MCP_TOOL_CALL_ROUNDS * 2 + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_search", executed + ) + else: + called_tool = "deepwiki-read_wiki_contents" + expected_cap = MAX_MCP_TOOL_CALL_ROUNDS + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager.call_tool", + executed, + ) + + aresponses_mock = _patch_aresponses( + monkeypatch, + [ + _function_call_response(f"resp-{i}", f"call-{i}", called_tool, {"query": "wiki"}) + for i in range(MAX_MCP_TOOL_CALL_ROUNDS * 2 + 5) + ], + ) + + await aresponses_api_with_mcp( + input="describe the wiki", + model="gpt-5.5", + tools=[MCP_TOOL_CONFIG], + litellm_metadata={"user_api_key_auth": _auth(tool_search_enabled)}, + user_api_key_auth=_auth(tool_search_enabled), + ) + + assert executed.await_count == expected_cap + assert aresponses_mock.await_count == expected_cap + 1 + assert aresponses_mock.await_args_list[-1].kwargs["tools"] is None