mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge 9f0cfce792 into 252c71c0b2
This commit is contained in:
commit
dc83339a11
11 changed files with 1401 additions and 183 deletions
|
|
@ -174,6 +174,11 @@ MCP_PER_USER_TOKEN_DEFAULT_TTL: Final = int(
|
|||
)
|
||||
MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS: Final = int(os.getenv("MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS", "60"))
|
||||
|
||||
# Sentinel server name for the mcp_tool_search / mcp_tool_call virtual tools,
|
||||
# stored in the Responses API tool_server_map to tell them apart from a real
|
||||
# upstream tool of the same name.
|
||||
MCP_VIRTUAL_TOOL_SEARCH_SERVER_NAME = "litellm_virtual_mcp_tool_search"
|
||||
|
||||
# MCP timeout defaults (seconds). Override via env vars for slow/custom MCP servers.
|
||||
MCP_CLIENT_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_CLIENT_TIMEOUT", "60.0"))
|
||||
MCP_TOOL_LISTING_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_TOOL_LISTING_TIMEOUT", "30.0"))
|
||||
|
|
|
|||
|
|
@ -84,6 +84,10 @@ module materially harder to understand.
|
|||
permissions, no-accessible-server rejection, per-request auth headers, server
|
||||
scope, error to `isError` conversion, and spend logging. Reuse `_list_mcp_tools`
|
||||
and `execute_mcp_tool` rather than reimplementing any of these checks.
|
||||
- The same flag also drives the Responses API / chat completions `type: "mcp"`
|
||||
tools that point at `litellm_proxy/`. That bridge lives in
|
||||
`litellm/responses/mcp/tool_search_bridge.py` and calls `handle_mcp_tool_search`
|
||||
/ `handle_mcp_tool_call` here; keep those two entry points backwards compatible.
|
||||
|
||||
## Tests
|
||||
|
||||
|
|
|
|||
|
|
@ -60,6 +60,8 @@ from litellm.utils import (
|
|||
if TYPE_CHECKING:
|
||||
from fastapi import WebSocket
|
||||
from mcp.types import Tool as MCPTool
|
||||
|
||||
from litellm.responses.mcp.litellm_proxy_mcp_handler import MCPToolResult
|
||||
else:
|
||||
MCPTool = Any
|
||||
|
||||
|
|
@ -174,6 +176,7 @@ async def aresponses_api_with_mcp(
|
|||
from litellm.responses.mcp.litellm_proxy_mcp_handler import (
|
||||
LiteLLM_Proxy_MCP_Handler,
|
||||
)
|
||||
from litellm.responses.mcp.tool_search_bridge import is_mcp_tool_search_enabled
|
||||
|
||||
# Parse MCP tools and separate from other tools
|
||||
(
|
||||
|
|
@ -211,6 +214,12 @@ async def aresponses_api_with_mcp(
|
|||
)
|
||||
openai_tools: Final = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai(original_mcp_tools)
|
||||
|
||||
virtual_tool_scope = (
|
||||
LiteLLM_Proxy_MCP_Handler._build_virtual_tool_scope(mcp_tools_with_litellm_proxy)
|
||||
if is_mcp_tool_search_enabled(user_api_key_auth)
|
||||
else None
|
||||
)
|
||||
|
||||
# Combine with other tools
|
||||
all_tools: Final = openai_tools + other_tools if (openai_tools or other_tools) else None
|
||||
|
||||
|
|
@ -300,105 +309,104 @@ async def aresponses_api_with_mcp(
|
|||
# Auto-Execute Tools Handling
|
||||
# If auto-execute tools is True, then we need to execute the tool calls
|
||||
#########################################################
|
||||
if should_auto_execute and isinstance(response, ResponsesAPIResponse):
|
||||
tool_calls: Final = LiteLLM_Proxy_MCP_Handler._extract_tool_calls_from_response(response=response)
|
||||
if not (should_auto_execute and isinstance(response, ResponsesAPIResponse)):
|
||||
return response
|
||||
|
||||
if tool_calls:
|
||||
user_api_key_auth = kwargs.get("litellm_metadata", {}).get("user_api_key_auth")
|
||||
from litellm.responses.mcp.tool_search_bridge import max_tool_call_rounds
|
||||
|
||||
# Extract MCP auth headers from the request to pass to MCP server
|
||||
secret_fields = kwargs.get("secret_fields")
|
||||
(
|
||||
mcp_auth_header,
|
||||
mcp_server_auth_headers,
|
||||
oauth2_headers,
|
||||
raw_headers_from_request,
|
||||
) = ResponsesAPIRequestUtils.extract_mcp_headers_from_request(
|
||||
secret_fields=secret_fields,
|
||||
tools=tools,
|
||||
max_rounds = max_tool_call_rounds(virtual_tool_scope)
|
||||
persistence_disabled: Final = LiteLLM_Proxy_MCP_Handler._is_persistence_disabled(call_params)
|
||||
|
||||
# Extract MCP auth headers from the request to pass to MCP server
|
||||
(
|
||||
mcp_auth_header,
|
||||
mcp_server_auth_headers,
|
||||
oauth2_headers,
|
||||
raw_headers_from_request,
|
||||
) = ResponsesAPIRequestUtils.extract_mcp_headers_from_request(
|
||||
secret_fields=kwargs.get("secret_fields"),
|
||||
tools=tools,
|
||||
)
|
||||
|
||||
current_response: ResponsesAPIResponse = response
|
||||
conversation_input: Any = input
|
||||
all_tool_results: Final[list[MCPToolResult]] = [] # mutable-ok: round accumulator
|
||||
completed_rounds = 0
|
||||
|
||||
while completed_rounds < max_rounds:
|
||||
tool_calls = LiteLLM_Proxy_MCP_Handler._extract_tool_calls_from_response(response=current_response)
|
||||
if not tool_calls:
|
||||
break
|
||||
|
||||
tool_results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls(
|
||||
tool_server_map=tool_server_map,
|
||||
tool_calls=tool_calls,
|
||||
user_api_key_auth=user_api_key_auth,
|
||||
mcp_auth_header=mcp_auth_header,
|
||||
mcp_server_auth_headers=mcp_server_auth_headers,
|
||||
oauth2_headers=oauth2_headers,
|
||||
raw_headers=raw_headers_from_request,
|
||||
litellm_call_id=kwargs.get("litellm_call_id"),
|
||||
litellm_trace_id=kwargs.get("litellm_trace_id"),
|
||||
request_tags=LiteLLM_Proxy_MCP_Handler._get_parent_request_tags(kwargs),
|
||||
virtual_tool_scope=virtual_tool_scope,
|
||||
)
|
||||
if not tool_results:
|
||||
break
|
||||
|
||||
completed_rounds += 1
|
||||
all_tool_results.extend(tool_results)
|
||||
|
||||
follow_up_input = LiteLLM_Proxy_MCP_Handler._create_follow_up_input(
|
||||
response=current_response,
|
||||
tool_results=tool_results,
|
||||
original_input=conversation_input,
|
||||
preserve_reasoning=persistence_disabled,
|
||||
)
|
||||
|
||||
follow_up_call_params = LiteLLM_Proxy_MCP_Handler._prepare_follow_up_call_params(
|
||||
call_params=call_params, original_stream_setting=stream or False
|
||||
)
|
||||
|
||||
follow_up_tools = all_tools if completed_rounds < max_rounds else None
|
||||
if follow_up_tools is None:
|
||||
verbose_logger.warning(
|
||||
"MCP auto-execute hit its %s-round cap; forcing a text-only follow-up.",
|
||||
max_rounds,
|
||||
)
|
||||
|
||||
tool_results: Final = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls(
|
||||
tool_server_map=tool_server_map,
|
||||
tool_calls=tool_calls,
|
||||
user_api_key_auth=user_api_key_auth,
|
||||
mcp_auth_header=mcp_auth_header,
|
||||
mcp_server_auth_headers=mcp_server_auth_headers,
|
||||
oauth2_headers=oauth2_headers,
|
||||
raw_headers=raw_headers_from_request,
|
||||
litellm_call_id=kwargs.get("litellm_call_id"),
|
||||
litellm_trace_id=kwargs.get("litellm_trace_id"),
|
||||
request_tags=LiteLLM_Proxy_MCP_Handler._get_parent_request_tags(kwargs),
|
||||
)
|
||||
follow_up_response = await LiteLLM_Proxy_MCP_Handler._make_follow_up_call(
|
||||
follow_up_input=follow_up_input,
|
||||
model=model,
|
||||
all_tools=follow_up_tools,
|
||||
response_id=previous_response_id if persistence_disabled else current_response.id,
|
||||
**follow_up_call_params,
|
||||
)
|
||||
if not isinstance(follow_up_response, ResponsesAPIResponse):
|
||||
return follow_up_response
|
||||
|
||||
if tool_results:
|
||||
persistence_disabled: Final = LiteLLM_Proxy_MCP_Handler._is_persistence_disabled(call_params)
|
||||
conversation_input = follow_up_input
|
||||
current_response = follow_up_response
|
||||
|
||||
follow_up_input: Final = LiteLLM_Proxy_MCP_Handler._create_follow_up_input(
|
||||
response=response,
|
||||
tool_results=tool_results,
|
||||
original_input=input,
|
||||
preserve_reasoning=persistence_disabled,
|
||||
)
|
||||
if not all_tool_results:
|
||||
return response
|
||||
|
||||
# Prepare parameters for follow-up call (restores original stream setting)
|
||||
follow_up_call_params: Final = LiteLLM_Proxy_MCP_Handler._prepare_follow_up_call_params(
|
||||
call_params=call_params, original_stream_setting=stream or False
|
||||
)
|
||||
|
||||
# Create tool execution events for streaming if needed
|
||||
tool_execution_events = []
|
||||
if stream:
|
||||
tool_execution_events = LiteLLM_Proxy_MCP_Handler._create_tool_execution_events(
|
||||
tool_calls=tool_calls, tool_results=tool_results
|
||||
)
|
||||
|
||||
final_response = await LiteLLM_Proxy_MCP_Handler._make_follow_up_call(
|
||||
follow_up_input=follow_up_input,
|
||||
model=model,
|
||||
all_tools=all_tools,
|
||||
response_id=previous_response_id if persistence_disabled else response.id,
|
||||
**follow_up_call_params,
|
||||
)
|
||||
|
||||
# If streaming and we have tool execution events, wrap the response
|
||||
if (
|
||||
stream
|
||||
and tool_execution_events
|
||||
and (hasattr(final_response, "__aiter__") or hasattr(final_response, "__iter__"))
|
||||
):
|
||||
from litellm.responses.mcp.mcp_streaming_iterator import (
|
||||
MCPEnhancedStreamingIterator,
|
||||
)
|
||||
|
||||
final_response = MCPEnhancedStreamingIterator(
|
||||
tool_server_map=tool_server_map,
|
||||
base_iterator=final_response,
|
||||
mcp_events=tool_execution_events,
|
||||
user_api_key_auth=user_api_key_auth,
|
||||
)
|
||||
|
||||
# Add custom output elements to the final response (for non-streaming)
|
||||
elif isinstance(final_response, ResponsesAPIResponse):
|
||||
# Fetch MCP tools again for output elements (without OpenAI transformation)
|
||||
(
|
||||
mcp_tools_for_output,
|
||||
_,
|
||||
) = await LiteLLM_Proxy_MCP_Handler._process_mcp_tools_without_openai_transform(
|
||||
user_api_key_auth=user_api_key_auth,
|
||||
mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy,
|
||||
mcp_auth_header=mcp_auth_header,
|
||||
mcp_server_auth_headers=mcp_server_auth_headers,
|
||||
request_tags=LiteLLM_Proxy_MCP_Handler._get_parent_request_tags(kwargs),
|
||||
)
|
||||
final_response = LiteLLM_Proxy_MCP_Handler._add_mcp_output_elements_to_response(
|
||||
response=final_response,
|
||||
mcp_tools_fetched=mcp_tools_for_output,
|
||||
tool_results=tool_results,
|
||||
)
|
||||
return final_response
|
||||
|
||||
return response
|
||||
# Fetch MCP tools again for output elements (without OpenAI transformation)
|
||||
(
|
||||
mcp_tools_for_output,
|
||||
_,
|
||||
) = await LiteLLM_Proxy_MCP_Handler._process_mcp_tools_without_openai_transform(
|
||||
user_api_key_auth=user_api_key_auth,
|
||||
mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy,
|
||||
mcp_auth_header=mcp_auth_header,
|
||||
mcp_server_auth_headers=mcp_server_auth_headers,
|
||||
request_tags=LiteLLM_Proxy_MCP_Handler._get_parent_request_tags(kwargs),
|
||||
)
|
||||
return LiteLLM_Proxy_MCP_Handler._add_mcp_output_elements_to_response(
|
||||
response=current_response,
|
||||
mcp_tools_fetched=mcp_tools_for_output,
|
||||
tool_results=all_tool_results,
|
||||
)
|
||||
|
||||
|
||||
def _bridges_to_chat_completions(
|
||||
|
|
|
|||
|
|
@ -5,15 +5,21 @@ from typing import TYPE_CHECKING, Final, cast
|
|||
|
||||
from typing_extensions import TypedDict, Unpack
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.responses.mcp.litellm_proxy_mcp_handler import (
|
||||
LiteLLM_Proxy_MCP_Handler,
|
||||
)
|
||||
from litellm.responses.mcp.request_context import MCPRequestContext
|
||||
from litellm.responses.mcp.tool_search_bridge import (
|
||||
is_mcp_tool_search_enabled,
|
||||
max_tool_call_rounds,
|
||||
)
|
||||
from litellm.types.utils import Message, ModelResponse
|
||||
from litellm.utils import CustomStreamWrapper
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.responses.mcp.litellm_proxy_mcp_handler import MCPToolResult
|
||||
|
||||
|
||||
class _MCPCompletionKwargs(TypedDict, total=False, extra_items=object):
|
||||
|
|
@ -126,6 +132,14 @@ async def acompletion_with_mcp(
|
|||
oauth2_headers: Final = context.oauth2_headers
|
||||
raw_headers: Final = context.raw_headers
|
||||
|
||||
stream: Final[object] = kwargs.get("stream", False)
|
||||
|
||||
if stream and is_mcp_tool_search_enabled(user_api_key_auth):
|
||||
verbose_logger.debug(
|
||||
"mcp_tool_search_enabled is set but streaming /chat/completions only supports a single "
|
||||
"tool-call round; falling back to the full catalog."
|
||||
)
|
||||
|
||||
# Process MCP tools (pass auth headers for dynamic auth)
|
||||
(
|
||||
deduplicated_mcp_tools,
|
||||
|
|
@ -137,6 +151,13 @@ async def acompletion_with_mcp(
|
|||
mcp_auth_header=mcp_auth_header,
|
||||
mcp_server_auth_headers=mcp_server_auth_headers,
|
||||
request_tags=request_tags,
|
||||
allow_virtual_tools=not stream,
|
||||
)
|
||||
|
||||
virtual_tool_scope = (
|
||||
LiteLLM_Proxy_MCP_Handler._build_virtual_tool_scope(mcp_tools_with_litellm_proxy)
|
||||
if not stream and is_mcp_tool_search_enabled(user_api_key_auth)
|
||||
else None
|
||||
)
|
||||
|
||||
openai_tools: Final = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai(
|
||||
|
|
@ -175,7 +196,6 @@ async def acompletion_with_mcp(
|
|||
return response
|
||||
|
||||
# For auto-execute: handle streaming vs non-streaming differently
|
||||
stream: Final[object] = kwargs.get("stream", False)
|
||||
mock_tool_calls: Final = base_call_args.pop("mock_tool_calls", None)
|
||||
|
||||
if stream:
|
||||
|
|
@ -592,56 +612,73 @@ async def acompletion_with_mcp(
|
|||
if not isinstance(initial_response, ModelResponse):
|
||||
return initial_response
|
||||
|
||||
# Extract tool calls from response
|
||||
tool_calls: Final = LiteLLM_Proxy_MCP_Handler._extract_tool_calls_from_chat_response(response=initial_response)
|
||||
max_rounds: Final = max_tool_call_rounds(virtual_tool_scope)
|
||||
current_response: ModelResponse = initial_response
|
||||
current_messages: list[object] = messages # mutable-ok: chat follow-up takes list
|
||||
all_tool_calls: Final[list[object]] = [] # mutable-ok: round accumulator
|
||||
all_tool_results: Final[list[MCPToolResult]] = [] # mutable-ok: round accumulator
|
||||
completed_rounds = 0
|
||||
|
||||
if not tool_calls:
|
||||
_add_mcp_metadata_to_response(
|
||||
response=initial_response,
|
||||
openai_tools=openai_tools,
|
||||
)
|
||||
return initial_response
|
||||
while completed_rounds < max_rounds:
|
||||
tool_calls = LiteLLM_Proxy_MCP_Handler._extract_tool_calls_from_chat_response(response=current_response)
|
||||
if not tool_calls:
|
||||
break
|
||||
all_tool_calls.extend(tool_calls)
|
||||
|
||||
# Execute tool calls
|
||||
tool_results: Final = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls(
|
||||
tool_server_map=tool_server_map,
|
||||
tool_calls=tool_calls,
|
||||
user_api_key_auth=user_api_key_auth,
|
||||
mcp_auth_header=mcp_auth_header,
|
||||
mcp_server_auth_headers=mcp_server_auth_headers,
|
||||
oauth2_headers=oauth2_headers,
|
||||
raw_headers=raw_headers,
|
||||
litellm_call_id=context.litellm_call_id,
|
||||
litellm_trace_id=context.litellm_trace_id,
|
||||
request_tags=request_tags,
|
||||
)
|
||||
|
||||
if not tool_results:
|
||||
_add_mcp_metadata_to_response(
|
||||
response=initial_response,
|
||||
openai_tools=openai_tools,
|
||||
tool_results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls(
|
||||
tool_server_map=tool_server_map,
|
||||
tool_calls=tool_calls,
|
||||
user_api_key_auth=user_api_key_auth,
|
||||
mcp_auth_header=mcp_auth_header,
|
||||
mcp_server_auth_headers=mcp_server_auth_headers,
|
||||
oauth2_headers=oauth2_headers,
|
||||
raw_headers=raw_headers,
|
||||
litellm_call_id=context.litellm_call_id,
|
||||
litellm_trace_id=context.litellm_trace_id,
|
||||
request_tags=request_tags,
|
||||
virtual_tool_scope=virtual_tool_scope,
|
||||
)
|
||||
return initial_response
|
||||
if not tool_results:
|
||||
break
|
||||
|
||||
# Create follow-up messages with tool results
|
||||
follow_up_messages: Final = LiteLLM_Proxy_MCP_Handler._create_follow_up_messages_for_chat(
|
||||
original_messages=messages,
|
||||
response=initial_response,
|
||||
tool_results=tool_results,
|
||||
)
|
||||
completed_rounds += 1
|
||||
all_tool_results.extend(tool_results)
|
||||
|
||||
# Make follow-up call with original stream setting
|
||||
follow_up_call_args: Final = dict(base_call_args)
|
||||
follow_up_call_args["messages"] = follow_up_messages
|
||||
follow_up_call_args["stream"] = stream
|
||||
|
||||
response = await litellm_acompletion(**follow_up_call_args)
|
||||
if isinstance(response, (ModelResponse, CustomStreamWrapper)):
|
||||
_add_mcp_metadata_to_response(
|
||||
response=response,
|
||||
openai_tools=openai_tools,
|
||||
tool_calls=tool_calls,
|
||||
follow_up_messages = LiteLLM_Proxy_MCP_Handler._create_follow_up_messages_for_chat(
|
||||
original_messages=current_messages,
|
||||
response=current_response,
|
||||
tool_results=tool_results,
|
||||
)
|
||||
return response
|
||||
|
||||
follow_up_call_args = dict(base_call_args)
|
||||
follow_up_call_args["messages"] = follow_up_messages
|
||||
follow_up_call_args["stream"] = stream
|
||||
if completed_rounds >= max_rounds:
|
||||
follow_up_call_args["tools"] = None
|
||||
verbose_logger.warning(
|
||||
"MCP auto-execute hit its %s-round cap; forcing a text-only follow-up.",
|
||||
max_rounds,
|
||||
)
|
||||
|
||||
follow_up_response = await litellm_acompletion(**follow_up_call_args)
|
||||
current_messages = follow_up_messages
|
||||
|
||||
if not isinstance(follow_up_response, ModelResponse):
|
||||
if isinstance(follow_up_response, CustomStreamWrapper):
|
||||
_add_mcp_metadata_to_response(
|
||||
response=follow_up_response,
|
||||
openai_tools=openai_tools,
|
||||
tool_calls=all_tool_calls,
|
||||
tool_results=all_tool_results,
|
||||
)
|
||||
return follow_up_response
|
||||
|
||||
current_response = follow_up_response
|
||||
|
||||
_add_mcp_metadata_to_response(
|
||||
response=current_response,
|
||||
openai_tools=openai_tools,
|
||||
tool_calls=all_tool_calls or None,
|
||||
tool_results=all_tool_results or None,
|
||||
)
|
||||
return current_response
|
||||
|
|
|
|||
|
|
@ -6,9 +6,10 @@ from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypeAlias, Type
|
|||
|
||||
from openai.types.chat import ChatCompletionToolParam
|
||||
from openai.types.responses.function_tool_param import FunctionToolParam
|
||||
from typing_extensions import NotRequired, ReadOnly
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import MAXIMUM_TRACEBACK_LINES_TO_LOG
|
||||
from litellm.constants import MAXIMUM_TRACEBACK_LINES_TO_LOG, MCP_VIRTUAL_TOOL_SEARCH_SERVER_NAME
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.proxy._experimental.mcp_server.utils import (
|
||||
iter_known_server_prefixes,
|
||||
|
|
@ -17,6 +18,16 @@ from litellm.proxy._experimental.mcp_server.utils import (
|
|||
strip_known_server_prefix,
|
||||
)
|
||||
from litellm.responses.main import aresponses
|
||||
from litellm.responses.mcp.tool_search_bridge import (
|
||||
VirtualToolCallPlan,
|
||||
VirtualToolScope,
|
||||
is_mcp_tool_search_enabled,
|
||||
resolve_virtual_tool_call,
|
||||
run_virtual_tool_call,
|
||||
run_virtual_tool_search,
|
||||
virtual_tool_definitions,
|
||||
virtual_tool_server_map,
|
||||
)
|
||||
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
|
||||
from litellm.types.llms.openai import (
|
||||
ResponseInputParam,
|
||||
|
|
@ -54,6 +65,7 @@ class MCPToolResult(TypedDict):
|
|||
tool_call_id: str | None
|
||||
result: str
|
||||
name: str | None
|
||||
display_name: NotRequired[ReadOnly[str]]
|
||||
|
||||
|
||||
LITELLM_PROXY_MCP_SERVER_URL: Final = "litellm_proxy"
|
||||
|
|
@ -258,12 +270,10 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
_get_tools_from_mcp_servers,
|
||||
)
|
||||
|
||||
mcp_servers: Final = [
|
||||
server_url.split("/")[-1]
|
||||
for _tool in (mcp_tools_with_litellm_proxy or ())
|
||||
for server_url in (_tool.get("server_url", "") if isinstance(_tool, dict) else "",)
|
||||
if isinstance(server_url, str) and server_url.startswith(LITELLM_PROXY_MCP_SERVER_URL_PREFIX)
|
||||
]
|
||||
# if user specifies servers as server_url: litellm_proxy/mcp/zapier,github then return zapier,github
|
||||
mcp_servers: Final = list(
|
||||
LiteLLM_Proxy_MCP_Handler._extract_mcp_server_names(mcp_tools_with_litellm_proxy or ())
|
||||
)
|
||||
|
||||
# Resolve toolset names: collect all toolset IDs first, then apply their
|
||||
# combined permissions in a single pass so multiple toolsets are unioned
|
||||
|
|
@ -383,18 +393,47 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
|
||||
return deduplicated_tools, tool_server_map
|
||||
|
||||
@staticmethod
|
||||
def _collect_allowed_tool_names(
|
||||
mcp_tools_with_litellm_proxy: Iterable[ToolParam],
|
||||
) -> frozenset[str]:
|
||||
"""Union of the ``allowed_tools`` names declared across the request's MCP tool configs."""
|
||||
return frozenset(
|
||||
tool_name
|
||||
for tool_config in mcp_tools_with_litellm_proxy
|
||||
if isinstance(tool_config, dict) and isinstance(allowed := tool_config.get("allowed_tools"), list)
|
||||
for tool_name in allowed
|
||||
if isinstance(tool_name, str)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _extract_mcp_server_names(
|
||||
mcp_tools_with_litellm_proxy: Iterable[ToolParam],
|
||||
) -> tuple[str, ...]:
|
||||
"""Server aliases named by ``litellm_proxy/mcp/<alias>`` server_urls, in request order."""
|
||||
return tuple(
|
||||
server_url.split("/")[-1]
|
||||
for tool_config in mcp_tools_with_litellm_proxy
|
||||
if isinstance(tool_config, dict)
|
||||
and isinstance(server_url := tool_config.get("server_url", ""), str)
|
||||
and server_url.startswith(LITELLM_PROXY_MCP_SERVER_URL_PREFIX)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _build_virtual_tool_scope(
|
||||
mcp_tools_with_litellm_proxy: Sequence[Mapping[str, object]],
|
||||
) -> VirtualToolScope:
|
||||
return VirtualToolScope(
|
||||
mcp_servers=LiteLLM_Proxy_MCP_Handler._extract_mcp_server_names(mcp_tools_with_litellm_proxy),
|
||||
allowed_tools=LiteLLM_Proxy_MCP_Handler._collect_allowed_tool_names(mcp_tools_with_litellm_proxy),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _filter_mcp_tools_by_allowed_tools(
|
||||
mcp_tools: list[MCPTool], mcp_tools_with_litellm_proxy: Sequence[Mapping[str, object]]
|
||||
) -> list[MCPTool]:
|
||||
"""Filter MCP tools based on allowed_tools parameter from the original tool configs."""
|
||||
# Collect all allowed tool names from all MCP tool configs
|
||||
allowed_tool_names: Final[set[str]] = set()
|
||||
for tool_config in mcp_tools_with_litellm_proxy:
|
||||
if isinstance(tool_config, dict) and "allowed_tools" in tool_config:
|
||||
allowed_tools = tool_config.get("allowed_tools", [])
|
||||
if isinstance(allowed_tools, list):
|
||||
allowed_tool_names.update(allowed_tools)
|
||||
allowed_tool_names: Final = LiteLLM_Proxy_MCP_Handler._collect_allowed_tool_names(mcp_tools_with_litellm_proxy)
|
||||
|
||||
# If no allowed_tools specified, return all tools
|
||||
if not allowed_tool_names:
|
||||
|
|
@ -452,6 +491,7 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
mcp_auth_header: str | None = None,
|
||||
mcp_server_auth_headers: dict[str, dict[str, str]] | None = None,
|
||||
request_tags: list[str] | None = None,
|
||||
allow_virtual_tools: bool = True,
|
||||
) -> tuple[list[MCPTool], dict[str, str]]:
|
||||
"""
|
||||
Process MCP tools through filtering and deduplication pipeline without OpenAI transformation.
|
||||
|
|
@ -469,6 +509,15 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
if not mcp_tools_with_litellm_proxy:
|
||||
return [], {}
|
||||
|
||||
if allow_virtual_tools and is_mcp_tool_search_enabled(user_api_key_auth):
|
||||
if LiteLLM_Proxy_MCP_Handler._should_auto_execute_tools(mcp_tools_with_litellm_proxy):
|
||||
# mutable-ok: the caller's contract is tuple[list[MCPTool], dict[str, str]]
|
||||
return list(virtual_tool_definitions()), dict(virtual_tool_server_map())
|
||||
verbose_logger.debug(
|
||||
"mcp_tool_search_enabled is set but no MCP tool has require_approval='never'; "
|
||||
"the virtual tools cannot be auto-executed, falling back to the full catalog."
|
||||
)
|
||||
|
||||
typed_user_api_key_auth: Final[UserAPIKeyAuth | None] = user_api_key_auth
|
||||
|
||||
# Step 1: Fetch MCP tools from manager
|
||||
|
|
@ -679,6 +728,44 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
|
||||
return result_text or "Tool executed successfully"
|
||||
|
||||
@staticmethod
|
||||
async def _resolve_virtual_tool_call_step(
|
||||
tool_name: str,
|
||||
parsed_arguments: Mapping[str, object],
|
||||
virtual_tool_scope: VirtualToolScope,
|
||||
user_api_key_auth: "UserAPIKeyAuth",
|
||||
mcp_auth_header: str | None,
|
||||
mcp_server_auth_headers: dict[str, dict[str, str]] | None, # mutable-ok: handle_mcp_tool_* takes dict
|
||||
oauth2_headers: dict[str, str] | None, # mutable-ok: handle_mcp_tool_* takes dict
|
||||
raw_headers: dict[str, str] | None, # mutable-ok: handle_mcp_tool_* takes dict
|
||||
) -> VirtualToolCallPlan | str:
|
||||
"""Run one virtual tool hop.
|
||||
|
||||
Returns finished result text for ``mcp_tool_search`` and for a rejected
|
||||
``mcp_tool_call``, or the plan naming the real tool that
|
||||
``mcp_tool_call`` unwraps to, so the caller spend-logs it as that tool.
|
||||
"""
|
||||
from litellm.proxy._experimental.mcp_server.tool_search import (
|
||||
MCP_TOOL_SEARCH_TOOL_NAME,
|
||||
coerce_top_k,
|
||||
)
|
||||
|
||||
if tool_name == MCP_TOOL_SEARCH_TOOL_NAME:
|
||||
query = parsed_arguments.get("query", "")
|
||||
search_result = await run_virtual_tool_search(
|
||||
query=query if isinstance(query, str) else "",
|
||||
top_k=coerce_top_k(parsed_arguments.get("top_k", 5)),
|
||||
scope=virtual_tool_scope,
|
||||
user_api_key_auth=user_api_key_auth,
|
||||
mcp_auth_header=mcp_auth_header,
|
||||
mcp_server_auth_headers=mcp_server_auth_headers,
|
||||
oauth2_headers=oauth2_headers,
|
||||
raw_headers=raw_headers,
|
||||
)
|
||||
return LiteLLM_Proxy_MCP_Handler._parse_mcp_result(search_result)
|
||||
|
||||
return resolve_virtual_tool_call(parsed_arguments, virtual_tool_scope)
|
||||
|
||||
@staticmethod
|
||||
async def _execute_tool_calls(
|
||||
tool_server_map: dict[str, str],
|
||||
|
|
@ -691,6 +778,7 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
litellm_call_id: str | None = None,
|
||||
litellm_trace_id: str | None = None,
|
||||
request_tags: list[str] | None = None,
|
||||
virtual_tool_scope: VirtualToolScope | None = None,
|
||||
) -> list[MCPToolResult]:
|
||||
"""Execute tool calls and return results."""
|
||||
from fastapi import HTTPException
|
||||
|
|
@ -729,13 +817,50 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
# Import here to avoid circular import
|
||||
from litellm.proxy.proxy_server import proxy_logging_obj
|
||||
|
||||
server_name = tool_server_map[tool_name]
|
||||
virtual_call_plan: VirtualToolCallPlan | None = None
|
||||
if (
|
||||
virtual_tool_scope is not None
|
||||
and typed_user_api_key_auth is not None
|
||||
and tool_server_map.get(tool_name) == MCP_VIRTUAL_TOOL_SEARCH_SERVER_NAME
|
||||
):
|
||||
virtual_outcome = await LiteLLM_Proxy_MCP_Handler._resolve_virtual_tool_call_step(
|
||||
tool_name=tool_name,
|
||||
parsed_arguments=parsed_arguments,
|
||||
virtual_tool_scope=virtual_tool_scope,
|
||||
user_api_key_auth=typed_user_api_key_auth,
|
||||
mcp_auth_header=mcp_auth_header,
|
||||
mcp_server_auth_headers=mcp_server_auth_headers,
|
||||
oauth2_headers=oauth2_headers,
|
||||
raw_headers=raw_headers,
|
||||
)
|
||||
if isinstance(virtual_outcome, str):
|
||||
tool_results.append(
|
||||
{
|
||||
"tool_call_id": tool_call_id,
|
||||
"result": virtual_outcome,
|
||||
"name": tool_name,
|
||||
"display_name": tool_name,
|
||||
}
|
||||
)
|
||||
continue
|
||||
virtual_call_plan = virtual_outcome
|
||||
|
||||
mcp_server = global_mcp_server_manager.get_mcp_server_by_name(
|
||||
server_name
|
||||
) or global_mcp_server_manager._get_mcp_server_from_tool_name(tool_name)
|
||||
effective_tool_name = virtual_call_plan.tool_name if virtual_call_plan is not None else tool_name
|
||||
effective_arguments: dict[str, object] = ( # mutable-ok: call_tool takes dict[str, Any]
|
||||
dict(virtual_call_plan.arguments) # mutable-ok: same
|
||||
if virtual_call_plan is not None
|
||||
else parsed_arguments
|
||||
)
|
||||
|
||||
mapped_server_name = tool_server_map[tool_name] if virtual_call_plan is None else None
|
||||
mcp_server = (
|
||||
global_mcp_server_manager.get_mcp_server_by_name(mapped_server_name) if mapped_server_name else None
|
||||
) or global_mcp_server_manager._get_mcp_server_from_tool_name(effective_tool_name)
|
||||
server_name = mapped_server_name or (mcp_server.name if mcp_server is not None else None)
|
||||
resolved_tool_name = (
|
||||
_resolve_display_name_to_original(tool_name, [mcp_server]) if mcp_server else tool_name
|
||||
_resolve_display_name_to_original(effective_tool_name, [mcp_server])
|
||||
if mcp_server
|
||||
else effective_tool_name
|
||||
)
|
||||
sanitized_tool_name = strip_known_server_prefix(resolved_tool_name, mcp_server)
|
||||
|
||||
|
|
@ -745,7 +870,7 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
"role": "tool",
|
||||
"content": {
|
||||
"tool_name": sanitized_tool_name,
|
||||
"arguments": parsed_arguments,
|
||||
"arguments": effective_arguments,
|
||||
},
|
||||
}
|
||||
]
|
||||
|
|
@ -757,7 +882,7 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
"headers": logging_safe_headers,
|
||||
}
|
||||
logging_request_data = {
|
||||
"model": f"MCP: {tool_name}",
|
||||
"model": f"MCP: {effective_tool_name}",
|
||||
"metadata": logging_metadata,
|
||||
"input": logging_input,
|
||||
"call_type": CallTypes.call_mcp_tool.value,
|
||||
|
|
@ -769,7 +894,7 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
"headers": logging_safe_headers,
|
||||
"body": {
|
||||
"name": sanitized_tool_name,
|
||||
"arguments": parsed_arguments,
|
||||
"arguments": effective_arguments,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
|
@ -810,7 +935,7 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
litellm_logging_obj = None
|
||||
|
||||
logging_request_data["litellm_logging_obj"] = litellm_logging_obj
|
||||
logging_request_data["arguments"] = parsed_arguments
|
||||
logging_request_data["arguments"] = effective_arguments
|
||||
|
||||
if litellm_logging_obj:
|
||||
try:
|
||||
|
|
@ -823,8 +948,8 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
|
||||
standard_logging_mcp_tool_call: StandardLoggingMCPToolCall = {
|
||||
"name": sanitized_tool_name,
|
||||
"arguments": parsed_arguments,
|
||||
"namespaced_tool_name": tool_name,
|
||||
"arguments": effective_arguments,
|
||||
"namespaced_tool_name": effective_tool_name,
|
||||
}
|
||||
if mcp_server:
|
||||
mcp_info = mcp_server.mcp_info or {}
|
||||
|
|
@ -840,21 +965,37 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
|
||||
if litellm_logging_obj:
|
||||
litellm_logging_obj.model_call_details["mcp_tool_call_metadata"] = standard_logging_mcp_tool_call
|
||||
litellm_logging_obj.model = f"MCP: {tool_name}"
|
||||
litellm_logging_obj.model = f"MCP: {effective_tool_name}"
|
||||
litellm_logging_obj.call_type = CallTypes.call_mcp_tool.value
|
||||
|
||||
result = await global_mcp_server_manager.call_tool(
|
||||
server_name=server_name,
|
||||
name=sanitized_tool_name,
|
||||
arguments=parsed_arguments,
|
||||
user_api_key_auth=typed_user_api_key_auth,
|
||||
mcp_auth_header=mcp_auth_header,
|
||||
mcp_server_auth_headers=mcp_server_auth_headers,
|
||||
oauth2_headers=oauth2_headers,
|
||||
raw_headers=raw_headers,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
)
|
||||
if (
|
||||
virtual_call_plan is not None
|
||||
and virtual_tool_scope is not None
|
||||
and typed_user_api_key_auth is not None
|
||||
):
|
||||
result = await run_virtual_tool_call(
|
||||
plan=virtual_call_plan,
|
||||
scope=virtual_tool_scope,
|
||||
user_api_key_auth=typed_user_api_key_auth,
|
||||
mcp_auth_header=mcp_auth_header,
|
||||
mcp_server_auth_headers=mcp_server_auth_headers,
|
||||
oauth2_headers=oauth2_headers,
|
||||
raw_headers=raw_headers,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
)
|
||||
else:
|
||||
result = await global_mcp_server_manager.call_tool(
|
||||
server_name=tool_server_map[tool_name],
|
||||
name=sanitized_tool_name,
|
||||
arguments=effective_arguments,
|
||||
user_api_key_auth=typed_user_api_key_auth,
|
||||
mcp_auth_header=mcp_auth_header,
|
||||
mcp_server_auth_headers=mcp_server_auth_headers,
|
||||
oauth2_headers=oauth2_headers,
|
||||
raw_headers=raw_headers,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
)
|
||||
|
||||
if proxy_logging_obj:
|
||||
result = await proxy_logging_obj.post_mcp_call_hook(
|
||||
|
|
@ -894,6 +1035,7 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
"tool_call_id": tool_call_id,
|
||||
"result": result_text,
|
||||
"name": tool_name,
|
||||
"display_name": effective_tool_name,
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -1240,7 +1382,7 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
result_text = tool_result.get("result", "")
|
||||
|
||||
# Extract tool name and arguments from tool calls
|
||||
tool_name = "unknown"
|
||||
tool_name = tool_result.get("display_name") or "unknown"
|
||||
tool_arguments = "{}"
|
||||
for tool_call in tool_calls:
|
||||
(
|
||||
|
|
@ -1249,7 +1391,7 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
call_id,
|
||||
) = LiteLLM_Proxy_MCP_Handler._extract_tool_call_details(tool_call)
|
||||
if call_id == tool_call_id:
|
||||
tool_name = name or "unknown"
|
||||
tool_name = tool_result.get("display_name") or name or "unknown"
|
||||
tool_arguments = args or "{}"
|
||||
break
|
||||
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ if TYPE_CHECKING:
|
|||
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.responses.mcp.litellm_proxy_mcp_handler import MCPToolResult
|
||||
from litellm.responses.mcp.tool_search_bridge import VirtualToolScope
|
||||
else:
|
||||
MCPTool = Any
|
||||
|
||||
|
|
@ -272,6 +273,8 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator):
|
|||
self.user_api_key_auth = user_api_key_auth
|
||||
self.original_request_params = original_request_params or {}
|
||||
self.should_auto_execute = self._should_auto_execute_tools()
|
||||
self.virtual_tool_scope = self._build_virtual_tool_scope()
|
||||
self.max_tool_call_rounds = self._max_tool_call_rounds()
|
||||
|
||||
# Streaming state management
|
||||
self.phase = "initial_response" # initial_response -> mcp_discovery -> (continue_initial_response <-> tool_execution) -> finished
|
||||
|
|
@ -395,6 +398,22 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator):
|
|||
|
||||
return LiteLLM_Proxy_MCP_Handler._should_auto_execute_tools(self.mcp_tools_with_litellm_proxy)
|
||||
|
||||
def _build_virtual_tool_scope(self) -> "VirtualToolScope | None":
|
||||
"""Scope for the mcp_tool_search / mcp_tool_call virtual tools, or None when they are not in play."""
|
||||
from litellm.responses.mcp.litellm_proxy_mcp_handler import (
|
||||
LiteLLM_Proxy_MCP_Handler,
|
||||
)
|
||||
from litellm.responses.mcp.tool_search_bridge import is_mcp_tool_search_enabled
|
||||
|
||||
if not self.should_auto_execute or not is_mcp_tool_search_enabled(self.user_api_key_auth):
|
||||
return None
|
||||
return LiteLLM_Proxy_MCP_Handler._build_virtual_tool_scope(self.mcp_tools_with_litellm_proxy)
|
||||
|
||||
def _max_tool_call_rounds(self) -> int:
|
||||
from litellm.responses.mcp.tool_search_bridge import max_tool_call_rounds
|
||||
|
||||
return max_tool_call_rounds(self.virtual_tool_scope)
|
||||
|
||||
def _make_stream_error_event(self) -> ResponsesAPIStreamingResponse:
|
||||
err: Final = self._stream_error
|
||||
status_code: Final[object] = getattr(err, "status_code", None)
|
||||
|
|
@ -698,6 +717,7 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator):
|
|||
litellm_call_id=self.litellm_call_id,
|
||||
litellm_trace_id=self.litellm_trace_id,
|
||||
request_tags=LiteLLM_Proxy_MCP_Handler._get_parent_request_tags(self.original_request_params),
|
||||
virtual_tool_scope=self.virtual_tool_scope,
|
||||
)
|
||||
|
||||
# Create completion events and output_item.done events for tool execution
|
||||
|
|
@ -706,7 +726,7 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator):
|
|||
result_text = tool_result.get("result", "")
|
||||
|
||||
# Find matching tool name and arguments
|
||||
tool_name = "unknown"
|
||||
tool_name = tool_result.get("display_name") or "unknown"
|
||||
tool_arguments = "{}"
|
||||
for tool_call in tool_calls:
|
||||
(
|
||||
|
|
@ -715,7 +735,7 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator):
|
|||
call_id,
|
||||
) = LiteLLM_Proxy_MCP_Handler._extract_tool_call_details(tool_call)
|
||||
if call_id == tool_call_id:
|
||||
tool_name = name or "unknown"
|
||||
tool_name = tool_result.get("display_name") or name or "unknown"
|
||||
tool_arguments = args or "{}"
|
||||
break
|
||||
|
||||
|
|
@ -805,14 +825,14 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator):
|
|||
# Remove tool_choice to avoid forcing more tool calls
|
||||
follow_up_params.pop("tool_choice", None)
|
||||
|
||||
if self.tool_call_round >= MAX_MCP_TOOL_CALL_ROUNDS:
|
||||
if self.tool_call_round >= self.max_tool_call_rounds:
|
||||
# Hit the round cap: drop tools entirely so the model must
|
||||
# answer in text instead of emitting another (unexecuted)
|
||||
# tool call that would otherwise end the stream in silence.
|
||||
follow_up_params.pop("tools", None)
|
||||
verbose_logger.warning(
|
||||
"MCP auto-execute hit MAX_MCP_TOOL_CALL_ROUNDS=%s; forcing a text-only follow-up.",
|
||||
MAX_MCP_TOOL_CALL_ROUNDS,
|
||||
"MCP auto-execute hit its %s-round cap; forcing a text-only follow-up.",
|
||||
self.max_tool_call_rounds,
|
||||
)
|
||||
|
||||
follow_up_response: Final = await aresponses(**follow_up_params)
|
||||
|
|
|
|||
206
litellm/responses/mcp/tool_search_bridge.py
Normal file
206
litellm/responses/mcp/tool_search_bridge.py
Normal file
|
|
@ -0,0 +1,206 @@
|
|||
"""Responses-side adapter for the proxy's mcp_tool_search virtual tools.
|
||||
|
||||
``server.py`` owns the virtual tools; this module only supplies what the
|
||||
Responses path needs on top of them: the request's ``allowed_tools`` (a
|
||||
Responses-level concept ``execute_mcp_tool`` knows nothing about) and the inner
|
||||
tool name, so the caller can spend-log the real tool rather than the wrapper.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from pydantic import BaseModel, Field, TypeAdapter, ValidationError
|
||||
|
||||
from litellm.constants import MCP_VIRTUAL_TOOL_SEARCH_SERVER_NAME
|
||||
from litellm.proxy._experimental.mcp_server.utils import split_server_prefix_from_name
|
||||
from litellm.responses.mcp.mcp_streaming_iterator import MAX_MCP_TOOL_CALL_ROUNDS
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from mcp.types import CallToolResult
|
||||
from mcp.types import Tool as MCPTool
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
_SEARCH_RESULTS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...])
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class VirtualToolScope:
|
||||
"""Request-level scoping the virtual tools must reproduce, so a two-hop
|
||||
search + call sees exactly what a direct catalog listing would have."""
|
||||
|
||||
mcp_servers: tuple[str, ...]
|
||||
allowed_tools: frozenset[str]
|
||||
|
||||
@property
|
||||
def server_filter(self) -> list[str] | None: # mutable-ok: handle_mcp_tool_* takes Optional[list[str]]
|
||||
return list(self.mcp_servers) or None # mutable-ok: same
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class VirtualToolCallPlan:
|
||||
"""The real tool an ``mcp_tool_call`` unwraps to."""
|
||||
|
||||
tool_name: str
|
||||
arguments: Mapping[str, object]
|
||||
|
||||
|
||||
class _VirtualToolCallArguments(BaseModel):
|
||||
tool_name: str
|
||||
arguments: Mapping[str, object] = Field(default_factory=dict)
|
||||
|
||||
|
||||
def max_tool_call_rounds(scope: VirtualToolScope | None) -> int:
|
||||
"""Round budget for one auto-executed request. Doubled for the virtual
|
||||
tools, which spend two rounds per real tool use, so the flag does not halve
|
||||
a key's effective tool budget."""
|
||||
if scope is None:
|
||||
return MAX_MCP_TOOL_CALL_ROUNDS
|
||||
return MAX_MCP_TOOL_CALL_ROUNDS * 2
|
||||
|
||||
|
||||
def is_mcp_tool_search_enabled(user_api_key_auth: object) -> bool:
|
||||
object_permission: Final = getattr(user_api_key_auth, "object_permission", None)
|
||||
return bool(getattr(object_permission, "mcp_tool_search_enabled", False))
|
||||
|
||||
|
||||
def virtual_tool_definitions() -> tuple[MCPTool, ...]:
|
||||
"""The two virtual tools, as MCP ``Tool`` objects. Any other definition the
|
||||
``/mcp`` path may grow is excluded: this path only supports search + call."""
|
||||
from mcp.types import Tool
|
||||
|
||||
from litellm.proxy._experimental.mcp_server.tool_search import (
|
||||
MCP_TOOL_CALL_TOOL_NAME,
|
||||
MCP_TOOL_SEARCH_TOOL_NAME,
|
||||
get_virtual_tool_definitions,
|
||||
)
|
||||
|
||||
wanted: Final = (MCP_TOOL_SEARCH_TOOL_NAME, MCP_TOOL_CALL_TOOL_NAME)
|
||||
return tuple(
|
||||
Tool.model_validate(definition)
|
||||
for definition in get_virtual_tool_definitions()
|
||||
if definition.get("name") in wanted
|
||||
)
|
||||
|
||||
|
||||
def virtual_tool_server_map() -> Mapping[str, str]:
|
||||
"""``tool_server_map`` entries marking the two names as virtual, so a real
|
||||
upstream tool of the same name still dispatches normally."""
|
||||
return MappingProxyType({tool.name: MCP_VIRTUAL_TOOL_SEARCH_SERVER_NAME for tool in virtual_tool_definitions()})
|
||||
|
||||
|
||||
def is_tool_name_allowed(tool_name: str, allowed_tools: frozenset[str]) -> bool:
|
||||
if not allowed_tools or tool_name in allowed_tools:
|
||||
return True
|
||||
unprefixed_name, _ = split_server_prefix_from_name(tool_name)
|
||||
return unprefixed_name in allowed_tools
|
||||
|
||||
|
||||
def resolve_virtual_tool_call(
|
||||
arguments: Mapping[str, object],
|
||||
scope: VirtualToolScope,
|
||||
) -> VirtualToolCallPlan | str:
|
||||
"""The plan naming the real tool, or the rejection text to hand back as the tool result."""
|
||||
try:
|
||||
parsed: Final = _VirtualToolCallArguments.model_validate(arguments)
|
||||
except ValidationError:
|
||||
return "mcp_tool_call requires a 'tool_name' string argument"
|
||||
|
||||
if not parsed.tool_name:
|
||||
return "mcp_tool_call requires a non-empty 'tool_name'"
|
||||
|
||||
if not is_tool_name_allowed(parsed.tool_name, scope.allowed_tools):
|
||||
return f"Tool '{parsed.tool_name}' is not in allowed_tools for this request"
|
||||
|
||||
return VirtualToolCallPlan(tool_name=parsed.tool_name, arguments=parsed.arguments)
|
||||
|
||||
|
||||
async def run_virtual_tool_search(
|
||||
query: str,
|
||||
top_k: int,
|
||||
scope: VirtualToolScope,
|
||||
user_api_key_auth: UserAPIKeyAuth,
|
||||
mcp_auth_header: str | None = None,
|
||||
mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, # mutable-ok: handle_mcp_tool_* takes dict
|
||||
oauth2_headers: dict[str, str] | None = None, # mutable-ok: handle_mcp_tool_* takes dict
|
||||
raw_headers: dict[str, str] | None = None, # mutable-ok: handle_mcp_tool_* takes dict
|
||||
) -> CallToolResult:
|
||||
from litellm.proxy._experimental.mcp_server.tool_search import handle_mcp_tool_search
|
||||
|
||||
result: Final = await handle_mcp_tool_search(
|
||||
query=query,
|
||||
top_k=top_k,
|
||||
user_api_key_dict=user_api_key_auth,
|
||||
mcp_servers=scope.server_filter,
|
||||
mcp_auth_header=mcp_auth_header,
|
||||
mcp_server_auth_headers=mcp_server_auth_headers,
|
||||
oauth2_headers=oauth2_headers,
|
||||
raw_headers=raw_headers,
|
||||
)
|
||||
return _apply_allowed_tools_to_search_result(result, scope.allowed_tools)
|
||||
|
||||
|
||||
async def run_virtual_tool_call(
|
||||
plan: VirtualToolCallPlan,
|
||||
scope: VirtualToolScope,
|
||||
user_api_key_auth: UserAPIKeyAuth,
|
||||
mcp_auth_header: str | None = None,
|
||||
mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, # mutable-ok: handle_mcp_tool_* takes dict
|
||||
oauth2_headers: dict[str, str] | None = None, # mutable-ok: handle_mcp_tool_* takes dict
|
||||
raw_headers: dict[str, str] | None = None, # mutable-ok: handle_mcp_tool_* takes dict
|
||||
litellm_logging_obj: LiteLLMLoggingObj | None = None,
|
||||
) -> CallToolResult:
|
||||
from litellm.proxy._experimental.mcp_server.tool_search import handle_mcp_tool_call
|
||||
|
||||
return await handle_mcp_tool_call(
|
||||
tool_name=plan.tool_name,
|
||||
arguments=dict(plan.arguments), # mutable-ok: handle_mcp_tool_call takes dict[str, Any]
|
||||
user_api_key_dict=user_api_key_auth,
|
||||
mcp_servers=scope.server_filter,
|
||||
mcp_auth_header=mcp_auth_header,
|
||||
mcp_server_auth_headers=mcp_server_auth_headers,
|
||||
oauth2_headers=oauth2_headers,
|
||||
raw_headers=raw_headers,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
)
|
||||
|
||||
|
||||
def _apply_allowed_tools_to_search_result(
|
||||
result: CallToolResult,
|
||||
allowed_tools: frozenset[str],
|
||||
) -> CallToolResult:
|
||||
if not allowed_tools or result.isError:
|
||||
return result
|
||||
|
||||
from mcp.types import CallToolResult, TextContent
|
||||
|
||||
return CallToolResult(
|
||||
content=[ # mutable-ok: CallToolResult.content is a list field
|
||||
TextContent(type="text", text=_filtered_search_text(item.text, allowed_tools))
|
||||
if isinstance(item, TextContent)
|
||||
else item
|
||||
for item in result.content
|
||||
],
|
||||
isError=result.isError,
|
||||
)
|
||||
|
||||
|
||||
def _filtered_search_text(text: str, allowed_tools: frozenset[str]) -> str:
|
||||
try:
|
||||
entries: Final[Sequence[Mapping[str, object]]] = _SEARCH_RESULTS_ADAPTER.validate_json(text)
|
||||
except ValidationError:
|
||||
return text
|
||||
|
||||
return json.dumps(
|
||||
tuple(
|
||||
entry
|
||||
for entry in entries
|
||||
if isinstance(name := entry.get("name"), str) and is_tool_name_allowed(name, allowed_tools)
|
||||
)
|
||||
)
|
||||
|
|
@ -1387,3 +1387,178 @@ async def test_acompletion_with_mcp_forwards_unserved_external_mcp_tool_to_the_p
|
|||
assert isinstance(result, ModelResponse)
|
||||
assert result.id == "chatcmpl-zapier"
|
||||
assert json.loads(provider.calls.last.request.content)["tools"] == [zapier_tool]
|
||||
# ---------------------------------------------------------------------------
|
||||
# mcp_tool_search virtual tools on the /chat/completions MCP bridge
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
VIRTUAL_TOOL_MCP_CONFIG = {
|
||||
"type": "mcp",
|
||||
"server_label": "x",
|
||||
"server_url": "litellm_proxy/mcp/deepwiki",
|
||||
"require_approval": "never",
|
||||
}
|
||||
|
||||
|
||||
def _tool_search_auth():
|
||||
from litellm.models.object_permission import LiteLLM_ObjectPermissionTable
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
return UserAPIKeyAuth(
|
||||
api_key="k",
|
||||
user_id="u",
|
||||
object_permission=LiteLLM_ObjectPermissionTable(
|
||||
object_permission_id="test", mcp_tool_search_enabled=True
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _chat_tool_call_response(call_id: str, name: str, arguments: str) -> ModelResponse:
|
||||
return ModelResponse(
|
||||
id=call_id,
|
||||
model="gpt-5.5",
|
||||
created=0,
|
||||
object="chat.completion",
|
||||
choices=[
|
||||
{
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": call_id,
|
||||
"type": "function",
|
||||
"function": {"name": name, "arguments": arguments},
|
||||
}
|
||||
],
|
||||
},
|
||||
"finish_reason": "tool_calls",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _chat_text_response(text: str) -> ModelResponse:
|
||||
return ModelResponse(
|
||||
id="final",
|
||||
model="gpt-5.5",
|
||||
created=0,
|
||||
object="chat.completion",
|
||||
choices=[{"index": 0, "message": {"role": "assistant", "content": text}, "finish_reason": "stop"}],
|
||||
)
|
||||
|
||||
|
||||
def _patch_chat_mcp_globals(monkeypatch):
|
||||
import sys
|
||||
import types as _types
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from mcp.types import Tool
|
||||
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"litellm.proxy.proxy_server",
|
||||
_types.SimpleNamespace(proxy_logging_obj=MagicMock()),
|
||||
)
|
||||
from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing
|
||||
|
||||
mock_get_tools = AsyncMock(
|
||||
return_value=AggregateToolListing(
|
||||
tools=[Tool(name="deepwiki-read_wiki_contents", description="d", inputSchema={"type": "object"})],
|
||||
outcomes={},
|
||||
)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy._experimental.mcp_server.server._get_tools_from_mcp_servers",
|
||||
mock_get_tools,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager",
|
||||
_types.SimpleNamespace(
|
||||
get_registry=MagicMock(return_value={}),
|
||||
call_tool=AsyncMock(),
|
||||
_get_mcp_server_from_tool_name=MagicMock(return_value=None),
|
||||
get_mcp_server_by_name=MagicMock(return_value=None),
|
||||
get_allowed_mcp_servers=AsyncMock(return_value=[]),
|
||||
get_mcp_servers_from_ids=MagicMock(return_value=[]),
|
||||
),
|
||||
)
|
||||
return mock_get_tools
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_acompletion_with_mcp_virtual_tools_search_then_call(monkeypatch):
|
||||
"""Non-streaming /chat/completions must complete the two-hop virtual tool
|
||||
flow in a single request and never list the catalog."""
|
||||
import litellm
|
||||
from mcp.types import CallToolResult, TextContent
|
||||
|
||||
mock_get_tools = _patch_chat_mcp_globals(monkeypatch)
|
||||
|
||||
search_mock = AsyncMock(
|
||||
return_value=CallToolResult(
|
||||
content=[TextContent(type="text", text='[{"name": "deepwiki-read_wiki_contents"}]')], isError=False
|
||||
)
|
||||
)
|
||||
call_mock = AsyncMock(
|
||||
return_value=CallToolResult(content=[TextContent(type="text", text="wiki body")], isError=False)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_search", search_mock
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_call", call_mock
|
||||
)
|
||||
|
||||
mock_acompletion = AsyncMock(
|
||||
side_effect=[
|
||||
_chat_tool_call_response("call-1", "mcp_tool_search", '{"query": "wiki"}'),
|
||||
_chat_tool_call_response(
|
||||
"call-2",
|
||||
"mcp_tool_call",
|
||||
'{"tool_name": "deepwiki-read_wiki_contents", "arguments": {"repo": "x"}}',
|
||||
),
|
||||
_chat_text_response("The wiki says hello."),
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(litellm, "acompletion", mock_acompletion)
|
||||
|
||||
response = await acompletion_with_mcp(
|
||||
model="gpt-5.5",
|
||||
messages=[{"role": "user", "content": "describe the wiki"}],
|
||||
tools=[VIRTUAL_TOOL_MCP_CONFIG],
|
||||
user_api_key_auth=_tool_search_auth(),
|
||||
)
|
||||
|
||||
assert mock_get_tools.await_count == 0
|
||||
assert mock_acompletion.await_count == 3
|
||||
assert search_mock.await_count == 1
|
||||
assert call_mock.await_count == 1
|
||||
assert call_mock.await_args.kwargs["tool_name"] == "deepwiki-read_wiki_contents"
|
||||
|
||||
injected_tools = mock_acompletion.await_args_list[0].kwargs["tools"]
|
||||
assert [tool["function"]["name"] for tool in injected_tools] == ["mcp_tool_search", "mcp_tool_call"]
|
||||
assert response.choices[0].message.content == "The wiki says hello."
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_acompletion_with_mcp_streaming_keeps_catalog_when_tool_search_enabled(monkeypatch):
|
||||
"""The streaming chat bridge only does one tool-call round, so the virtual
|
||||
tools stay off there and the full catalog is injected as before."""
|
||||
import litellm
|
||||
|
||||
mock_get_tools = _patch_chat_mcp_globals(monkeypatch)
|
||||
mock_acompletion = AsyncMock(return_value=_chat_text_response("hi"))
|
||||
monkeypatch.setattr(litellm, "acompletion", mock_acompletion)
|
||||
|
||||
await acompletion_with_mcp(
|
||||
model="gpt-5.5",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=[VIRTUAL_TOOL_MCP_CONFIG],
|
||||
user_api_key_auth=_tool_search_auth(),
|
||||
stream=True,
|
||||
)
|
||||
|
||||
assert mock_get_tools.await_count == 1
|
||||
injected_tools = mock_acompletion.await_args_list[0].kwargs["tools"]
|
||||
assert [tool["function"]["name"] for tool in injected_tools] == ["deepwiki-read_wiki_contents"]
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import textwrap
|
||||
|
|
@ -15,6 +16,8 @@ from litellm.responses.mcp import litellm_proxy_mcp_handler as mcp_handler_modul
|
|||
from litellm.responses.mcp.litellm_proxy_mcp_handler import (
|
||||
LiteLLM_Proxy_MCP_Handler,
|
||||
)
|
||||
from litellm.constants import MCP_VIRTUAL_TOOL_SEARCH_SERVER_NAME
|
||||
from litellm.responses.mcp.tool_search_bridge import virtual_tool_server_map
|
||||
from typing import Any, cast
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
|
@ -1102,3 +1105,292 @@ async def test_mcp_follow_up_call_is_stateless_when_store_is_false(
|
|||
item for item in follow_up_call["input"] if isinstance(item, dict) and item.get("type") == "reasoning"
|
||||
]
|
||||
assert bool(reasoning_items) is (store is False)
|
||||
# mcp_tool_search virtual tools on the Responses API type: "mcp" path
|
||||
|
||||
|
||||
def _mcp_tool_config(server: str = "deepwiki", allowed_tools=None):
|
||||
config = {
|
||||
"type": "mcp",
|
||||
"server_label": "x",
|
||||
"server_url": f"litellm_proxy/mcp/{server}",
|
||||
"require_approval": "never",
|
||||
}
|
||||
if allowed_tools is not None:
|
||||
config["allowed_tools"] = allowed_tools
|
||||
return config
|
||||
|
||||
|
||||
def _auth(tool_search_enabled=None):
|
||||
"""Real UserAPIKeyAuth; tool_search_enabled=None means no object_permission at all."""
|
||||
from litellm.models.object_permission import LiteLLM_ObjectPermissionTable
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
if tool_search_enabled is None:
|
||||
return UserAPIKeyAuth(api_key="k", user_id="u")
|
||||
return UserAPIKeyAuth(
|
||||
api_key="k",
|
||||
user_id="u",
|
||||
object_permission=LiteLLM_ObjectPermissionTable(
|
||||
object_permission_id="test",
|
||||
mcp_tool_search_enabled=tool_search_enabled,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _setup_async_proxy_logging(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""proxy_logging_obj with awaitable hooks, needed once a tool actually dispatches."""
|
||||
proxy_logging_obj = MagicMock()
|
||||
proxy_logging_obj.post_call_failure_hook = AsyncMock()
|
||||
proxy_logging_obj.post_mcp_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["response"])
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"litellm.proxy.proxy_server",
|
||||
types.SimpleNamespace(proxy_logging_obj=proxy_logging_obj),
|
||||
)
|
||||
|
||||
|
||||
def _text_result(text: str, is_error: bool = False):
|
||||
from mcp.types import CallToolResult, TextContent
|
||||
|
||||
return CallToolResult(content=[TextContent(type="text", text=text)], isError=is_error)
|
||||
|
||||
|
||||
def _patch_catalog_listing(monkeypatch):
|
||||
"""Patch the manager listing seam and return the mock, so a test can assert it was never used."""
|
||||
from mcp.types import Tool
|
||||
|
||||
mock_get_tools = AsyncMock(
|
||||
return_value=AggregateToolListing(
|
||||
tools=[
|
||||
Tool(
|
||||
name="deepwiki-read_wiki_structure",
|
||||
description="read the wiki structure",
|
||||
inputSchema={"type": "object", "properties": {}},
|
||||
)
|
||||
],
|
||||
outcomes={},
|
||||
)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy._experimental.mcp_server.server._get_tools_from_mcp_servers",
|
||||
mock_get_tools,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager",
|
||||
types.SimpleNamespace(
|
||||
get_registry=MagicMock(return_value={}),
|
||||
get_allowed_mcp_servers=AsyncMock(return_value=[]),
|
||||
get_mcp_servers_from_ids=MagicMock(return_value=[]),
|
||||
get_mcp_server_by_name=MagicMock(return_value=None),
|
||||
),
|
||||
)
|
||||
return mock_get_tools
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_process_mcp_tools_returns_only_virtual_tools_when_flag_enabled(monkeypatch):
|
||||
"""With mcp_tool_search_enabled the catalog must never be listed; the model
|
||||
sees only mcp_tool_search + mcp_tool_call."""
|
||||
mock_get_tools = _patch_catalog_listing(monkeypatch)
|
||||
|
||||
openai_tools, tool_server_map = await LiteLLM_Proxy_MCP_Handler._process_mcp_tools_to_openai_format(
|
||||
user_api_key_auth=_auth(True),
|
||||
mcp_tools_with_litellm_proxy=[_mcp_tool_config()],
|
||||
)
|
||||
|
||||
assert [tool["name"] for tool in openai_tools] == ["mcp_tool_search", "mcp_tool_call"]
|
||||
assert all(tool["type"] == "function" for tool in openai_tools)
|
||||
assert tool_server_map == {
|
||||
"mcp_tool_search": MCP_VIRTUAL_TOOL_SEARCH_SERVER_NAME,
|
||||
"mcp_tool_call": MCP_VIRTUAL_TOOL_SEARCH_SERVER_NAME,
|
||||
}
|
||||
assert mock_get_tools.await_count == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"tool_search_enabled,require_approval",
|
||||
[
|
||||
(False, "never"),
|
||||
(None, "never"),
|
||||
(True, None),
|
||||
],
|
||||
ids=["flag_false", "no_object_permission", "flag_on_but_approval_required"],
|
||||
)
|
||||
async def test_process_mcp_tools_injects_catalog_unchanged(monkeypatch, tool_search_enabled, require_approval):
|
||||
"""Regression: the catalog path is untouched for keys without the flag, and
|
||||
the virtual tools are skipped when they could not be auto-executed."""
|
||||
mock_get_tools = _patch_catalog_listing(monkeypatch)
|
||||
tool_config = _mcp_tool_config()
|
||||
if require_approval is None:
|
||||
tool_config.pop("require_approval")
|
||||
|
||||
openai_tools, tool_server_map = await LiteLLM_Proxy_MCP_Handler._process_mcp_tools_to_openai_format(
|
||||
user_api_key_auth=_auth(tool_search_enabled),
|
||||
mcp_tools_with_litellm_proxy=[tool_config],
|
||||
)
|
||||
|
||||
assert [tool["name"] for tool in openai_tools] == ["deepwiki-read_wiki_structure"]
|
||||
assert tool_server_map == {"deepwiki-read_wiki_structure": "deepwiki"}
|
||||
assert mock_get_tools.await_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"allowed_tools,expected_names",
|
||||
[
|
||||
(["read_wiki_structure"], ["deepwiki-read_wiki_structure"]),
|
||||
(["something_else"], []),
|
||||
],
|
||||
ids=["keeps_allowed", "filters_everything_out"],
|
||||
)
|
||||
async def test_execute_tool_calls_scopes_mcp_tool_search(monkeypatch, allowed_tools, expected_names):
|
||||
"""mcp_tool_search is scoped to the request's server aliases, and its results
|
||||
are filtered to allowed_tools; filtering everything out is an empty list, not an error."""
|
||||
_setup_proxy_logging(monkeypatch)
|
||||
_setup_mcp_call_environment(monkeypatch)
|
||||
|
||||
search_mock = AsyncMock(
|
||||
return_value=_text_result(
|
||||
json.dumps(
|
||||
[
|
||||
{"name": "deepwiki-read_wiki_structure", "description": "d", "inputSchema": {}},
|
||||
{"name": "deepwiki-ask_question", "description": "d", "inputSchema": {}},
|
||||
]
|
||||
)
|
||||
)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_search",
|
||||
search_mock,
|
||||
)
|
||||
|
||||
results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls(
|
||||
tool_server_map=virtual_tool_server_map(),
|
||||
tool_calls=[
|
||||
{
|
||||
"id": "call-search",
|
||||
"function": {"name": "mcp_tool_search", "arguments": json.dumps({"query": "wiki", "top_k": 3})},
|
||||
}
|
||||
],
|
||||
user_api_key_auth=_auth(True),
|
||||
virtual_tool_scope=LiteLLM_Proxy_MCP_Handler._build_virtual_tool_scope(
|
||||
[_mcp_tool_config(allowed_tools=allowed_tools)]
|
||||
),
|
||||
)
|
||||
|
||||
assert search_mock.await_args.kwargs["mcp_servers"] == ["deepwiki"]
|
||||
assert search_mock.await_args.kwargs["query"] == "wiki"
|
||||
assert search_mock.await_args.kwargs["top_k"] == 3
|
||||
assert [entry["name"] for entry in json.loads(results[0]["result"])] == expected_names
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_execute_tool_calls_mcp_tool_call_dispatches_inner_tool_with_auth_headers(monkeypatch):
|
||||
"""mcp_tool_call unwraps to the real tool, forwards the per-request auth
|
||||
headers and spend-logs the inner tool name rather than mcp_tool_call."""
|
||||
_setup_mcp_call_environment(monkeypatch)
|
||||
_setup_async_proxy_logging(monkeypatch)
|
||||
|
||||
call_mock = AsyncMock(return_value=_text_result("wiki structure"))
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_call",
|
||||
call_mock,
|
||||
)
|
||||
|
||||
captured = {}
|
||||
|
||||
def fake_function_setup(*_args, **kwargs):
|
||||
captured.update(kwargs)
|
||||
return None, None
|
||||
|
||||
handler_module = importlib.import_module("litellm.responses.mcp.litellm_proxy_mcp_handler")
|
||||
monkeypatch.setattr(handler_module, "function_setup", fake_function_setup)
|
||||
|
||||
results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls(
|
||||
tool_server_map=virtual_tool_server_map(),
|
||||
tool_calls=[
|
||||
{
|
||||
"id": "call-inner",
|
||||
"function": {
|
||||
"name": "mcp_tool_call",
|
||||
"arguments": json.dumps(
|
||||
{"tool_name": "deepwiki-read_wiki_structure", "arguments": {"repo": "litellm"}}
|
||||
),
|
||||
},
|
||||
}
|
||||
],
|
||||
user_api_key_auth=_auth(True),
|
||||
mcp_auth_header="legacy-token",
|
||||
mcp_server_auth_headers={"deepwiki": {"authorization": "Bearer per-server"}},
|
||||
oauth2_headers={"x-oauth": "1"},
|
||||
raw_headers={"x-raw": "1"},
|
||||
virtual_tool_scope=LiteLLM_Proxy_MCP_Handler._build_virtual_tool_scope([_mcp_tool_config()]),
|
||||
)
|
||||
|
||||
call_kwargs = call_mock.await_args.kwargs
|
||||
assert call_kwargs["tool_name"] == "deepwiki-read_wiki_structure"
|
||||
assert call_kwargs["arguments"] == {"repo": "litellm"}
|
||||
assert call_kwargs["mcp_servers"] == ["deepwiki"]
|
||||
assert call_kwargs["mcp_auth_header"] == "legacy-token"
|
||||
assert call_kwargs["mcp_server_auth_headers"] == {"deepwiki": {"authorization": "Bearer per-server"}}
|
||||
assert call_kwargs["oauth2_headers"] == {"x-oauth": "1"}
|
||||
assert call_kwargs["raw_headers"] == {"x-raw": "1"}
|
||||
|
||||
assert captured["model"] == "MCP: deepwiki-read_wiki_structure"
|
||||
assert captured["metadata"]["tool_name"] == "read_wiki_structure"
|
||||
assert captured["input"][0]["content"]["arguments"] == {"repo": "litellm"}
|
||||
assert results[0]["result"] == "wiki structure"
|
||||
assert results[0]["display_name"] == "deepwiki-read_wiki_structure"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_execute_tool_calls_mcp_tool_call_rejects_tool_outside_allowed_tools(monkeypatch):
|
||||
_setup_proxy_logging(monkeypatch)
|
||||
_setup_mcp_call_environment(monkeypatch)
|
||||
call_mock = AsyncMock(return_value=_text_result("should not run"))
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_call",
|
||||
call_mock,
|
||||
)
|
||||
|
||||
results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls(
|
||||
tool_server_map=virtual_tool_server_map(),
|
||||
tool_calls=[
|
||||
{
|
||||
"id": "call-inner",
|
||||
"function": {
|
||||
"name": "mcp_tool_call",
|
||||
"arguments": json.dumps({"tool_name": "deepwiki-delete_everything", "arguments": {}}),
|
||||
},
|
||||
}
|
||||
],
|
||||
user_api_key_auth=_auth(True),
|
||||
virtual_tool_scope=LiteLLM_Proxy_MCP_Handler._build_virtual_tool_scope(
|
||||
[_mcp_tool_config(allowed_tools=["read_wiki_structure"])]
|
||||
),
|
||||
)
|
||||
|
||||
assert call_mock.await_count == 0
|
||||
assert "not in allowed_tools" in results[0]["result"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_execute_tool_calls_ignores_virtual_names_without_scope(monkeypatch):
|
||||
"""A real upstream tool literally named mcp_tool_search must still dispatch
|
||||
normally when the virtual tools are not in play."""
|
||||
call_tool_mock = _setup_mcp_call_environment(monkeypatch)
|
||||
search_mock = AsyncMock(return_value=_text_result("[]"))
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_search",
|
||||
search_mock,
|
||||
)
|
||||
|
||||
await LiteLLM_Proxy_MCP_Handler._execute_tool_calls(
|
||||
tool_server_map={"mcp_tool_search": "deepwiki"},
|
||||
tool_calls=[{"id": "c", "function": {"name": "mcp_tool_search", "arguments": "{}"}}],
|
||||
user_api_key_auth=None,
|
||||
)
|
||||
|
||||
assert search_mock.await_count == 0
|
||||
assert call_tool_mock.await_count == 1
|
||||
|
|
|
|||
|
|
@ -336,3 +336,100 @@ async def test_streaming_follow_up_keeps_previous_response_id_when_stored(monkey
|
|||
follow_up_kwargs = aresponses_mock.call_args_list[0].kwargs
|
||||
assert follow_up_kwargs["previous_response_id"] == "resp_prev"
|
||||
assert not [item for item in follow_up_kwargs["input"] if item.get("type") == "reasoning"]
|
||||
def _virtual_tool_iterator(initial_chunks) -> MCPEnhancedStreamingIterator:
|
||||
from litellm.models.object_permission import LiteLLM_ObjectPermissionTable
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.responses.mcp.tool_search_bridge import virtual_tool_server_map
|
||||
|
||||
return MCPEnhancedStreamingIterator(
|
||||
base_iterator=_FakeAsyncStream(initial_chunks),
|
||||
mcp_events=[],
|
||||
tool_server_map=virtual_tool_server_map(),
|
||||
mcp_tools_with_litellm_proxy=[
|
||||
{
|
||||
"type": "mcp",
|
||||
"server_url": "litellm_proxy/mcp/deepwiki",
|
||||
"require_approval": "never",
|
||||
}
|
||||
],
|
||||
user_api_key_auth=UserAPIKeyAuth(
|
||||
api_key="k",
|
||||
user_id="u",
|
||||
object_permission=LiteLLM_ObjectPermissionTable(
|
||||
object_permission_id="test", mcp_tool_search_enabled=True
|
||||
),
|
||||
),
|
||||
original_request_params={
|
||||
"model": "gpt-5.5",
|
||||
"input": "what is berriai/litellm?",
|
||||
"tools": [{"type": "mcp"}],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_virtual_tool_search_then_call_shows_inner_tool_name(monkeypatch):
|
||||
"""With mcp_tool_search_enabled the stream must run search -> call -> text,
|
||||
dispatch through the virtual tool handlers (never the manager directly) and
|
||||
label the mcp_call item with the real tool the model reached."""
|
||||
call_tool = _mock_mcp_environment(monkeypatch)
|
||||
proxy_logging_obj = sys.modules["litellm.proxy.proxy_server"].proxy_logging_obj
|
||||
proxy_logging_obj.post_mcp_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["response"])
|
||||
|
||||
search_mock = AsyncMock(
|
||||
return_value=CallToolResult(
|
||||
content=[TextContent(type="text", text='[{"name": "deepwiki-read_wiki_contents"}]')], isError=False
|
||||
)
|
||||
)
|
||||
call_mock = AsyncMock(
|
||||
return_value=CallToolResult(content=[TextContent(type="text", text="wiki body")], isError=False)
|
||||
)
|
||||
monkeypatch.setattr("litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_search", search_mock)
|
||||
monkeypatch.setattr("litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_call", call_mock)
|
||||
|
||||
aresponses_mock = AsyncMock(
|
||||
side_effect=[
|
||||
_FakeAsyncStream(
|
||||
[
|
||||
_completed_chunk(
|
||||
[
|
||||
_function_call(
|
||||
"call_2",
|
||||
"mcp_tool_call",
|
||||
'{"tool_name": "deepwiki-read_wiki_contents", "arguments": {"repo": "x"}}',
|
||||
)
|
||||
]
|
||||
)
|
||||
]
|
||||
),
|
||||
_text_only_stream("The wiki says hello."),
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(responses_main_module, "aresponses", aresponses_mock)
|
||||
|
||||
iterator = _virtual_tool_iterator(
|
||||
[
|
||||
_output_item_added_chunk(),
|
||||
_completed_chunk([_function_call("call_1", "mcp_tool_search", '{"query": "wiki"}')]),
|
||||
]
|
||||
)
|
||||
|
||||
chunks = [chunk async for chunk in iterator]
|
||||
|
||||
assert iterator.max_tool_call_rounds == MAX_MCP_TOOL_CALL_ROUNDS * 2
|
||||
assert search_mock.await_count == 1
|
||||
assert search_mock.await_args.kwargs["mcp_servers"] == ["deepwiki"]
|
||||
assert call_mock.await_count == 1
|
||||
assert call_mock.await_args.kwargs["tool_name"] == "deepwiki-read_wiki_contents"
|
||||
assert call_tool.call_count == 0
|
||||
|
||||
mcp_call_names = [
|
||||
chunk.item.name
|
||||
for chunk in chunks
|
||||
if getattr(chunk, "type", None) == ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE
|
||||
and getattr(getattr(chunk, "item", None), "type", None) == "mcp_call"
|
||||
]
|
||||
assert mcp_call_names == ["mcp_tool_search", "deepwiki-read_wiki_contents"]
|
||||
|
||||
completed_chunks = [c for c in chunks if getattr(c, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED]
|
||||
assert completed_chunks[-1].response.output[0]["content"][0]["text"] == "The wiki says hello."
|
||||
|
|
|
|||
232
tests/test_litellm/responses/test_main.py
Normal file
232
tests/test_litellm/responses/test_main.py
Normal file
|
|
@ -0,0 +1,232 @@
|
|||
"""Tests for litellm/responses/main.py MCP auto-execution."""
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import types
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.models.object_permission import LiteLLM_ObjectPermissionTable
|
||||
from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.responses.main import aresponses_api_with_mcp
|
||||
from litellm.responses.mcp.mcp_streaming_iterator import MAX_MCP_TOOL_CALL_ROUNDS
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
|
||||
MCP_TOOL_CONFIG = {
|
||||
"type": "mcp",
|
||||
"server_label": "x",
|
||||
"server_url": "litellm_proxy/mcp/deepwiki",
|
||||
"require_approval": "never",
|
||||
}
|
||||
|
||||
|
||||
def _auth(tool_search_enabled: bool) -> UserAPIKeyAuth:
|
||||
return UserAPIKeyAuth(
|
||||
api_key="k",
|
||||
user_id="u",
|
||||
object_permission=LiteLLM_ObjectPermissionTable(
|
||||
object_permission_id="test",
|
||||
mcp_tool_search_enabled=tool_search_enabled,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _function_call_response(response_id: str, call_id: str, name: str, arguments: dict) -> ResponsesAPIResponse:
|
||||
return ResponsesAPIResponse(
|
||||
id=response_id,
|
||||
created_at=0,
|
||||
model="gpt-5.5",
|
||||
object="response",
|
||||
status="completed",
|
||||
output=[
|
||||
{
|
||||
"type": "function_call",
|
||||
"id": call_id,
|
||||
"call_id": call_id,
|
||||
"name": name,
|
||||
"arguments": json.dumps(arguments),
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _text_response(response_id: str, text: str) -> ResponsesAPIResponse:
|
||||
return ResponsesAPIResponse(
|
||||
id=response_id,
|
||||
created_at=0,
|
||||
model="gpt-5.5",
|
||||
object="response",
|
||||
status="completed",
|
||||
output=[
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg-1",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{"type": "output_text", "text": text, "annotations": []}],
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _text_result(text: str):
|
||||
from mcp.types import CallToolResult, TextContent
|
||||
|
||||
return CallToolResult(content=[TextContent(type="text", text=text)], isError=False)
|
||||
|
||||
|
||||
def _patch_aresponses(monkeypatch, responses):
|
||||
"""Patch both references to aresponses (main.py's own and the handler's follow-up import)."""
|
||||
mock = AsyncMock(side_effect=list(responses))
|
||||
main_module = importlib.import_module("litellm.responses.main")
|
||||
handler_module = importlib.import_module("litellm.responses.mcp.litellm_proxy_mcp_handler")
|
||||
monkeypatch.setattr(main_module, "aresponses", mock)
|
||||
monkeypatch.setattr(handler_module, "aresponses", mock)
|
||||
return mock
|
||||
|
||||
|
||||
def _patch_proxy_globals(monkeypatch):
|
||||
import sys
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
proxy_logging_obj = MagicMock()
|
||||
proxy_logging_obj.post_call_failure_hook = AsyncMock()
|
||||
proxy_logging_obj.post_mcp_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["response"])
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"litellm.proxy.proxy_server",
|
||||
types.SimpleNamespace(proxy_logging_obj=proxy_logging_obj),
|
||||
)
|
||||
fake_manager = types.SimpleNamespace(
|
||||
get_registry=MagicMock(return_value={}),
|
||||
call_tool=AsyncMock(),
|
||||
_get_mcp_server_from_tool_name=MagicMock(return_value=None),
|
||||
get_mcp_server_by_name=MagicMock(return_value=None),
|
||||
get_allowed_mcp_servers=AsyncMock(return_value=[]),
|
||||
get_mcp_servers_from_ids=MagicMock(return_value=[]),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager",
|
||||
fake_manager,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_then_call_then_answer_completes_in_one_request(monkeypatch):
|
||||
"""The two-hop virtual tool flow (search -> call -> text) must finish inside a
|
||||
single client request; a single-round implementation would return the
|
||||
unexecuted mcp_tool_call instead."""
|
||||
_patch_proxy_globals(monkeypatch)
|
||||
|
||||
search_mock = AsyncMock(
|
||||
return_value=_text_result(
|
||||
json.dumps([{"name": "deepwiki-read_wiki_structure", "description": "d", "inputSchema": {}}])
|
||||
)
|
||||
)
|
||||
call_mock = AsyncMock(return_value=_text_result("wiki structure"))
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_search", search_mock
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_call", call_mock
|
||||
)
|
||||
|
||||
aresponses_mock = _patch_aresponses(
|
||||
monkeypatch,
|
||||
[
|
||||
_function_call_response("resp-1", "call-1", "mcp_tool_search", {"query": "wiki"}),
|
||||
_function_call_response(
|
||||
"resp-2",
|
||||
"call-2",
|
||||
"mcp_tool_call",
|
||||
{"tool_name": "deepwiki-read_wiki_structure", "arguments": {"repo": "litellm"}},
|
||||
),
|
||||
_text_response("resp-3", "The wiki has 3 sections."),
|
||||
],
|
||||
)
|
||||
|
||||
final = await aresponses_api_with_mcp(
|
||||
input="describe the wiki",
|
||||
model="gpt-5.5",
|
||||
tools=[MCP_TOOL_CONFIG],
|
||||
litellm_metadata={"user_api_key_auth": _auth(True)},
|
||||
user_api_key_auth=_auth(True),
|
||||
)
|
||||
|
||||
assert aresponses_mock.await_count == 3
|
||||
assert search_mock.await_count == 1
|
||||
assert call_mock.await_count == 1
|
||||
assert call_mock.await_args.kwargs["tool_name"] == "deepwiki-read_wiki_structure"
|
||||
|
||||
# Only the two virtual tools ever reach the model.
|
||||
first_call_tools = aresponses_mock.await_args_list[0].kwargs["tools"]
|
||||
assert [tool["name"] for tool in first_call_tools] == ["mcp_tool_search", "mcp_tool_call"]
|
||||
|
||||
assert isinstance(final, ResponsesAPIResponse)
|
||||
output_types = [item.get("type") if isinstance(item, dict) else item.type for item in final.output]
|
||||
assert "tool_execution_results" in output_types
|
||||
tool_results_item = next(
|
||||
item for item in final.output if isinstance(item, dict) and item.get("type") == "tool_execution_results"
|
||||
)
|
||||
logged_results = json.loads(tool_results_item["content"][0]["text"])
|
||||
assert [entry["name"] for entry in logged_results] == ["mcp_tool_search", "mcp_tool_call"]
|
||||
assert [entry["display_name"] for entry in logged_results] == [
|
||||
"mcp_tool_search",
|
||||
"deepwiki-read_wiki_structure",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("tool_search_enabled", [True, False], ids=["virtual_tools", "plain_catalog"])
|
||||
async def test_auto_execute_round_cap(monkeypatch, tool_search_enabled):
|
||||
"""A model that keeps calling tools is capped and the final follow-up drops
|
||||
tools so it has to answer in text. The virtual tools spend two rounds per
|
||||
real tool use, so the flagged path gets twice the budget."""
|
||||
from mcp.types import Tool
|
||||
|
||||
_patch_proxy_globals(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy._experimental.mcp_server.server._get_tools_from_mcp_servers",
|
||||
AsyncMock(
|
||||
return_value=AggregateToolListing(
|
||||
tools=[Tool(name="deepwiki-read_wiki_contents", description="d", inputSchema={})], outcomes={}
|
||||
)
|
||||
),
|
||||
)
|
||||
executed = AsyncMock(return_value=_text_result("[]"))
|
||||
if tool_search_enabled:
|
||||
called_tool = "mcp_tool_search"
|
||||
expected_cap = MAX_MCP_TOOL_CALL_ROUNDS * 2
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_search", executed
|
||||
)
|
||||
else:
|
||||
called_tool = "deepwiki-read_wiki_contents"
|
||||
expected_cap = MAX_MCP_TOOL_CALL_ROUNDS
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager.call_tool",
|
||||
executed,
|
||||
)
|
||||
|
||||
aresponses_mock = _patch_aresponses(
|
||||
monkeypatch,
|
||||
[
|
||||
_function_call_response(f"resp-{i}", f"call-{i}", called_tool, {"query": "wiki"})
|
||||
for i in range(MAX_MCP_TOOL_CALL_ROUNDS * 2 + 5)
|
||||
],
|
||||
)
|
||||
|
||||
await aresponses_api_with_mcp(
|
||||
input="describe the wiki",
|
||||
model="gpt-5.5",
|
||||
tools=[MCP_TOOL_CONFIG],
|
||||
litellm_metadata={"user_api_key_auth": _auth(tool_search_enabled)},
|
||||
user_api_key_auth=_auth(tool_search_enabled),
|
||||
)
|
||||
|
||||
assert executed.await_count == expected_cap
|
||||
assert aresponses_mock.await_count == expected_cap + 1
|
||||
assert aresponses_mock.await_args_list[-1].kwargs["tools"] is None
|
||||
Loading…
Add table
Reference in a new issue