This commit is contained in:
Gaurav Pandey 2026-10-27 02:24:43 +00:00 • committed by GitHub
commit 6a7ef42d0f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 361 additions and 136 deletions

View file

@ -100,7 +100,7 @@ from litellm.constants import (
MCP_TOOL_LISTING_MAX_PAGES,
MCP_TOOL_LISTING_TIMEOUT,
)
from litellm.experimental_mcp_client.tools import list_tools_with_pagination
from litellm.experimental_mcp_client.tools import list_tools_result_with_pagination, list_tools_with_pagination
from litellm.llms.custom_httpx.http_handler import get_ssl_configuration
from litellm.proxy._experimental.mcp_server.mcp_context import get_active_mcp_request_ctx
from litellm.proxy._experimental.mcp_server.mcp_debug import capture_upstream_error_response
@ -1057,6 +1057,9 @@ class MCPClient:
return await self.run_with_session(lambda session: session.list_tools(params=params), quiet_on_error=True)
async def list_tools(self, raise_on_error: bool = False) -> list[MCPTool]:
return (await self.list_tools_result(raise_on_error=raise_on_error)).tools
async def list_tools_result(self, raise_on_error: bool = False) -> ListToolsResult:
"""List available tools from the server.
Args:
@ -1071,16 +1074,20 @@ class MCPClient:
try:
# A per-server timeout above the global default extends the whole-walk deadline
listing_deadline: Final = max(self.timeout, MCP_TOOL_LISTING_TIMEOUT)
tools: Final = await self.run_with_session(
partial(list_tools_with_pagination, listing_deadline=listing_deadline, require_complete=raise_on_error),
listing: Final = await self.run_with_session(
partial(
list_tools_result_with_pagination,
listing_deadline=listing_deadline,
require_complete=raise_on_error,
),
quiet_on_error=raise_on_error,
)
tool_count: Final = len(tools)
tool_names: Final = tuple(tool.name for tool in tools)
tool_count: Final = len(listing.tools)
tool_names: Final = tuple(tool.name for tool in listing.tools)
verbose_logger.info(
"MCP client listed %s tools from %s: %s", tool_count, self.server_url or "stdio", tool_names
)
return tools
return listing
except asyncio.CancelledError:
verbose_logger.warning("MCP client list_tools was cancelled")
raise
@ -1110,7 +1117,7 @@ class MCPClient:
if raise_on_error:
raise
# Return empty list instead of raising to allow graceful degradation
return []
return ListToolsResult(tools=[])
@staticmethod
def error_tool_result(exc: Exception) -> MCPCallToolResult:

View file

@ -1,9 +1,11 @@
import json
import time
from collections.abc import Sequence
from typing import Final, Literal
import anyio
from mcp import ClientSession, MCPError
from mcp.types import INTERNAL_ERROR, PaginatedRequestParams
from mcp.types import INTERNAL_ERROR, ListToolsResult, PaginatedRequestParams
from mcp.types import CallToolRequestParams as MCPCallToolRequestParams
from mcp.types import CallToolResult as MCPCallToolResult
from mcp.types import Tool as MCPTool
@ -18,6 +20,7 @@ from litellm.constants import (
MCP_TOOL_LISTING_MAX_PAGES,
MCP_TOOL_LISTING_TIMEOUT,
)
from litellm.proxy._experimental.mcp_server.result_conversion import age_freshness, aggregate_freshness
from litellm.types.llms.anthropic import AnthropicMessagesTool
from litellm.types.utils import ChatCompletionMessageToolCall
@ -101,9 +104,21 @@ def transform_mcp_tool_to_anthropic_tool(mcp_tool: MCPTool) -> AnthropicMessages
)
def _complete_listing(tools: Sequence[MCPTool], pages: Sequence[tuple[ListToolsResult, float]]) -> ListToolsResult:
now: Final = time.monotonic()
freshness: Final = aggregate_freshness(tuple(age_freshness(page, now - received) for page, received in pages))
return ListToolsResult(tools=list(tools), ttl_ms=freshness.ttl_ms, cache_scope=freshness.cache_scope)
async def list_tools_with_pagination(
session: ClientSession, listing_deadline: float | None = None, *, require_complete: bool = False
) -> list[MCPTool]: # mutable-ok: list return contract
return (await list_tools_result_with_pagination(session, listing_deadline, require_complete=require_complete)).tools
async def list_tools_result_with_pagination(
session: ClientSession, listing_deadline: float | None = None, *, require_complete: bool = False
) -> ListToolsResult:
"""Collect tools from every tools/list page by following nextCursor.
Stops and returns the tools collected so far when the upstream repeats a
@ -111,8 +126,10 @@ async def list_tools_with_pagination(
buggy or slow upstream yields a partial catalog instead of an error.
listing_deadline overrides the default whole-walk deadline; callers with a
per-server timeout above the global default pass it through here.
Only a complete walk carries the upstream ttlMs, so a partial catalog is never cached.
"""
tools: Final[list[MCPTool]] = [] # mutable-ok: accumulates each page's tools
pages: Final[list[tuple[ListToolsResult, float]]] = [] # mutable-ok: bounded pagination evidence
seen_cursors: Final[set[str]] = set() # mutable-ok: guards against cursor loops
cursor: str | None = None # rebind-ok: advances to each page's nextCursor
# The per-request session read timeout restarts on every page, so a multi-page
@ -131,10 +148,11 @@ async def list_tools_with_pagination(
else await session.list_tools(params=PaginatedRequestParams(cursor=cursor))
)
tools.extend(result.tools)
pages.append((result, time.monotonic()))
next_cursor = result.next_cursor
if not isinstance(next_cursor, str) or not next_cursor:
return tools
return _complete_listing(tools, pages)
if next_cursor in seen_cursors:
verbose_logger.warning(
"MCP server repeated a tools/list cursor while listing tools; returning %s tools collected so far",
@ -144,7 +162,7 @@ async def list_tools_with_pagination(
raise MCPError(
code=INTERNAL_ERROR, message="Upstream tool discovery is incomplete: repeated cursor"
)
return tools
return ListToolsResult(tools=tools)
seen_cursors.add(next_cursor)
cursor = next_cursor
@ -155,7 +173,7 @@ async def list_tools_with_pagination(
)
if require_complete:
raise MCPError(code=INTERNAL_ERROR, message="Upstream tool discovery is incomplete: pagination limit")
return tools
return ListToolsResult(tools=tools)
verbose_logger.warning(
"MCP server tools/list pagination exceeded the %s second listing deadline; returning %s tools collected so far",
@ -164,7 +182,7 @@ async def list_tools_with_pagination(
)
if require_complete:
raise MCPError(code=INTERNAL_ERROR, message="Upstream tool discovery is incomplete: listing deadline")
return tools
return ListToolsResult(tools=tools)
async def load_mcp_tools(

View file

@ -26,7 +26,7 @@ from collections.abc import (
)
from contextlib import AbstractAsyncContextManager, ExitStack, asynccontextmanager, nullcontext
from dataclasses import dataclass, replace
from functools import lru_cache
from functools import lru_cache, partial
from itertools import chain, groupby
from types import EllipsisType, MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypedDict, cast
@ -2035,6 +2035,9 @@ class MCPServerManager:
self._template_discovery_cache = _DiscoveryCache[ListResourceTemplatesResult](
discovery_ttl, discovery_clock, TypeAdapter(ListResourceTemplatesResult)
)
self._tool_discovery_cache = _DiscoveryCache[ListToolsResult](
discovery_ttl, discovery_clock, TypeAdapter(ListToolsResult)
)
from litellm.proxy._experimental.mcp_server.catalog import CatalogSnapshots
self.catalog = CatalogSnapshots(self)
@ -4582,14 +4585,27 @@ class MCPServerManager:
tools=[t.model_copy(update={"name": registered_names[t.name]}) for t in guarded_openapi]
)
else:
page: Final = (
await retry_upstream_rate_limit(
lambda: self._fetch_tools_page_with_timeout(client, server, params),
key: Final = self._discovery_key(
server,
user_api_key_auth,
mcp_auth_header,
extra_headers,
stdio_env,
subject_token,
await client.discovery_auth_fingerprint(),
raw_headers=raw_headers,
page_cursor=None if params is None else (params.cursor,),
)
page: Final = await self._tool_discovery_cache.get(
key,
partial(self._fetch_tools_with_timeout, client, server.name)
if params is None
else partial(
retry_upstream_rate_limit,
partial(self._fetch_tools_page_with_timeout, client, server, params),
server.name,
deadline_seconds=MCP_TOOL_LISTING_TIMEOUT,
)
if params is not None
else ListToolsResult(tools=await self._fetch_tools_with_timeout(client, server.name))
),
)
tools = page.tools
self._remember_upstream_initialize_instructions(server, client)
@ -4660,6 +4676,7 @@ class MCPServerManager:
self._prompt_discovery_cache.invalidate(server_id)
self._resource_discovery_cache.invalidate(server_id)
self._template_discovery_cache.invalidate(server_id)
self._tool_discovery_cache.invalidate(server_id)
def invalidate_server_definition_caches(self, server_id: str) -> None:
from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( # noqa: PLC0415 # lazy: discoverable_endpoints lazily imports this module's manager singleton
@ -4761,6 +4778,7 @@ class MCPServerManager:
subject_token: str | None,
credential_fingerprint: str | None = None,
raw_headers: Mapping[str, str] | None = None,
page_cursor: tuple[str | None] | None = None,
) -> _DiscoveryKey:
identity: Final = (
user_api_key_auth.model_dump(
@ -4787,6 +4805,7 @@ class MCPServerManager:
stdio_env,
subject_token,
credential_fingerprint,
page_cursor,
),
sort_keys=True,
separators=(",", ":"),
@ -5628,7 +5647,7 @@ class MCPServerManager:
self,
client: MCPClient,
server_name: str,
) -> list[MCPTool]:
) -> ListToolsResult:
"""
Fetch tools from MCP client with timeout and error handling.
@ -5652,17 +5671,17 @@ class MCPServerManager:
server_name: Name of the server for logging
Returns:
List of tools from the server
The tools from the server with the upstream freshness of the complete walk
"""
try:
with anyio.fail_after(MCP_TOOL_LISTING_TIMEOUT):
tools: Final = await retry_upstream_rate_limit(
lambda: client.list_tools(raise_on_error=True),
listing: Final = await retry_upstream_rate_limit(
lambda: client.list_tools_result(raise_on_error=True),
server_name,
deadline_seconds=MCP_TOOL_LISTING_TIMEOUT,
)
verbose_logger.debug("Tools from %s: %s", server_name, tools)
return tools
verbose_logger.debug("Tools from %s: %s", server_name, listing.tools)
return listing
except TimeoutError as e:
verbose_logger.warning("Timeout while listing tools from %s", server_name)
raise MCPServerListError(ServerListFault(tag="timeout"), server_name) from e

View file

@ -14,6 +14,7 @@ from mcp.types import (
from mcp.types import Tool as MCPTool
from litellm.experimental_mcp_client.tools import (
list_tools_result_with_pagination,
list_tools_with_pagination,
transform_mcp_tool_to_anthropic_tool,
_get_function_arguments,
@ -171,6 +172,22 @@ async def test_pagination_walk_stops_on_repeated_cursor(mock_session):
assert mock_session.list_tools.call_count == 2
@pytest.mark.asyncio()
@pytest.mark.parametrize("last_cursor", [None, "page-1"])
async def test_pagination_walk_keeps_the_shortest_ttl_only_for_a_complete_walk(mock_session, last_cursor):
mock_session.list_tools.side_effect = [
ListToolsResult(tools=[MCPTool(name="tool_0", inputSchema={})], nextCursor="page-1", ttl_ms=30000),
ListToolsResult(tools=[MCPTool(name="tool_1", inputSchema={})], nextCursor=last_cursor, ttl_ms=20000),
]
result = await list_tools_result_with_pagination(mock_session)
assert [tool.name for tool in result.tools] == ["tool_0", "tool_1"]
if last_cursor is None:
assert 19000 < result.ttl_ms <= 20000
else:
assert result.ttl_ms == 0
@pytest.mark.asyncio()
async def test_pagination_walk_treats_empty_cursor_as_terminal(mock_session):
mock_session.list_tools.side_effect = [

View file

@ -208,12 +208,12 @@ async def test_cancelled_fetch_propagates_instead_of_masquerading_as_a_healthy_e
client = MagicMock()
started = asyncio.Event()
async def list_tools(raise_on_error: bool = False) -> list[object]:
async def list_tools_result(raise_on_error: bool = False) -> list[object]:
started.set()
await asyncio.Event().wait()
return []
client.list_tools = list_tools
client.list_tools_result = list_tools_result
listing = asyncio.create_task(manager._fetch_tools_with_timeout(client, "cancelled_srv"))
await started.wait()
listing.cancel(cancel_message)
@ -235,7 +235,7 @@ async def test_client_side_cancelled_error_is_a_classified_fault_not_a_cancellat
manager = MCPServerManager()
client = MagicMock()
client.list_tools = AsyncMock(side_effect=asyncio.CancelledError(SESSION_CLEANUP_TIMEOUT_MESSAGE))
client.list_tools_result = AsyncMock(side_effect=asyncio.CancelledError(SESSION_CLEANUP_TIMEOUT_MESSAGE))
with pytest.raises(MCPServerListError) as exc_info:
await manager._fetch_tools_with_timeout(client, "slow_teardown_srv")

View file

@ -186,9 +186,7 @@ class TestMCPClientUnitTests:
},
)
]
mock_result = MagicMock()
mock_result.tools = mock_tools
mock_session_instance.list_tools.return_value = mock_result
mock_session_instance.list_tools.return_value = ListToolsResult(tools=mock_tools)
client = MCPClient("http://example.com")
result = await client.list_tools()

View file

@ -10,7 +10,8 @@ from unittest.mock import AsyncMock, MagicMock
import httpx
import pytest
from mcp.types import ListToolsResult
from mcp.types import Tool as MCPTool
if sys.version_info < (3, 11):
from exceptiongroup import ExceptionGroup
@ -115,7 +116,7 @@ async def test_fetch_tools_from_passthrough_raises_on_upstream_401():
)
mock_client = MagicMock()
mock_client.list_tools = AsyncMock(side_effect=upstream_error)
mock_client.list_tools_result = AsyncMock(side_effect=upstream_error)
with pytest.raises(MCPUpstreamAuthError) as exc_info:
await manager._fetch_tools_with_timeout(mock_client, passthrough_server.name)
@ -125,7 +126,7 @@ async def test_fetch_tools_from_passthrough_raises_on_upstream_401():
'Bearer resource_metadata="https://upstream"'
)
assert exc_info.value.server_name == "sample_docs"
mock_client.list_tools.assert_awaited_with(raise_on_error=True)
mock_client.list_tools_result.assert_awaited_with(raise_on_error=True)
@pytest.mark.asyncio
@ -150,7 +151,7 @@ async def test_fetch_tools_from_delegated_oauth2_raises_on_upstream_401():
)
mock_client = MagicMock()
mock_client.list_tools = AsyncMock(side_effect=upstream_error)
mock_client.list_tools_result = AsyncMock(side_effect=upstream_error)
with pytest.raises(MCPUpstreamAuthError) as exc_info:
await manager._fetch_tools_with_timeout(mock_client, delegated_server.name)
@ -160,7 +161,7 @@ async def test_fetch_tools_from_delegated_oauth2_raises_on_upstream_401():
'Bearer resource_metadata="https://upstream"'
)
assert exc_info.value.server_name == "delegated_docs"
mock_client.list_tools.assert_awaited_with(raise_on_error=True)
mock_client.list_tools_result.assert_awaited_with(raise_on_error=True)
@pytest.mark.asyncio
@ -189,14 +190,14 @@ async def test_fetch_tools_from_client_credentials_oauth2_surfaces_upstream_401(
)
mock_client = MagicMock()
mock_client.list_tools = AsyncMock(side_effect=upstream_error)
mock_client.list_tools_result = AsyncMock(side_effect=upstream_error)
with pytest.raises(MCPUpstreamAuthError) as exc_info:
await manager._fetch_tools_with_timeout(mock_client, m2m_server.name)
assert exc_info.value.status_code == 401
assert exc_info.value.server_name == "m2m_docs"
mock_client.list_tools.assert_awaited_with(raise_on_error=True)
mock_client.list_tools_result.assert_awaited_with(raise_on_error=True)
@pytest.mark.asyncio
@ -212,13 +213,12 @@ async def test_fetch_tools_from_passthrough_returns_tools_on_success():
oauth_passthrough=True,
)
tool = MagicMock()
tool.name = "list_documents"
tool = MCPTool(name="list_documents", inputSchema={"type": "object"})
mock_client = MagicMock()
mock_client.list_tools = AsyncMock(return_value=[tool])
mock_client.list_tools_result = AsyncMock(return_value=ListToolsResult(tools=[tool]))
tools = await manager._fetch_tools_with_timeout(mock_client, passthrough_server.name)
assert tools == [tool]
listing = await manager._fetch_tools_with_timeout(mock_client, passthrough_server.name)
assert listing.tools == [tool]
@pytest.mark.parametrize("status_code", [401, 403])
@ -308,7 +308,7 @@ async def test_fetch_tools_from_gateway_managed_surfaces_upstream_401():
"401", request=response.request, response=response
)
mock_client = MagicMock()
mock_client.list_tools = AsyncMock(side_effect=upstream_error)
mock_client.list_tools_result = AsyncMock(side_effect=upstream_error)
with pytest.raises(MCPUpstreamAuthError) as exc_info:
await manager._fetch_tools_with_timeout(mock_client, oauth2_server.name)
@ -316,7 +316,7 @@ async def test_fetch_tools_from_gateway_managed_surfaces_upstream_401():
assert exc_info.value.status_code == 401
assert exc_info.value.www_authenticate is None
assert exc_info.value.server_name == "keycloak_whoami"
mock_client.list_tools.assert_awaited_with(raise_on_error=True)
mock_client.list_tools_result.assert_awaited_with(raise_on_error=True)
def _http_server(server_id: str, name: str, **kwargs) -> MCPServer:
@ -464,7 +464,7 @@ async def test_fetch_tools_logs_upstream_request_details_on_500(caplog):
)
response = httpx.Response(500, request=request)
mock_client = MagicMock()
mock_client.list_tools = AsyncMock(
mock_client.list_tools_result = AsyncMock(
side_effect=httpx.HTTPStatusError("500", request=request, response=response)
)
@ -554,14 +554,16 @@ async def test_scope_authorization_releases_progress_without_waiting_for_tool_re
from mcp.server.context import ServerRequestContext
from starlette.requests import Request
from litellm.proxy._experimental.mcp_server.legacy_callbacks import (
get_scope_response,
record_upstream_tool_authorization,
)
from litellm.proxy._experimental.mcp_server.mcp_context import active_mcp_request_ctx_var
from litellm.proxy._experimental.mcp_server.scope_challenge import (
SCOPE_RESPONSE_KEY,
OAuthScopeResponse,
)
from litellm.proxy._experimental.mcp_server.legacy_callbacks import get_scope_response, record_upstream_tool_authorization
send = AsyncMock()
gate = OAuthScopeResponse(send, "https://gateway.example", "/mcp/docs")
request = Request({"type": "http", SCOPE_RESPONSE_KEY: gate})
@ -602,7 +604,11 @@ async def test_scope_middleware_releases_waiting_response_on_validation_or_cance
from mcp.server.context import ServerRequestContext
from starlette.requests import Request
from litellm.proxy._experimental.mcp_server.scope_challenge import SCOPE_RESPONSE_KEY, OAuthScopeResponse, finish_scope_response
from litellm.proxy._experimental.mcp_server.scope_challenge import (
SCOPE_RESPONSE_KEY,
OAuthScopeResponse,
finish_scope_response,
)
send: Final = AsyncMock(side_effect=BrokenPipeError("peer disconnected") if fail_send else None)
gate: Final = OAuthScopeResponse(send, "https://gateway.example", "/mcp/docs")
@ -720,7 +726,11 @@ async def test_streamable_sdk_relays_scope_challenge_before_committing_http_resp
from litellm.proxy._experimental.mcp_server.legacy_callbacks import get_scope_response
from litellm.proxy._experimental.mcp_server.mcp_context import active_mcp_request_ctx_var
from litellm.proxy._experimental.mcp_server.scope_challenge import SCOPE_RESPONSE_KEY, OAuthScopeResponse, finish_scope_response
from litellm.proxy._experimental.mcp_server.scope_challenge import (
SCOPE_RESPONSE_KEY,
OAuthScopeResponse,
finish_scope_response,
)
async def call_tool(context, params):
token = active_mcp_request_ctx_var.set(context)
@ -772,7 +782,10 @@ async def test_scope_challenges_are_isolated_between_concurrent_requests():
from mcp.server.context import ServerRequestContext
from starlette.requests import Request
from litellm.proxy._experimental.mcp_server.legacy_callbacks import get_scope_response, record_upstream_tool_authorization
from litellm.proxy._experimental.mcp_server.legacy_callbacks import (
get_scope_response,
record_upstream_tool_authorization,
)
from litellm.proxy._experimental.mcp_server.mcp_context import active_mcp_request_ctx_var
from litellm.proxy._experimental.mcp_server.scope_challenge import SCOPE_RESPONSE_KEY, OAuthScopeResponse
@ -824,9 +837,10 @@ async def test_scope_middleware_preserves_non_http_requests():
@pytest.mark.parametrize("path", ["/mcp", "/toolset/demo/mcp"])
def test_managed_step_up_metadata_matches_the_sdk_configured_resource(path):
from mcp.shared.auth_utils import check_resource_allowed
from urllib.parse import urlsplit
from mcp.shared.auth_utils import check_resource_allowed
error = MCPUpstreamAuthError(403, None, "docs", required_scope="tools.write")
response = error.to_http_exception("https://gateway.test", path)
challenge = response.headers["www-authenticate"]

View file

@ -3,12 +3,13 @@ import asyncio
import json
import logging
from datetime import datetime
from types import SimpleNamespace
from unittest.mock import AsyncMock, patch
import pytest
from fastapi import HTTPException
from mcp.shared.exceptions import MCPError
from mcp.types import CallToolResult, TextContent
from mcp.types import CallToolResult, ListToolsResult, TextContent
from mcp.types import Tool as MCPTool
from pydantic import AnyUrl
@ -29,6 +30,10 @@ from litellm.types.mcp_server.mcp_server_manager import MCPServer
AUTH = UserAPIKeyAuth(api_key="key")
def _listing_client() -> SimpleNamespace:
return SimpleNamespace(discovery_auth_fingerprint=AsyncMock(return_value="test-credential-hash"))
@pytest.fixture
def proxy_mode():
token = _mcp_proxy_mode.set(True)
@ -165,8 +170,8 @@ async def test_proxy_call_tool_on_a_never_listed_tool_hands_the_pre_hook_no_list
patch.dict(manager.registry, {server.server_id: server}),
patch.dict(manager.tool_name_to_mcp_server_name_mapping),
patch.object(mcp_operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[server])),
patch.object(manager, "create_mcp_client", AsyncMock(return_value=object())),
patch.object(manager, "_fetch_tools_with_timeout", AsyncMock(return_value=upstream)),
patch.object(manager, "create_mcp_client", AsyncMock(return_value=_listing_client())),
patch.object(manager, "_fetch_tools_with_timeout", AsyncMock(return_value=ListToolsResult(tools=upstream))),
patch.object(manager, "pre_call_tool_check", pre_call_tool_check),
patch.object(manager, "_call_regular_mcp_tool", call_regular_mcp_tool),
):
@ -204,14 +209,15 @@ async def test_proxy_schema_and_call_resolve_tool_id_without_waiting_on_unrelate
slow_teardown = asyncio.Event()
served_as = with_mcp_proxy_identity(MCPTool(name="proxy-fast-echo", inputSchema={}), fast.server_id)
async def fetch_tools(client: object, server_name: str) -> list[MCPTool]:
async def fetch_tools(client: object, server_name: str) -> ListToolsResult:
if server_name == slow.name:
try:
await slow_release.wait()
except asyncio.CancelledError:
await slow_teardown.wait()
raise
return [MCPTool(name="echo", description="Echo text back", inputSchema={"type": "object"})]
echo = MCPTool(name="echo", description="Echo text back", inputSchema={"type": "object"})
return ListToolsResult(tools=[echo])
async def call_regular_mcp_tool(*, tasks: list[asyncio.Task[object]], **_: object) -> CallToolResult:
await asyncio.gather(*tasks)
@ -222,7 +228,7 @@ async def test_proxy_schema_and_call_resolve_tool_id_without_waiting_on_unrelate
patch.dict(manager.registry, {fast.server_id: fast, slow.server_id: slow}),
patch.dict(manager.tool_name_to_mcp_server_name_mapping),
patch.object(mcp_operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[slow, fast])),
patch.object(manager, "create_mcp_client", AsyncMock(return_value=object())),
patch.object(manager, "create_mcp_client", AsyncMock(return_value=_listing_client())),
patch.object(manager, "_fetch_tools_with_timeout", fetch_tools),
patch.object(manager, "pre_call_tool_check", AsyncMock(return_value={})),
patch.object(manager, "_call_regular_mcp_tool", call_regular_mcp_tool),

View file

@ -69,7 +69,8 @@ async def test_mcp_server_manager_https_server():
# Create a mock MCPClient
mock_client = AsyncMock()
mock_client.list_tools = AsyncMock(return_value=mock_tools)
mock_client.list_tools_result = AsyncMock(return_value=ListToolsResult(tools=mock_tools))
mock_client.discovery_auth_fingerprint = AsyncMock(return_value="test-credential-hash")
mock_client.call_tool = AsyncMock(return_value=mock_result)
# Mock the MCPClient constructor
@ -133,7 +134,7 @@ async def test_mcp_server_manager_https_server():
assert result.content[0].text == "Email sent successfully"
# Verify client methods were called
mock_client.list_tools.assert_called()
mock_client.list_tools_result.assert_called()
mock_client.call_tool.assert_called_once()
@ -176,7 +177,8 @@ async def test_mcp_http_transport_list_tools_mock():
# Create a mock MCPClient that returns our test tools
mock_client = AsyncMock()
mock_client.list_tools = AsyncMock(return_value=mock_tools)
mock_client.list_tools_result = AsyncMock(return_value=ListToolsResult(tools=mock_tools))
mock_client.discovery_auth_fingerprint = AsyncMock(return_value="test-credential-hash")
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
mock_client.__aexit__ = AsyncMock(return_value=None)
@ -218,7 +220,7 @@ async def test_mcp_http_transport_list_tools_mock():
assert tools[1].name == f"{expected_prefix}-calendar_create_event"
# Verify client methods were called
mock_client.list_tools.assert_called()
mock_client.list_tools_result.assert_called()
# Verify tool mapping was updated
expected_prefix = "test_http_server"
@ -1104,9 +1106,10 @@ async def test_list_tools_only_returns_allowed_servers(monkeypatch):
mock_client = AsyncMock()
# Return tools based on server URL
if kwargs.get("server_url") == "https://server-a.com/mcp":
mock_client.list_tools = AsyncMock(return_value=mock_tools_a)
mock_client.list_tools_result = AsyncMock(return_value=ListToolsResult(tools=mock_tools_a))
else:
mock_client.list_tools = AsyncMock(return_value=mock_tools_b)
mock_client.list_tools_result = AsyncMock(return_value=ListToolsResult(tools=mock_tools_b))
mock_client.discovery_auth_fingerprint = AsyncMock(return_value="test-credential-hash")
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
mock_client.__aexit__ = AsyncMock(return_value=None)
return mock_client
@ -1401,7 +1404,8 @@ async def test_mcp_server_manager_alias_tool_prefixing():
# Mock MCPClient
mock_client = AsyncMock()
mock_client.list_tools = AsyncMock(return_value=mock_tools)
mock_client.list_tools_result = AsyncMock(return_value=ListToolsResult(tools=mock_tools))
mock_client.discovery_auth_fingerprint = AsyncMock(return_value="test-credential-hash")
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
mock_client.__aexit__ = AsyncMock(return_value=None)
@ -1461,7 +1465,8 @@ async def test_mcp_server_manager_server_name_tool_prefixing():
# Mock MCPClient
mock_client = AsyncMock()
mock_client.list_tools = AsyncMock(return_value=mock_tools)
mock_client.list_tools_result = AsyncMock(return_value=ListToolsResult(tools=mock_tools))
mock_client.discovery_auth_fingerprint = AsyncMock(return_value="test-credential-hash")
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
mock_client.__aexit__ = AsyncMock(return_value=None)
@ -1521,7 +1526,8 @@ async def test_mcp_server_manager_server_id_tool_prefixing():
# Mock MCPClient
mock_client = AsyncMock()
mock_client.list_tools = AsyncMock(return_value=mock_tools)
mock_client.list_tools_result = AsyncMock(return_value=ListToolsResult(tools=mock_tools))
mock_client.discovery_auth_fingerprint = AsyncMock(return_value="test-credential-hash")
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
mock_client.__aexit__ = AsyncMock(return_value=None)

View file

@ -11,7 +11,7 @@ import anyio
from collections.abc import AsyncIterator, Awaitable, Callable, Iterator
from datetime import datetime
from pathlib import Path
from types import SimpleNamespace
from types import EllipsisType, SimpleNamespace
from typing import Any, Dict, Final, Literal, Optional, TypeVar, cast
from unittest.mock import AsyncMock, MagicMock, patch
@ -242,6 +242,12 @@ async def test_sampling_callbacks_isolate_callers_and_cancellation():
assert cancelled.is_set()
def _listing_client() -> AsyncMock:
client: Final = AsyncMock()
client.discovery_auth_fingerprint = AsyncMock(return_value="test-credential-hash")
return client
def _reload_mcp_manager_module():
utils_module = sys.modules["litellm.proxy._experimental.mcp_server.utils"]
manager_module = sys.modules["litellm.proxy._experimental.mcp_server.mcp_server_manager"]
@ -2353,10 +2359,10 @@ class TestMCPServerManager:
server, mcp_auth_header, extra_headers, stdio_env, subject_token=None, **kwargs
): # pragma: no cover - helper
captured["subject_token"] = subject_token
return AsyncMock()
return _listing_client()
manager.create_mcp_client = AsyncMock(side_effect=capture_create_mcp_client)
manager._fetch_tools_with_timeout = AsyncMock(return_value=[])
manager._fetch_tools_with_timeout = AsyncMock(return_value=ListToolsResult(tools=[]))
await manager._get_tools_from_server(server=server, oauth2_headers=oauth2_headers, raw_headers=raw_headers)
return captured["subject_token"]
@ -3175,7 +3181,7 @@ class TestMCPServerManager:
server, mcp_auth_header, extra_headers, stdio_env, subject_token=None, **kwargs
): # pragma: no cover - helper
captured["subject_token"] = subject_token
return AsyncMock()
return _listing_client()
manager.create_mcp_client = AsyncMock(side_effect=capture_create_mcp_client)
await call(manager)
@ -6492,7 +6498,7 @@ class TestMCPServerManager:
)
# Mock client creation and fetching tools
manager.create_mcp_client = AsyncMock(return_value=object())
manager.create_mcp_client = AsyncMock(return_value=_listing_client())
# Tools returned upstream (unprefixed from provider)
upstream_tool = MCPTool(
@ -6501,7 +6507,7 @@ class TestMCPServerManager:
inputSchema={},
)
manager._fetch_tools_with_timeout = AsyncMock(return_value=[upstream_tool])
manager._fetch_tools_with_timeout = AsyncMock(return_value=ListToolsResult(tools=[upstream_tool]))
# Case 1: add_prefix=True (default for multi-server) -> expect prefixed
tools_prefixed = await manager._get_tools_from_server(server, add_prefix=True)
@ -6530,8 +6536,8 @@ class TestMCPServerManager:
transport=MCPTransport.http,
)
manager.create_mcp_client = AsyncMock(return_value=object())
manager._fetch_tools_with_timeout = AsyncMock(return_value=[])
manager.create_mcp_client = AsyncMock(return_value=_listing_client())
manager._fetch_tools_with_timeout = AsyncMock(return_value=ListToolsResult(tools=[]))
user_auth = UserAPIKeyAuth(api_key="sk-test", user_id="alice")
@ -7868,9 +7874,9 @@ class TestMCPServerManager:
async def fetch(client, name):
fetch_started.set()
await release_fetch.wait()
return [MCPTool(name="turn", description="before save", inputSchema={})]
return ListToolsResult(tools=[MCPTool(name="turn", description="before save", inputSchema={})])
manager.create_mcp_client = AsyncMock(return_value=AsyncMock())
manager.create_mcp_client = AsyncMock(return_value=_listing_client())
manager._fetch_tools_with_timeout = fetch
caller = ListedToolsCaller(user_api_key_auth=user)
@ -7886,7 +7892,7 @@ class TestMCPServerManager:
assert manager.get_listed_tool(server, "turn", caller) is None
manager._fetch_tools_with_timeout = AsyncMock(
return_value=[MCPTool(name="turn", description="after save", inputSchema={})]
return_value=ListToolsResult(tools=[MCPTool(name="turn", description="after save", inputSchema={})])
)
await manager._get_tools_from_server(server=server, user_api_key_auth=user, record_listing=True)
listed = manager.get_listed_tool(server, "turn", caller)
@ -8122,9 +8128,9 @@ class TestMCPServerManager:
auth_type=MCPAuth.api_key,
)
user = UserAPIKeyAuth(api_key="sk-litellm", user_id="byok-cold-user")
manager.create_mcp_client = AsyncMock(return_value=AsyncMock())
manager.create_mcp_client = AsyncMock(return_value=_listing_client())
manager._fetch_tools_with_timeout = AsyncMock(
return_value=[MCPTool(name="turn", description="listed while db down", inputSchema={})]
return_value=ListToolsResult(tools=[MCPTool(name="turn", description="listed while db down", inputSchema={})])
)
with (
@ -8172,11 +8178,11 @@ class TestMCPServerManager:
)
manager.registry = {"byok-catalog": server}
user = UserAPIKeyAuth(api_key="sk-litellm", user_id="byok-user")
mock_client = AsyncMock()
mock_client = _listing_client()
mock_client.call_tool.return_value = MagicMock(spec=CallToolResult, content=[], isError=False)
manager.create_mcp_client = AsyncMock(return_value=mock_client)
manager._fetch_tools_with_timeout = AsyncMock(
return_value=[MCPTool(name="turn", description="stored cred catalog", inputSchema={})]
return_value=ListToolsResult(tools=[MCPTool(name="turn", description="stored cred catalog", inputSchema={})])
)
proxy_logging_obj = _mock_proxy_logging()
proxy_logging_obj.create_mcp_request_object_from_kwargs = MagicMock(return_value={})
@ -8217,9 +8223,9 @@ class TestMCPServerManager:
url="http://byok-catalog",
is_byok=True,
)
manager.create_mcp_client = AsyncMock(return_value=AsyncMock())
manager.create_mcp_client = AsyncMock(return_value=_listing_client())
manager._fetch_tools_with_timeout = AsyncMock(
return_value=[MCPTool(name="turn", description="t", inputSchema={})]
return_value=ListToolsResult(tools=[MCPTool(name="turn", description="t", inputSchema={})])
)
with patch("litellm.proxy.proxy_server.prisma_client", None):
@ -8280,9 +8286,9 @@ class TestMCPServerManager:
**server_auth,
)
alice = UserAPIKeyAuth(api_key="sk-alice", user_id="alice")
manager.create_mcp_client = AsyncMock(return_value=AsyncMock())
manager.create_mcp_client = AsyncMock(return_value=_listing_client())
manager._fetch_tools_with_timeout = AsyncMock(
return_value=[MCPTool(name="echo", description="listed catalog", inputSchema={})]
return_value=ListToolsResult(tools=[MCPTool(name="echo", description="listed catalog", inputSchema={})])
)
signer_headers = AsyncMock(return_value={"Authorization": "Bearer signed-jwt"})
cache_byok_credential("alice", "cc1", "BYOK-ALICE-SECRET")
@ -8496,10 +8502,10 @@ class TestMCPServerManager:
)
],
}
mock_client = AsyncMock()
mock_client = _listing_client()
mock_client.call_tool.return_value = MagicMock(spec=CallToolResult, content=[], isError=False)
manager.create_mcp_client = AsyncMock(return_value=mock_client)
manager._fetch_tools_with_timeout = AsyncMock(side_effect=lambda client, name: catalogs[client.workspace])
manager._fetch_tools_with_timeout = AsyncMock(side_effect=lambda client, name: ListToolsResult(tools=catalogs[client.workspace]))
for workspace in ("A", "B"):
manager.create_mcp_client.return_value.workspace = workspace
await manager._get_tools_from_server(
@ -11787,7 +11793,7 @@ class TestMCPToolsListAuthSurfacing:
manager = MCPServerManager()
challenge = 'Bearer resource_metadata="https://upstream.example/.well-known/oauth-protected-resource"'
client = MagicMock()
client.list_tools = AsyncMock(side_effect=_upstream_status_error(401, challenge))
client.list_tools_result = AsyncMock(side_effect=_upstream_status_error(401, challenge))
with pytest.raises(MCPUpstreamAuthError) as exc_info:
await manager._fetch_tools_with_timeout(client, "static-key-server")
@ -11805,7 +11811,7 @@ class TestMCPToolsListAuthSurfacing:
manager = MCPServerManager()
challenge = 'Bearer error="insufficient_scope", scope="read:tools"'
client = MagicMock()
client.list_tools = AsyncMock(side_effect=_upstream_status_error(403, challenge))
client.list_tools_result = AsyncMock(side_effect=_upstream_status_error(403, challenge))
with pytest.raises(MCPUpstreamAuthError) as exc_info:
await manager._fetch_tools_with_timeout(client, "forbidden-server")
@ -11822,7 +11828,7 @@ class TestMCPToolsListAuthSurfacing:
own fault ("internal")."""
manager = MCPServerManager()
client = MagicMock()
client.list_tools = AsyncMock(side_effect=RuntimeError("upstream 500"))
client.list_tools_result = AsyncMock(side_effect=RuntimeError("upstream 500"))
with pytest.raises(MCPServerListError) as exc_info:
await manager._fetch_tools_with_timeout(client, "srv")
@ -11969,7 +11975,8 @@ class TestMCPToolsListAuthSurfacing:
)
upstream_challenge = 'Bearer resource_metadata="https://upstream.example/.well-known/oauth-protected-resource"'
client = MagicMock()
client.list_tools = AsyncMock(side_effect=_upstream_status_error(401, upstream_challenge))
client.discovery_auth_fingerprint = AsyncMock(return_value="test-credential-hash")
client.list_tools_result = AsyncMock(side_effect=_upstream_status_error(401, upstream_challenge))
manager.create_mcp_client = AsyncMock(return_value=client)
with pytest.raises(MCPUpstreamAuthError) as exc_info:
@ -14316,7 +14323,9 @@ class TestClientForwardedDiscoveryFailureIsNotFatal:
server = await self._registered(manager, auth_type, None)
manager._set_oauth_discovery_deferred(server.server_id, True)
manager._fetch_tools_with_timeout = AsyncMock(
return_value=[MCPTool(name="list_reports", description="d", inputSchema={"type": "object"})]
return_value=ListToolsResult(
tools=[MCPTool(name="list_reports", description="d", inputSchema={"type": "object"})]
)
)
with patch.object(manager, "_discover_oauth_metadata_for_server", new=AsyncMock(return_value=None)):
@ -15224,6 +15233,7 @@ class TestLitellmAdmissionKeyIsNeverTheSubjectToken:
manager: Final = MCPServerManager()
client: Final = AsyncMock()
client.call_tool = AsyncMock(return_value=CallToolResult(content=[], isError=False))
client.discovery_auth_fingerprint = AsyncMock(return_value="test-credential-hash")
client.list_prompts_result = AsyncMock(return_value=ListPromptsResult(prompts=[]))
client.read_resource = AsyncMock(return_value=ReadResourceResult(contents=[]))
manager.create_mcp_client = AsyncMock(return_value=client)
@ -15340,7 +15350,7 @@ class TestLitellmAdmissionKeyIsNeverTheSubjectToken:
@pytest.mark.asyncio
async def test_tools_list_with_only_the_litellm_key_has_no_subject(self):
manager: Final = self._manager_with_recording_client()
manager._fetch_tools_with_timeout = AsyncMock(return_value=[])
manager._fetch_tools_with_timeout = AsyncMock(return_value=ListToolsResult(tools=[]))
await manager._get_tools_from_server(
server=self._token_exchange_server("te-list-key"),
oauth2_headers={"Authorization": f"Bearer {self._ADMISSION_KEY}"},
@ -16446,9 +16456,12 @@ async def test_health_check_logs_upstream_exchange_only_for_http_failures(
class _DiscoveryUpstream:
def __init__(self) -> None:
self.requests: tuple[tuple[str, str], ...] = ()
self.tools: tuple[dict[str, object], ...] = ()
self.tools: tuple[dict[str, object], ...] = (
{"name": "example", "description": "original", "inputSchema": {"type": "object"}},
)
self.outcome = "supported"
self.ttl_ms = 60000
self.last_page_ttl_ms = 60000
self.entered = asyncio.Event()
self.release = asyncio.Event()
self.release.set()
@ -16523,7 +16536,7 @@ class _DiscoveryUpstream:
"result": {
**result,
**continuation,
"ttlMs": self.ttl_ms,
"ttlMs": self.last_page_ttl_ms if (payload.params or {}).get("cursor") else self.ttl_ms,
"cacheScope": "private",
"resultType": "complete",
},
@ -16634,6 +16647,100 @@ async def test_discovery_cache_reuses_raw_results_and_expires(kind: str) -> None
assert upstream.initializes == 2
def _list_discovery_tools(manager: MCPServerManager, cursor: str | None | EllipsisType = ..., **kwargs):
from mcp.types import PaginatedRequestParams
params: Final = None if cursor is ... else PaginatedRequestParams(cursor=cursor)
return manager.get_tools_page(_discovery_server(), params=params, **kwargs)
@pytest.mark.asyncio
@pytest.mark.parametrize("cursor", (..., None), ids=("full_walk", "first_page"))
async def test_tools_list_is_served_from_cache_within_the_upstream_ttl(cursor: str | None | EllipsisType) -> None:
clock: Final = _DiscoveryClock()
manager: Final = MCPServerManager(discovery_clock=clock)
upstream: Final = _DiscoveryUpstream()
upstream.ttl_ms = 30000
with _mcp_upstream(upstream.respond):
first: Final = await _list_discovery_tools(manager, cursor)
assert [tool.name for tool in first.tools] == ["discovery-example"]
assert 29000 < first.ttl_ms <= 30000
clock.now = 29.9
cached: Final = await _list_discovery_tools(manager, cursor, add_prefix=False)
assert [tool.name for tool in cached.tools] == ["example"]
assert 0 < cached.ttl_ms <= 100
assert upstream.initializes == 1
clock.now = 30.001
assert [tool.name for tool in (await _list_discovery_tools(manager, cursor)).tools] == ["discovery-example"]
assert upstream.initializes == 2
@pytest.mark.asyncio
@pytest.mark.parametrize("cursor", (..., None), ids=("full_walk", "first_page"))
async def test_tools_list_is_not_cached_without_an_upstream_ttl(cursor: str | None | EllipsisType) -> None:
manager: Final = MCPServerManager(discovery_clock=_DiscoveryClock())
upstream: Final = _DiscoveryUpstream()
upstream.ttl_ms = 0
with _mcp_upstream(upstream.respond):
for _ in range(2):
assert len((await _list_discovery_tools(manager, cursor)).tools) == 1
assert upstream.initializes == 2
@pytest.mark.asyncio
async def test_tools_list_full_walk_takes_the_shortest_page_ttl_and_keys_pages_by_cursor() -> None:
clock: Final = _DiscoveryClock()
manager: Final = MCPServerManager(discovery_clock=clock)
upstream: Final = _DiscoveryUpstream()
upstream.outcome = "paged"
upstream.last_page_ttl_ms = 20000
with _mcp_upstream(upstream.respond):
walk: Final = await _list_discovery_tools(manager)
assert len(walk.tools) == 2
assert 19000 < walk.ttl_ms <= 20000
clock.now = 20.001
assert len((await _list_discovery_tools(manager)).tools) == 2
assert upstream.initializes == 2
first_page: Final = await _list_discovery_tools(manager, None)
last_page: Final = await _list_discovery_tools(manager, "last-page")
assert first_page.next_cursor == "last-page"
assert last_page.next_cursor is None
assert upstream.initializes == 4
assert (await _list_discovery_tools(manager, "last-page")).next_cursor is None
assert (await _list_discovery_tools(manager, None)).next_cursor == "last-page"
assert upstream.initializes == 4
@pytest.mark.asyncio
async def test_tools_list_cache_isolates_callers_and_is_invalidated_with_the_server() -> None:
manager: Final = MCPServerManager(discovery_clock=_DiscoveryClock())
upstream: Final = _DiscoveryUpstream()
with _mcp_upstream(upstream.respond):
for credential in ("first-secret", "second-secret", "first-secret"):
assert len((await _list_discovery_tools(manager, extra_headers={"Authorization": credential})).tools) == 1
assert upstream.initializes == 2
assert {auth for method, auth in upstream.requests if method == "tools/list"} == {
"first-secret",
"second-secret",
}
manager.invalidate_discovery_lists("discovery")
assert len((await _list_discovery_tools(manager, extra_headers={"Authorization": "first-secret"})).tools) == 1
assert upstream.initializes == 3
@pytest.mark.asyncio
async def test_tools_list_failures_are_not_cached() -> None:
manager: Final = MCPServerManager(discovery_clock=_DiscoveryClock())
upstream: Final = _DiscoveryUpstream()
upstream.outcome = "failure"
with _mcp_upstream(upstream.respond):
with pytest.raises(MCPServerListError):
await _list_discovery_tools(manager)
upstream.outcome = "supported"
assert len((await _list_discovery_tools(manager)).tools) == 1
assert upstream.initializes == 2
@pytest.mark.asyncio
@pytest.mark.parametrize("kind", ("prompts", "resources", "templates"))
@pytest.mark.parametrize("outcome", ("unsupported", "rejected", "failure"))
@ -19917,8 +20024,8 @@ def catalog_guardrail(monkeypatch):
def _catalog_manager(*upstream_tools: MCPTool) -> MCPServerManager:
manager = MCPServerManager()
manager.create_mcp_client = AsyncMock(return_value=object())
manager._fetch_tools_with_timeout = AsyncMock(return_value=list(upstream_tools))
manager.create_mcp_client = AsyncMock(return_value=_listing_client())
manager._fetch_tools_with_timeout = AsyncMock(return_value=ListToolsResult(tools=list(upstream_tools)))
return manager
@ -20078,14 +20185,14 @@ class TestToolCatalogGuard:
await manager._get_tools_from_server(_notes_server(), add_prefix=False, proxy_logging_obj=proxy_logging_obj)
assert send_alert.await_count == 1
manager._fetch_tools_with_timeout = AsyncMock(return_value=[LIST_NOTES])
manager._fetch_tools_with_timeout = AsyncMock(return_value=ListToolsResult(tools=[LIST_NOTES]))
recovered = await manager._get_tools_from_server(
_notes_server(), add_prefix=False, proxy_logging_obj=proxy_logging_obj
)
assert [tool.name for tool in recovered] == ["list_notes"]
assert send_alert.await_count == 1
manager._fetch_tools_with_timeout = AsyncMock(return_value=[LIST_NOTES, POISONED_DELETE])
manager._fetch_tools_with_timeout = AsyncMock(return_value=ListToolsResult(tools=[LIST_NOTES, POISONED_DELETE]))
await manager._get_tools_from_server(_notes_server(), add_prefix=False, proxy_logging_obj=proxy_logging_obj)
assert send_alert.await_count == 2
@ -20291,12 +20398,12 @@ class TestToolCatalogGuard:
)
while send_alert.await_count == 0:
await asyncio.sleep(0)
manager._fetch_tools_with_timeout = AsyncMock(return_value=[LIST_NOTES])
manager._fetch_tools_with_timeout = AsyncMock(return_value=ListToolsResult(tools=[LIST_NOTES]))
await manager._get_tools_from_server(_notes_server(), add_prefix=False, proxy_logging_obj=proxy_logging_obj)
gate.set()
await poisoned_listing
manager._fetch_tools_with_timeout = AsyncMock(return_value=[LIST_NOTES, POISONED_DELETE])
manager._fetch_tools_with_timeout = AsyncMock(return_value=ListToolsResult(tools=[LIST_NOTES, POISONED_DELETE]))
await manager._get_tools_from_server(_notes_server(), add_prefix=False, proxy_logging_obj=proxy_logging_obj)
assert send_alert.await_count == 2
@ -20919,6 +21026,7 @@ async def test_catalog_page_registers_bare_routes_only_for_complete_initial_disc
manager.registry = {server.server_id: server, other.server_id: other}
manager._create_prefixed_tools([LIST_NOTES], other)
manager.create_mcp_client.return_value = SimpleNamespace(
discovery_auth_fingerprint=AsyncMock(return_value="test-credential-hash"),
list_tools_page=AsyncMock(return_value=ListToolsResult(tools=[LIST_NOTES], next_cursor=next_cursor))
)
@ -20944,7 +21052,7 @@ async def test_paginated_listing_keeps_earlier_tool_metadata_and_caller_isolatio
server = MCPServer(server_id="pages", name="pages", transport=MCPTransport.http)
manager = MCPServerManager()
manager.registry = {server.server_id: server}
client = AsyncMock()
client = _listing_client()
client._last_initialize_instructions = None
client.list_tools_page.side_effect = [
ListToolsResult(tools=[first], next_cursor="next"),
@ -21102,7 +21210,7 @@ async def test_failed_aggregate_continuation_preserves_only_delivered_tool_metad
servers = [MCPServer(server_id=name, name=name, transport=MCPTransport.http) for name in ("alpha", "beta")]
manager = MCPServerManager()
manager.registry = {server.server_id: server for server in servers}
clients = {server.server_id: AsyncMock() for server in servers}
clients = {server.server_id: _listing_client() for server in servers}
for client in clients.values():
client._last_initialize_instructions = None
client.list_tools_page.side_effect = [
@ -21146,7 +21254,7 @@ async def test_aggregate_publishes_complete_bare_routes_only_after_delivering_a_
servers = [MCPServer(server_id=name, name=name, transport=MCPTransport.http) for name in ("alpha", "beta")]
manager = MCPServerManager()
manager.registry = {server.server_id: server for server in servers}
clients = {server.server_id: AsyncMock() for server in servers}
clients = {server.server_id: _listing_client() for server in servers}
for name, client in clients.items():
client._last_initialize_instructions = None
client.list_tools_page.return_value = ListToolsResult(
@ -21481,55 +21589,55 @@ class TestMCPUpstreamRateLimitRetry:
manager: Final = MCPServerManager()
tool: Final = MCPTool(name="ping", inputSchema={"type": "object"})
client: Final = MagicMock()
client.list_tools = AsyncMock(
client.list_tools_result = AsyncMock(
side_effect=[
_upstream_http_status_error(429, {"Retry-After": "0"}),
_upstream_http_status_error(429, {"Retry-After": "0"}),
[tool],
ListToolsResult(tools=[tool]),
]
)
tools: Final = await manager._fetch_tools_with_timeout(client, "flaky-server")
listing: Final = await manager._fetch_tools_with_timeout(client, "flaky-server")
assert tools == [tool]
assert client.list_tools.await_count == 3
assert listing.tools == [tool]
assert client.list_tools_result.await_count == 3
@pytest.mark.asyncio
async def test_fetch_tools_with_timeout_gives_up_after_max_retries(self) -> None:
manager: Final = MCPServerManager()
client: Final = MagicMock()
client.list_tools = AsyncMock(side_effect=_upstream_http_status_error(429, {"Retry-After": "0"}))
client.list_tools_result = AsyncMock(side_effect=_upstream_http_status_error(429, {"Retry-After": "0"}))
expected_error: Final = pytest.raises(MCPServerListError)
with expected_error as exc_info:
await manager._fetch_tools_with_timeout(client, "saturated-server")
assert exc_info.value.fault == ServerListFault(tag="upstream_error", status_code=429)
assert client.list_tools.await_count == 1 + MCP_UPSTREAM_RATE_LIMIT_MAX_RETRIES
assert client.list_tools_result.await_count == 1 + MCP_UPSTREAM_RATE_LIMIT_MAX_RETRIES
@pytest.mark.asyncio
async def test_fetch_tools_with_timeout_does_not_retry_auth_error(self) -> None:
manager: Final = MCPServerManager()
client: Final = MagicMock()
client.list_tools = AsyncMock(side_effect=_upstream_http_status_error(401))
client.list_tools_result = AsyncMock(side_effect=_upstream_http_status_error(401))
with pytest.raises(MCPUpstreamAuthError):
await manager._fetch_tools_with_timeout(client, "srv")
assert client.list_tools.await_count == 1
assert client.list_tools_result.await_count == 1
@pytest.mark.asyncio
async def test_fetch_tools_with_timeout_does_not_retry_server_error(self) -> None:
manager: Final = MCPServerManager()
client: Final = MagicMock()
client.list_tools = AsyncMock(side_effect=_upstream_http_status_error(500))
client.list_tools_result = AsyncMock(side_effect=_upstream_http_status_error(500))
expected_error: Final = pytest.raises(MCPServerListError)
with expected_error as exc_info:
await manager._fetch_tools_with_timeout(client, "srv")
assert exc_info.value.fault == ServerListFault(tag="upstream_error", status_code=500)
assert client.list_tools.await_count == 1
assert client.list_tools_result.await_count == 1
@pytest.mark.asyncio
async def test_fetch_tools_with_timeout_reports_429_when_backoff_would_pass_listing_timeout(
@ -21540,7 +21648,7 @@ class TestMCPUpstreamRateLimitRetry:
)
manager: Final = MCPServerManager()
client: Final = MagicMock()
client.list_tools = AsyncMock(side_effect=_upstream_http_status_error(429, {"Retry-After": "5"}))
client.list_tools_result = AsyncMock(side_effect=_upstream_http_status_error(429, {"Retry-After": "5"}))
with anyio.fail_after(3):
expected_error: Final = pytest.raises(MCPServerListError)
@ -21548,7 +21656,7 @@ class TestMCPUpstreamRateLimitRetry:
await manager._fetch_tools_with_timeout(client, "srv")
assert exc_info.value.fault == ServerListFault(tag="upstream_error", status_code=429)
assert client.list_tools.await_count == 1
assert client.list_tools_result.await_count == 1
@pytest.mark.asyncio
async def test_get_tools_page_retries_429_then_returns_page(self) -> None:
@ -21559,6 +21667,7 @@ class TestMCPUpstreamRateLimitRetry:
manager.create_mcp_client = AsyncMock(
return_value=SimpleNamespace(
_last_initialize_instructions=None,
discovery_auth_fingerprint=AsyncMock(return_value="test-credential-hash"),
list_tools_page=AsyncMock(
side_effect=[
_upstream_http_status_error(429, {"Retry-After": "0"}),
@ -21596,7 +21705,11 @@ class TestMCPUpstreamRateLimitRetry:
raise AssertionError("unreachable")
list_tools_page: Final = AsyncMock(side_effect=_list_tools_page)
client: Final = SimpleNamespace(_last_initialize_instructions=None, list_tools_page=list_tools_page)
client: Final = SimpleNamespace(
_last_initialize_instructions=None,
discovery_auth_fingerprint=AsyncMock(return_value="test-credential-hash"),
list_tools_page=list_tools_page,
)
manager.create_mcp_client = AsyncMock(return_value=client)
with anyio.fail_after(5):

View file

@ -93,6 +93,12 @@ def cleanup_mcp_global_state():
yield
def _listing_client() -> AsyncMock:
client = AsyncMock()
client.discovery_auth_fingerprint = AsyncMock(return_value="test-credential-hash")
return client
def _call_tool_params(name, arguments=None):
from mcp.types import CallToolRequestParams
@ -7722,13 +7728,13 @@ def _worker_that_never_listed(server: MCPServer, upstream_tools: tuple[str, ...]
return CallToolResult(content=[TextContent(type="text", text="ok")], is_error=False)
async def fake_fetch_tools(client, server_name):
return [MCPTool(name=tool_name, inputSchema={}) for tool_name in upstream_tools]
return ListToolsResult(tools=[MCPTool(name=tool_name, inputSchema={}) for tool_name in upstream_tools])
with (
patch.object( # test-quality-ok: the upstream MCP session is the boundary; a real one needs an initialize handshake over a live server
mcp_operations.global_mcp_server_manager,
"create_mcp_client",
new=AsyncMock(return_value=MagicMock()),
new=AsyncMock(return_value=_listing_client()),
) as create_client,
patch.object( # test-quality-ok: same boundary, this is the tools/list answer the upstream would give
mcp_operations.global_mcp_server_manager,
@ -8575,7 +8581,7 @@ async def test_execute_mcp_tool_implicit_listing_before_the_first_call_hands_hoo
server = _never_listed_passthrough_server()
manager.registry[server.server_id] = server
manager._listed_tools_by_server_id.pop(server.server_id, None)
upstream = AsyncMock()
upstream = _listing_client()
upstream.call_tool.return_value = CallToolResult(content=[TextContent(type="text", text="ok")], isError=False)
proxy_logging = _mock_mcp_proxy_logging()
proxy_logging.create_mcp_request_object_from_kwargs = MagicMock(return_value={})
@ -8583,7 +8589,9 @@ async def test_execute_mcp_tool_implicit_listing_before_the_first_call_hands_hoo
proxy_logging.pre_call_hook = AsyncMock(return_value={})
proxy_logging.during_call_hook = AsyncMock(return_value=None)
fetch_tools = AsyncMock(
return_value=[MCPTool(name="add", description="Adds. FLAGWORD", inputSchema={"type": "object"})]
return_value=ListToolsResult(
tools=[MCPTool(name="add", description="Adds. FLAGWORD", inputSchema={"type": "object"})]
)
)
with (
@ -8630,11 +8638,13 @@ async def test_fetch_pinnable_tool_catalog_records_no_listed_catalog_for_the_adm
request.client.host = "10.1.2.3"
request.headers = {"x-litellm-api-key": "sk-admin"}
fetch_tools = AsyncMock(
return_value=[MCPTool(name="add", description="Upstream wording", inputSchema={"type": "object"})]
return_value=ListToolsResult(
tools=[MCPTool(name="add", description="Upstream wording", inputSchema={"type": "object"})]
)
)
with (
patch.object(manager, "create_mcp_client", new=AsyncMock(return_value=MagicMock())),
patch.object(manager, "create_mcp_client", new=AsyncMock(return_value=_listing_client())),
patch.object(manager, "_fetch_tools_with_timeout", new=fetch_tools),
patch("litellm.proxy.proxy_server.proxy_logging_obj", ProxyLogging(user_api_key_cache=DualCache())),
):

View file

@ -21,7 +21,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from pydantic import JsonValue
from mcp.types import Tool
from mcp.types import ListToolsResult, Tool
import litellm
from litellm.models.object_permission import LiteLLM_ObjectPermissionTable
@ -68,6 +68,10 @@ SAMPLE_TOOLS = _make_tools(
)
def _listing_client() -> SimpleNamespace:
return SimpleNamespace(discovery_auth_fingerprint=AsyncMock(return_value="test-credential-hash"))
@pytest.mark.parametrize(
("schema", "arguments", "error"),
(
@ -1416,8 +1420,8 @@ async def test_mcp_tool_search_leaves_the_listed_tools_slot_empty(monkeypatch: p
with (
patch.dict(manager.tool_name_to_mcp_server_name_mapping),
patch.object(mcp_operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[server])),
patch.object(manager, "create_mcp_client", AsyncMock(return_value=object())),
patch.object(manager, "_fetch_tools_with_timeout", AsyncMock(return_value=upstream)),
patch.object(manager, "create_mcp_client", AsyncMock(return_value=_listing_client())),
patch.object(manager, "_fetch_tools_with_timeout", AsyncMock(return_value=ListToolsResult(tools=upstream))),
):
try:
result = await handle_mcp_tool_search(query="echo", top_k=1, user_api_key_dict=user)

View file

@ -1,11 +1,12 @@
import asyncio
from collections.abc import Callable, Sequence
from types import SimpleNamespace
from typing import Final, Literal
from unittest.mock import AsyncMock, patch
import pytest
from mcp.server.context import ServerRequestContext
from mcp.types import GetPromptRequest, GetPromptRequestParams, GetPromptResult
from mcp.types import GetPromptRequest, GetPromptRequestParams, GetPromptResult, ListToolsResult
from mcp.types import Tool as MCPTool
import litellm
@ -44,6 +45,10 @@ async def _served_catalog_tool() -> str:
return "ok"
def _listing_client() -> SimpleNamespace:
return SimpleNamespace(discovery_auth_fingerprint=AsyncMock(return_value="test-credential-hash"))
@pytest.mark.parametrize(
("settings", "mcp_servers", "toolset_id", "auth_state", "raises"),
[
@ -1597,8 +1602,8 @@ async def test_list_mcp_tools_records_the_catalog_only_when_asked(
upstream = [MCPTool(name="echo", description="Echo text back", inputSchema={"type": "object"})]
with (
patch.object(operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[server])),
patch.object(manager, "create_mcp_client", AsyncMock(return_value=object())),
patch.object(manager, "_fetch_tools_with_timeout", AsyncMock(return_value=upstream)),
patch.object(manager, "create_mcp_client", AsyncMock(return_value=_listing_client())),
patch.object(manager, "_fetch_tools_with_timeout", AsyncMock(return_value=ListToolsResult(tools=upstream))),
patch.dict(manager.tool_name_to_mcp_server_name_mapping),
):
try:

View file

@ -1101,7 +1101,10 @@ async def test_aresponses_mcp_call_does_not_forward_admission_authorization(
monkeypatch.setattr(manager, "registry", {server.server_id: server})
mcp_client: Final = AsyncMock()
mcp_client.list_tools.return_value = [MCPTool(name="whoami", inputSchema={"type": "object", "properties": {}})]
mcp_client.discovery_auth_fingerprint.return_value = "test-credential-hash"
mcp_client.list_tools_result.return_value = ListToolsResult(
tools=[MCPTool(name="whoami", inputSchema={"type": "object", "properties": {}})]
)
mcp_client.call_tool.return_value = CallToolResult(
content=[TextContent(type="text", text="ok")],
isError=False,

View file

@ -10,7 +10,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from fastapi import HTTPException
from mcp.types import CallToolResult, TextContent
from mcp.types import CallToolResult, ListToolsResult, TextContent
from mcp.types import Tool as MCPTool
from openai.types.responses.tool_param import Mcp
@ -70,6 +70,10 @@ def _setup_proxy_logging(monkeypatch: pytest.MonkeyPatch) -> AsyncMock:
return proxy_logging_obj.post_call_failure_hook
def _listing_client() -> types.SimpleNamespace:
return types.SimpleNamespace(discovery_auth_fingerprint=AsyncMock(return_value="test-credential-hash"))
def test_deduplicate_mcp_tools_single_allowed_server():
tools = [{"name": "search"}, {"name": "search"}] # duplicate on purpose
@ -1371,8 +1375,8 @@ async def test_bridge_listing_leaves_the_callers_catalog_unchanged(
with (
patch.dict(manager.tool_name_to_mcp_server_name_mapping),
patch.object(mcp_operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[server])),
patch.object(manager, "create_mcp_client", AsyncMock(return_value=object())),
patch.object(manager, "_fetch_tools_with_timeout", AsyncMock(return_value=upstream)),
patch.object(manager, "create_mcp_client", AsyncMock(return_value=_listing_client())),
patch.object(manager, "_fetch_tools_with_timeout", AsyncMock(return_value=ListToolsResult(tools=upstream))),
):
try:
if real_listing:
@ -1444,9 +1448,10 @@ async def test_concurrent_bridge_calls_use_their_own_served_metadata(monkeypatch
MCPTool(name="status", description="Read status", inputSchema={"type": "object"}),
]
client: Final = AsyncMock()
client.discovery_auth_fingerprint = AsyncMock(return_value="test-credential-hash")
client.call_tool.return_value = CallToolResult(content=[TextContent(type="text", text="ok")])
manager.create_mcp_client = AsyncMock(return_value=client)
manager._fetch_tools_with_timeout = AsyncMock(return_value=upstream)
manager._fetch_tools_with_timeout = AsyncMock(return_value=ListToolsResult(tools=upstream))
guardrail: Final = _BridgeMetadataGuardrail()
logger: Final = ProxyLogging(user_api_key_cache=DualCache())
monkeypatch.setattr(litellm, "callbacks", [guardrail])