diff --git a/litellm/experimental_mcp_client/client.py b/litellm/experimental_mcp_client/client.py index 79887e4ab99..848f6baf780 100644 --- a/litellm/experimental_mcp_client/client.py +++ b/litellm/experimental_mcp_client/client.py @@ -100,7 +100,7 @@ from litellm.constants import ( MCP_TOOL_LISTING_MAX_PAGES, MCP_TOOL_LISTING_TIMEOUT, ) -from litellm.experimental_mcp_client.tools import list_tools_with_pagination +from litellm.experimental_mcp_client.tools import list_tools_result_with_pagination, list_tools_with_pagination from litellm.llms.custom_httpx.http_handler import get_ssl_configuration from litellm.proxy._experimental.mcp_server.mcp_context import get_active_mcp_request_ctx from litellm.proxy._experimental.mcp_server.mcp_debug import capture_upstream_error_response @@ -1057,6 +1057,9 @@ class MCPClient: return await self.run_with_session(lambda session: session.list_tools(params=params), quiet_on_error=True) async def list_tools(self, raise_on_error: bool = False) -> list[MCPTool]: + return (await self.list_tools_result(raise_on_error=raise_on_error)).tools + + async def list_tools_result(self, raise_on_error: bool = False) -> ListToolsResult: """List available tools from the server. Args: @@ -1071,16 +1074,20 @@ class MCPClient: try: # A per-server timeout above the global default extends the whole-walk deadline listing_deadline: Final = max(self.timeout, MCP_TOOL_LISTING_TIMEOUT) - tools: Final = await self.run_with_session( - partial(list_tools_with_pagination, listing_deadline=listing_deadline, require_complete=raise_on_error), + listing: Final = await self.run_with_session( + partial( + list_tools_result_with_pagination, + listing_deadline=listing_deadline, + require_complete=raise_on_error, + ), quiet_on_error=raise_on_error, ) - tool_count: Final = len(tools) - tool_names: Final = tuple(tool.name for tool in tools) + tool_count: Final = len(listing.tools) + tool_names: Final = tuple(tool.name for tool in listing.tools) verbose_logger.info( "MCP client listed %s tools from %s: %s", tool_count, self.server_url or "stdio", tool_names ) - return tools + return listing except asyncio.CancelledError: verbose_logger.warning("MCP client list_tools was cancelled") raise @@ -1110,7 +1117,7 @@ class MCPClient: if raise_on_error: raise # Return empty list instead of raising to allow graceful degradation - return [] + return ListToolsResult(tools=[]) @staticmethod def error_tool_result(exc: Exception) -> MCPCallToolResult: diff --git a/litellm/experimental_mcp_client/tools.py b/litellm/experimental_mcp_client/tools.py index 8015b065899..709e2df492c 100644 --- a/litellm/experimental_mcp_client/tools.py +++ b/litellm/experimental_mcp_client/tools.py @@ -1,9 +1,11 @@ import json +import time +from collections.abc import Sequence from typing import Final, Literal import anyio from mcp import ClientSession, MCPError -from mcp.types import INTERNAL_ERROR, PaginatedRequestParams +from mcp.types import INTERNAL_ERROR, ListToolsResult, PaginatedRequestParams from mcp.types import CallToolRequestParams as MCPCallToolRequestParams from mcp.types import CallToolResult as MCPCallToolResult from mcp.types import Tool as MCPTool @@ -17,6 +19,7 @@ from litellm.constants import ( MCP_TOOL_LISTING_MAX_PAGES, MCP_TOOL_LISTING_TIMEOUT, ) +from litellm.proxy._experimental.mcp_server.result_conversion import age_freshness, aggregate_freshness from litellm.types.llms.anthropic import AnthropicMessagesTool from litellm.types.utils import ChatCompletionMessageToolCall @@ -98,9 +101,21 @@ def transform_mcp_tool_to_anthropic_tool(mcp_tool: MCPTool) -> AnthropicMessages ) +def _complete_listing(tools: Sequence[MCPTool], pages: Sequence[tuple[ListToolsResult, float]]) -> ListToolsResult: + now: Final = time.monotonic() + freshness: Final = aggregate_freshness(tuple(age_freshness(page, now - received) for page, received in pages)) + return ListToolsResult(tools=list(tools), ttl_ms=freshness.ttl_ms, cache_scope=freshness.cache_scope) + + async def list_tools_with_pagination( session: ClientSession, listing_deadline: float | None = None, *, require_complete: bool = False ) -> list[MCPTool]: # mutable-ok: list return contract + return (await list_tools_result_with_pagination(session, listing_deadline, require_complete=require_complete)).tools + + +async def list_tools_result_with_pagination( + session: ClientSession, listing_deadline: float | None = None, *, require_complete: bool = False +) -> ListToolsResult: """Collect tools from every tools/list page by following nextCursor. Stops and returns the tools collected so far when the upstream repeats a @@ -108,8 +123,10 @@ async def list_tools_with_pagination( buggy or slow upstream yields a partial catalog instead of an error. listing_deadline overrides the default whole-walk deadline; callers with a per-server timeout above the global default pass it through here. + Only a complete walk carries the upstream ttlMs, so a partial catalog is never cached. """ tools: Final[list[MCPTool]] = [] # mutable-ok: accumulates each page's tools + pages: Final[list[tuple[ListToolsResult, float]]] = [] # mutable-ok: bounded pagination evidence seen_cursors: Final[set[str]] = set() # mutable-ok: guards against cursor loops cursor: str | None = None # rebind-ok: advances to each page's nextCursor # The per-request session read timeout restarts on every page, so a multi-page @@ -128,10 +145,11 @@ async def list_tools_with_pagination( else await session.list_tools(params=PaginatedRequestParams(cursor=cursor)) ) tools.extend(result.tools) + pages.append((result, time.monotonic())) next_cursor = result.next_cursor if not isinstance(next_cursor, str) or not next_cursor: - return tools + return _complete_listing(tools, pages) if next_cursor in seen_cursors: verbose_logger.warning( "MCP server repeated a tools/list cursor while listing tools; returning %s tools collected so far", @@ -141,7 +159,7 @@ async def list_tools_with_pagination( raise MCPError( code=INTERNAL_ERROR, message="Upstream tool discovery is incomplete: repeated cursor" ) - return tools + return ListToolsResult(tools=tools) seen_cursors.add(next_cursor) cursor = next_cursor @@ -152,7 +170,7 @@ async def list_tools_with_pagination( ) if require_complete: raise MCPError(code=INTERNAL_ERROR, message="Upstream tool discovery is incomplete: pagination limit") - return tools + return ListToolsResult(tools=tools) verbose_logger.warning( "MCP server tools/list pagination exceeded the %s second listing deadline; returning %s tools collected so far", @@ -161,7 +179,7 @@ async def list_tools_with_pagination( ) if require_complete: raise MCPError(code=INTERNAL_ERROR, message="Upstream tool discovery is incomplete: listing deadline") - return tools + return ListToolsResult(tools=tools) async def load_mcp_tools( diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 7eddf592262..52b475fa78f 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -26,7 +26,7 @@ from collections.abc import ( ) from contextlib import AbstractAsyncContextManager, ExitStack, asynccontextmanager, nullcontext from dataclasses import dataclass, replace -from functools import lru_cache +from functools import lru_cache, partial from itertools import chain, groupby from types import EllipsisType, MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypedDict, cast @@ -2031,6 +2031,9 @@ class MCPServerManager: self._template_discovery_cache = _DiscoveryCache[ListResourceTemplatesResult]( discovery_ttl, discovery_clock, TypeAdapter(ListResourceTemplatesResult) ) + self._tool_discovery_cache = _DiscoveryCache[ListToolsResult]( + discovery_ttl, discovery_clock, TypeAdapter(ListToolsResult) + ) from litellm.proxy._experimental.mcp_server.catalog import CatalogSnapshots self.catalog = CatalogSnapshots(self) @@ -4576,14 +4579,27 @@ class MCPServerManager: tools=[t.model_copy(update={"name": registered_names[t.name]}) for t in guarded_openapi] ) else: - page: Final = ( - await retry_upstream_rate_limit( - lambda: self._fetch_tools_page_with_timeout(client, server, params), + key: Final = self._discovery_key( + server, + user_api_key_auth, + mcp_auth_header, + extra_headers, + stdio_env, + subject_token, + await client.discovery_auth_fingerprint(), + raw_headers=raw_headers, + page_cursor=None if params is None else (params.cursor,), + ) + page: Final = await self._tool_discovery_cache.get( + key, + partial(self._fetch_tools_with_timeout, client, server.name) + if params is None + else partial( + retry_upstream_rate_limit, + partial(self._fetch_tools_page_with_timeout, client, server, params), server.name, deadline_seconds=MCP_TOOL_LISTING_TIMEOUT, - ) - if params is not None - else ListToolsResult(tools=await self._fetch_tools_with_timeout(client, server.name)) + ), ) tools = page.tools self._remember_upstream_initialize_instructions(server, client) @@ -4654,6 +4670,7 @@ class MCPServerManager: self._prompt_discovery_cache.invalidate(server_id) self._resource_discovery_cache.invalidate(server_id) self._template_discovery_cache.invalidate(server_id) + self._tool_discovery_cache.invalidate(server_id) def invalidate_server_definition_caches(self, server_id: str) -> None: from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( # noqa: PLC0415 # lazy: discoverable_endpoints lazily imports this module's manager singleton @@ -4755,6 +4772,7 @@ class MCPServerManager: subject_token: str | None, credential_fingerprint: str | None = None, raw_headers: Mapping[str, str] | None = None, + page_cursor: tuple[str | None] | None = None, ) -> _DiscoveryKey: identity: Final = ( user_api_key_auth.model_dump( @@ -4781,6 +4799,7 @@ class MCPServerManager: stdio_env, subject_token, credential_fingerprint, + page_cursor, ), sort_keys=True, separators=(",", ":"), @@ -5622,7 +5641,7 @@ class MCPServerManager: self, client: MCPClient, server_name: str, - ) -> list[MCPTool]: + ) -> ListToolsResult: """ Fetch tools from MCP client with timeout and error handling. @@ -5646,17 +5665,17 @@ class MCPServerManager: server_name: Name of the server for logging Returns: - List of tools from the server + The tools from the server with the upstream freshness of the complete walk """ try: with anyio.fail_after(MCP_TOOL_LISTING_TIMEOUT): - tools: Final = await retry_upstream_rate_limit( - lambda: client.list_tools(raise_on_error=True), + listing: Final = await retry_upstream_rate_limit( + lambda: client.list_tools_result(raise_on_error=True), server_name, deadline_seconds=MCP_TOOL_LISTING_TIMEOUT, ) - verbose_logger.debug("Tools from %s: %s", server_name, tools) - return tools + verbose_logger.debug("Tools from %s: %s", server_name, listing.tools) + return listing except TimeoutError as e: verbose_logger.warning("Timeout while listing tools from %s", server_name) raise MCPServerListError(ServerListFault(tag="timeout"), server_name) from e diff --git a/tests/unit/experimental_mcp_client/test_tools.py b/tests/unit/experimental_mcp_client/test_tools.py index ac7afe4c74b..06f6d4c2b9c 100644 --- a/tests/unit/experimental_mcp_client/test_tools.py +++ b/tests/unit/experimental_mcp_client/test_tools.py @@ -14,6 +14,7 @@ from mcp.types import ( from mcp.types import Tool as MCPTool from litellm.experimental_mcp_client.tools import ( + list_tools_result_with_pagination, list_tools_with_pagination, transform_mcp_tool_to_anthropic_tool, _get_function_arguments, @@ -171,6 +172,22 @@ async def test_pagination_walk_stops_on_repeated_cursor(mock_session): assert mock_session.list_tools.call_count == 2 +@pytest.mark.asyncio() +@pytest.mark.parametrize("last_cursor", [None, "page-1"]) +async def test_pagination_walk_keeps_the_shortest_ttl_only_for_a_complete_walk(mock_session, last_cursor): + mock_session.list_tools.side_effect = [ + ListToolsResult(tools=[MCPTool(name="tool_0", inputSchema={})], nextCursor="page-1", ttl_ms=30000), + ListToolsResult(tools=[MCPTool(name="tool_1", inputSchema={})], nextCursor=last_cursor, ttl_ms=20000), + ] + result = await list_tools_result_with_pagination(mock_session) + + assert [tool.name for tool in result.tools] == ["tool_0", "tool_1"] + if last_cursor is None: + assert 19000 < result.ttl_ms <= 20000 + else: + assert result.ttl_ms == 0 + + @pytest.mark.asyncio() async def test_pagination_walk_treats_empty_cursor_as_terminal(mock_session): mock_session.list_tools.side_effect = [ diff --git a/tests/unit/proxy/_experimental/mcp_server/faults/test_list_outcomes.py b/tests/unit/proxy/_experimental/mcp_server/faults/test_list_outcomes.py index f6902c778e5..bdbdd5ecbcb 100644 --- a/tests/unit/proxy/_experimental/mcp_server/faults/test_list_outcomes.py +++ b/tests/unit/proxy/_experimental/mcp_server/faults/test_list_outcomes.py @@ -145,12 +145,12 @@ async def test_cancelled_fetch_propagates_instead_of_masquerading_as_a_healthy_e client = MagicMock() started = asyncio.Event() - async def list_tools(raise_on_error: bool = False) -> list[object]: + async def list_tools_result(raise_on_error: bool = False) -> list[object]: started.set() await asyncio.Event().wait() return [] - client.list_tools = list_tools + client.list_tools_result = list_tools_result listing = asyncio.create_task(manager._fetch_tools_with_timeout(client, "cancelled_srv")) await started.wait() listing.cancel(cancel_message) @@ -172,7 +172,7 @@ async def test_client_side_cancelled_error_is_a_classified_fault_not_a_cancellat manager = MCPServerManager() client = MagicMock() - client.list_tools = AsyncMock(side_effect=asyncio.CancelledError(SESSION_CLEANUP_TIMEOUT_MESSAGE)) + client.list_tools_result = AsyncMock(side_effect=asyncio.CancelledError(SESSION_CLEANUP_TIMEOUT_MESSAGE)) with pytest.raises(MCPServerListError) as exc_info: await manager._fetch_tools_with_timeout(client, "slow_teardown_srv") diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_client_unit.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_client_unit.py index a270ea16a39..999d85920df 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_client_unit.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_client_unit.py @@ -186,9 +186,7 @@ class TestMCPClientUnitTests: }, ) ] - mock_result = MagicMock() - mock_result.tools = mock_tools - mock_session_instance.list_tools.return_value = mock_result + mock_session_instance.list_tools.return_value = ListToolsResult(tools=mock_tools) client = MCPClient("http://example.com") result = await client.list_tools() diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py index b0d53f9c93b..bd5b93edb56 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py @@ -9,7 +9,8 @@ from unittest.mock import AsyncMock, MagicMock import httpx import pytest - +from mcp.types import ListToolsResult +from mcp.types import Tool as MCPTool if sys.version_info < (3, 11): from exceptiongroup import ExceptionGroup @@ -114,7 +115,7 @@ async def test_fetch_tools_from_passthrough_raises_on_upstream_401(): ) mock_client = MagicMock() - mock_client.list_tools = AsyncMock(side_effect=upstream_error) + mock_client.list_tools_result = AsyncMock(side_effect=upstream_error) with pytest.raises(MCPUpstreamAuthError) as exc_info: await manager._fetch_tools_with_timeout(mock_client, passthrough_server.name) @@ -124,7 +125,7 @@ async def test_fetch_tools_from_passthrough_raises_on_upstream_401(): 'Bearer resource_metadata="https://upstream"' ) assert exc_info.value.server_name == "sample_docs" - mock_client.list_tools.assert_awaited_with(raise_on_error=True) + mock_client.list_tools_result.assert_awaited_with(raise_on_error=True) @pytest.mark.asyncio @@ -149,7 +150,7 @@ async def test_fetch_tools_from_delegated_oauth2_raises_on_upstream_401(): ) mock_client = MagicMock() - mock_client.list_tools = AsyncMock(side_effect=upstream_error) + mock_client.list_tools_result = AsyncMock(side_effect=upstream_error) with pytest.raises(MCPUpstreamAuthError) as exc_info: await manager._fetch_tools_with_timeout(mock_client, delegated_server.name) @@ -159,7 +160,7 @@ async def test_fetch_tools_from_delegated_oauth2_raises_on_upstream_401(): 'Bearer resource_metadata="https://upstream"' ) assert exc_info.value.server_name == "delegated_docs" - mock_client.list_tools.assert_awaited_with(raise_on_error=True) + mock_client.list_tools_result.assert_awaited_with(raise_on_error=True) @pytest.mark.asyncio @@ -188,14 +189,14 @@ async def test_fetch_tools_from_client_credentials_oauth2_surfaces_upstream_401( ) mock_client = MagicMock() - mock_client.list_tools = AsyncMock(side_effect=upstream_error) + mock_client.list_tools_result = AsyncMock(side_effect=upstream_error) with pytest.raises(MCPUpstreamAuthError) as exc_info: await manager._fetch_tools_with_timeout(mock_client, m2m_server.name) assert exc_info.value.status_code == 401 assert exc_info.value.server_name == "m2m_docs" - mock_client.list_tools.assert_awaited_with(raise_on_error=True) + mock_client.list_tools_result.assert_awaited_with(raise_on_error=True) @pytest.mark.asyncio @@ -211,13 +212,12 @@ async def test_fetch_tools_from_passthrough_returns_tools_on_success(): oauth_passthrough=True, ) - tool = MagicMock() - tool.name = "list_documents" + tool = MCPTool(name="list_documents", inputSchema={"type": "object"}) mock_client = MagicMock() - mock_client.list_tools = AsyncMock(return_value=[tool]) + mock_client.list_tools_result = AsyncMock(return_value=ListToolsResult(tools=[tool])) - tools = await manager._fetch_tools_with_timeout(mock_client, passthrough_server.name) - assert tools == [tool] + listing = await manager._fetch_tools_with_timeout(mock_client, passthrough_server.name) + assert listing.tools == [tool] @pytest.mark.parametrize("status_code", [401, 403]) @@ -307,7 +307,7 @@ async def test_fetch_tools_from_gateway_managed_surfaces_upstream_401(): "401", request=response.request, response=response ) mock_client = MagicMock() - mock_client.list_tools = AsyncMock(side_effect=upstream_error) + mock_client.list_tools_result = AsyncMock(side_effect=upstream_error) with pytest.raises(MCPUpstreamAuthError) as exc_info: await manager._fetch_tools_with_timeout(mock_client, oauth2_server.name) @@ -315,7 +315,7 @@ async def test_fetch_tools_from_gateway_managed_surfaces_upstream_401(): assert exc_info.value.status_code == 401 assert exc_info.value.www_authenticate is None assert exc_info.value.server_name == "keycloak_whoami" - mock_client.list_tools.assert_awaited_with(raise_on_error=True) + mock_client.list_tools_result.assert_awaited_with(raise_on_error=True) def _http_server(server_id: str, name: str, **kwargs) -> MCPServer: @@ -463,7 +463,7 @@ async def test_fetch_tools_logs_upstream_request_details_on_500(caplog): ) response = httpx.Response(500, request=request) mock_client = MagicMock() - mock_client.list_tools = AsyncMock( + mock_client.list_tools_result = AsyncMock( side_effect=httpx.HTTPStatusError("500", request=request, response=response) ) @@ -553,14 +553,16 @@ async def test_scope_authorization_releases_progress_without_waiting_for_tool_re from mcp.server.context import ServerRequestContext from starlette.requests import Request + from litellm.proxy._experimental.mcp_server.legacy_callbacks import ( + get_scope_response, + record_upstream_tool_authorization, + ) from litellm.proxy._experimental.mcp_server.mcp_context import active_mcp_request_ctx_var from litellm.proxy._experimental.mcp_server.scope_challenge import ( SCOPE_RESPONSE_KEY, OAuthScopeResponse, ) - from litellm.proxy._experimental.mcp_server.legacy_callbacks import get_scope_response, record_upstream_tool_authorization - send = AsyncMock() gate = OAuthScopeResponse(send, "https://gateway.example", "/mcp/docs") request = Request({"type": "http", SCOPE_RESPONSE_KEY: gate}) @@ -598,7 +600,11 @@ async def test_scope_middleware_releases_waiting_response_on_validation_or_cance from mcp.server.context import ServerRequestContext from starlette.requests import Request - from litellm.proxy._experimental.mcp_server.scope_challenge import SCOPE_RESPONSE_KEY, OAuthScopeResponse, finish_scope_response + from litellm.proxy._experimental.mcp_server.scope_challenge import ( + SCOPE_RESPONSE_KEY, + OAuthScopeResponse, + finish_scope_response, + ) send = AsyncMock() gate = OAuthScopeResponse(send, "https://gateway.example", "/mcp/docs") @@ -635,7 +641,11 @@ async def test_streamable_sdk_relays_scope_challenge_before_committing_http_resp from litellm.proxy._experimental.mcp_server.legacy_callbacks import get_scope_response from litellm.proxy._experimental.mcp_server.mcp_context import active_mcp_request_ctx_var - from litellm.proxy._experimental.mcp_server.scope_challenge import SCOPE_RESPONSE_KEY, OAuthScopeResponse, finish_scope_response + from litellm.proxy._experimental.mcp_server.scope_challenge import ( + SCOPE_RESPONSE_KEY, + OAuthScopeResponse, + finish_scope_response, + ) async def call_tool(context, params): token = active_mcp_request_ctx_var.set(context) @@ -687,7 +697,10 @@ async def test_scope_challenges_are_isolated_between_concurrent_requests(): from mcp.server.context import ServerRequestContext from starlette.requests import Request - from litellm.proxy._experimental.mcp_server.legacy_callbacks import get_scope_response, record_upstream_tool_authorization + from litellm.proxy._experimental.mcp_server.legacy_callbacks import ( + get_scope_response, + record_upstream_tool_authorization, + ) from litellm.proxy._experimental.mcp_server.mcp_context import active_mcp_request_ctx_var from litellm.proxy._experimental.mcp_server.scope_challenge import SCOPE_RESPONSE_KEY, OAuthScopeResponse @@ -738,9 +751,10 @@ async def test_scope_middleware_preserves_non_http_requests(): @pytest.mark.parametrize("path", ["/mcp", "/toolset/demo/mcp"]) def test_managed_step_up_metadata_matches_the_sdk_configured_resource(path): - from mcp.shared.auth_utils import check_resource_allowed from urllib.parse import urlsplit + from mcp.shared.auth_utils import check_resource_allowed + error = MCPUpstreamAuthError(403, None, "docs", required_scope="tools.write") response = error.to_http_exception("https://gateway.test", path) challenge = response.headers["www-authenticate"] diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_proxy_mode.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_proxy_mode.py index 7dadfc6493a..e4628a676c9 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_proxy_mode.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_proxy_mode.py @@ -3,12 +3,13 @@ import asyncio import json import logging from datetime import datetime +from types import SimpleNamespace from unittest.mock import AsyncMock, patch import pytest from fastapi import HTTPException from mcp.shared.exceptions import MCPError -from mcp.types import CallToolResult, TextContent +from mcp.types import CallToolResult, ListToolsResult, TextContent from mcp.types import Tool as MCPTool from pydantic import AnyUrl @@ -29,6 +30,10 @@ from litellm.types.mcp_server.mcp_server_manager import MCPServer AUTH = UserAPIKeyAuth(api_key="key") +def _listing_client() -> SimpleNamespace: + return SimpleNamespace(discovery_auth_fingerprint=AsyncMock(return_value="test-credential-hash")) + + @pytest.fixture def proxy_mode(): token = _mcp_proxy_mode.set(True) @@ -165,8 +170,8 @@ async def test_proxy_call_tool_on_a_never_listed_tool_hands_the_pre_hook_no_list patch.dict(manager.registry, {server.server_id: server}), patch.dict(manager.tool_name_to_mcp_server_name_mapping), patch.object(mcp_operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[server])), - patch.object(manager, "create_mcp_client", AsyncMock(return_value=object())), - patch.object(manager, "_fetch_tools_with_timeout", AsyncMock(return_value=upstream)), + patch.object(manager, "create_mcp_client", AsyncMock(return_value=_listing_client())), + patch.object(manager, "_fetch_tools_with_timeout", AsyncMock(return_value=ListToolsResult(tools=upstream))), patch.object(manager, "pre_call_tool_check", pre_call_tool_check), patch.object(manager, "_call_regular_mcp_tool", call_regular_mcp_tool), ): @@ -204,14 +209,15 @@ async def test_proxy_schema_and_call_resolve_tool_id_without_waiting_on_unrelate slow_teardown = asyncio.Event() served_as = with_mcp_proxy_identity(MCPTool(name="proxy-fast-echo", inputSchema={}), fast.server_id) - async def fetch_tools(client: object, server_name: str) -> list[MCPTool]: + async def fetch_tools(client: object, server_name: str) -> ListToolsResult: if server_name == slow.name: try: await slow_release.wait() except asyncio.CancelledError: await slow_teardown.wait() raise - return [MCPTool(name="echo", description="Echo text back", inputSchema={"type": "object"})] + echo = MCPTool(name="echo", description="Echo text back", inputSchema={"type": "object"}) + return ListToolsResult(tools=[echo]) async def call_regular_mcp_tool(*, tasks: list[asyncio.Task[object]], **_: object) -> CallToolResult: await asyncio.gather(*tasks) @@ -222,7 +228,7 @@ async def test_proxy_schema_and_call_resolve_tool_id_without_waiting_on_unrelate patch.dict(manager.registry, {fast.server_id: fast, slow.server_id: slow}), patch.dict(manager.tool_name_to_mcp_server_name_mapping), patch.object(mcp_operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[slow, fast])), - patch.object(manager, "create_mcp_client", AsyncMock(return_value=object())), + patch.object(manager, "create_mcp_client", AsyncMock(return_value=_listing_client())), patch.object(manager, "_fetch_tools_with_timeout", fetch_tools), patch.object(manager, "pre_call_tool_check", AsyncMock(return_value={})), patch.object(manager, "_call_regular_mcp_tool", call_regular_mcp_tool), diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py index de711d737d0..5118e39ed7d 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py @@ -69,7 +69,8 @@ async def test_mcp_server_manager_https_server(): # Create a mock MCPClient mock_client = AsyncMock() - mock_client.list_tools = AsyncMock(return_value=mock_tools) + mock_client.list_tools_result = AsyncMock(return_value=ListToolsResult(tools=mock_tools)) + mock_client.discovery_auth_fingerprint = AsyncMock(return_value="test-credential-hash") mock_client.call_tool = AsyncMock(return_value=mock_result) # Mock the MCPClient constructor @@ -133,7 +134,7 @@ async def test_mcp_server_manager_https_server(): assert result.content[0].text == "Email sent successfully" # Verify client methods were called - mock_client.list_tools.assert_called() + mock_client.list_tools_result.assert_called() mock_client.call_tool.assert_called_once() @@ -176,7 +177,8 @@ async def test_mcp_http_transport_list_tools_mock(): # Create a mock MCPClient that returns our test tools mock_client = AsyncMock() - mock_client.list_tools = AsyncMock(return_value=mock_tools) + mock_client.list_tools_result = AsyncMock(return_value=ListToolsResult(tools=mock_tools)) + mock_client.discovery_auth_fingerprint = AsyncMock(return_value="test-credential-hash") mock_client.__aenter__ = AsyncMock(return_value=mock_client) mock_client.__aexit__ = AsyncMock(return_value=None) @@ -218,7 +220,7 @@ async def test_mcp_http_transport_list_tools_mock(): assert tools[1].name == f"{expected_prefix}-calendar_create_event" # Verify client methods were called - mock_client.list_tools.assert_called() + mock_client.list_tools_result.assert_called() # Verify tool mapping was updated expected_prefix = "test_http_server" @@ -1104,9 +1106,10 @@ async def test_list_tools_only_returns_allowed_servers(monkeypatch): mock_client = AsyncMock() # Return tools based on server URL if kwargs.get("server_url") == "https://server-a.com/mcp": - mock_client.list_tools = AsyncMock(return_value=mock_tools_a) + mock_client.list_tools_result = AsyncMock(return_value=ListToolsResult(tools=mock_tools_a)) else: - mock_client.list_tools = AsyncMock(return_value=mock_tools_b) + mock_client.list_tools_result = AsyncMock(return_value=ListToolsResult(tools=mock_tools_b)) + mock_client.discovery_auth_fingerprint = AsyncMock(return_value="test-credential-hash") mock_client.__aenter__ = AsyncMock(return_value=mock_client) mock_client.__aexit__ = AsyncMock(return_value=None) return mock_client @@ -1401,7 +1404,8 @@ async def test_mcp_server_manager_alias_tool_prefixing(): # Mock MCPClient mock_client = AsyncMock() - mock_client.list_tools = AsyncMock(return_value=mock_tools) + mock_client.list_tools_result = AsyncMock(return_value=ListToolsResult(tools=mock_tools)) + mock_client.discovery_auth_fingerprint = AsyncMock(return_value="test-credential-hash") mock_client.__aenter__ = AsyncMock(return_value=mock_client) mock_client.__aexit__ = AsyncMock(return_value=None) @@ -1461,7 +1465,8 @@ async def test_mcp_server_manager_server_name_tool_prefixing(): # Mock MCPClient mock_client = AsyncMock() - mock_client.list_tools = AsyncMock(return_value=mock_tools) + mock_client.list_tools_result = AsyncMock(return_value=ListToolsResult(tools=mock_tools)) + mock_client.discovery_auth_fingerprint = AsyncMock(return_value="test-credential-hash") mock_client.__aenter__ = AsyncMock(return_value=mock_client) mock_client.__aexit__ = AsyncMock(return_value=None) @@ -1521,7 +1526,8 @@ async def test_mcp_server_manager_server_id_tool_prefixing(): # Mock MCPClient mock_client = AsyncMock() - mock_client.list_tools = AsyncMock(return_value=mock_tools) + mock_client.list_tools_result = AsyncMock(return_value=ListToolsResult(tools=mock_tools)) + mock_client.discovery_auth_fingerprint = AsyncMock(return_value="test-credential-hash") mock_client.__aenter__ = AsyncMock(return_value=mock_client) mock_client.__aexit__ = AsyncMock(return_value=None) diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py index 7315315a619..53a78f2a779 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -11,7 +11,7 @@ import anyio from collections.abc import AsyncIterator, Awaitable, Callable, Iterator from datetime import datetime from pathlib import Path -from types import SimpleNamespace +from types import EllipsisType, SimpleNamespace from typing import Any, Dict, Final, Literal, Optional, TypeVar, cast from unittest.mock import AsyncMock, MagicMock, patch @@ -242,6 +242,12 @@ async def test_sampling_callbacks_isolate_callers_and_cancellation(): assert cancelled.is_set() +def _listing_client() -> AsyncMock: + client: Final = AsyncMock() + client.discovery_auth_fingerprint = AsyncMock(return_value="test-credential-hash") + return client + + def _reload_mcp_manager_module(): utils_module = sys.modules["litellm.proxy._experimental.mcp_server.utils"] manager_module = sys.modules["litellm.proxy._experimental.mcp_server.mcp_server_manager"] @@ -2353,10 +2359,10 @@ class TestMCPServerManager: server, mcp_auth_header, extra_headers, stdio_env, subject_token=None, **kwargs ): # pragma: no cover - helper captured["subject_token"] = subject_token - return AsyncMock() + return _listing_client() manager.create_mcp_client = AsyncMock(side_effect=capture_create_mcp_client) - manager._fetch_tools_with_timeout = AsyncMock(return_value=[]) + manager._fetch_tools_with_timeout = AsyncMock(return_value=ListToolsResult(tools=[])) await manager._get_tools_from_server(server=server, oauth2_headers=oauth2_headers, raw_headers=raw_headers) return captured["subject_token"] @@ -3175,7 +3181,7 @@ class TestMCPServerManager: server, mcp_auth_header, extra_headers, stdio_env, subject_token=None, **kwargs ): # pragma: no cover - helper captured["subject_token"] = subject_token - return AsyncMock() + return _listing_client() manager.create_mcp_client = AsyncMock(side_effect=capture_create_mcp_client) await call(manager) @@ -6492,7 +6498,7 @@ class TestMCPServerManager: ) # Mock client creation and fetching tools - manager.create_mcp_client = AsyncMock(return_value=object()) + manager.create_mcp_client = AsyncMock(return_value=_listing_client()) # Tools returned upstream (unprefixed from provider) upstream_tool = MCPTool( @@ -6501,7 +6507,7 @@ class TestMCPServerManager: inputSchema={}, ) - manager._fetch_tools_with_timeout = AsyncMock(return_value=[upstream_tool]) + manager._fetch_tools_with_timeout = AsyncMock(return_value=ListToolsResult(tools=[upstream_tool])) # Case 1: add_prefix=True (default for multi-server) -> expect prefixed tools_prefixed = await manager._get_tools_from_server(server, add_prefix=True) @@ -6530,8 +6536,8 @@ class TestMCPServerManager: transport=MCPTransport.http, ) - manager.create_mcp_client = AsyncMock(return_value=object()) - manager._fetch_tools_with_timeout = AsyncMock(return_value=[]) + manager.create_mcp_client = AsyncMock(return_value=_listing_client()) + manager._fetch_tools_with_timeout = AsyncMock(return_value=ListToolsResult(tools=[])) user_auth = UserAPIKeyAuth(api_key="sk-test", user_id="alice") @@ -7868,9 +7874,9 @@ class TestMCPServerManager: async def fetch(client, name): fetch_started.set() await release_fetch.wait() - return [MCPTool(name="turn", description="before save", inputSchema={})] + return ListToolsResult(tools=[MCPTool(name="turn", description="before save", inputSchema={})]) - manager.create_mcp_client = AsyncMock(return_value=AsyncMock()) + manager.create_mcp_client = AsyncMock(return_value=_listing_client()) manager._fetch_tools_with_timeout = fetch caller = ListedToolsCaller(user_api_key_auth=user) @@ -7886,7 +7892,7 @@ class TestMCPServerManager: assert manager.get_listed_tool(server, "turn", caller) is None manager._fetch_tools_with_timeout = AsyncMock( - return_value=[MCPTool(name="turn", description="after save", inputSchema={})] + return_value=ListToolsResult(tools=[MCPTool(name="turn", description="after save", inputSchema={})]) ) await manager._get_tools_from_server(server=server, user_api_key_auth=user, record_listing=True) listed = manager.get_listed_tool(server, "turn", caller) @@ -8122,9 +8128,9 @@ class TestMCPServerManager: auth_type=MCPAuth.api_key, ) user = UserAPIKeyAuth(api_key="sk-litellm", user_id="byok-cold-user") - manager.create_mcp_client = AsyncMock(return_value=AsyncMock()) + manager.create_mcp_client = AsyncMock(return_value=_listing_client()) manager._fetch_tools_with_timeout = AsyncMock( - return_value=[MCPTool(name="turn", description="listed while db down", inputSchema={})] + return_value=ListToolsResult(tools=[MCPTool(name="turn", description="listed while db down", inputSchema={})]) ) with ( @@ -8172,11 +8178,11 @@ class TestMCPServerManager: ) manager.registry = {"byok-catalog": server} user = UserAPIKeyAuth(api_key="sk-litellm", user_id="byok-user") - mock_client = AsyncMock() + mock_client = _listing_client() mock_client.call_tool.return_value = MagicMock(spec=CallToolResult, content=[], isError=False) manager.create_mcp_client = AsyncMock(return_value=mock_client) manager._fetch_tools_with_timeout = AsyncMock( - return_value=[MCPTool(name="turn", description="stored cred catalog", inputSchema={})] + return_value=ListToolsResult(tools=[MCPTool(name="turn", description="stored cred catalog", inputSchema={})]) ) proxy_logging_obj = _mock_proxy_logging() proxy_logging_obj.create_mcp_request_object_from_kwargs = MagicMock(return_value={}) @@ -8217,9 +8223,9 @@ class TestMCPServerManager: url="http://byok-catalog", is_byok=True, ) - manager.create_mcp_client = AsyncMock(return_value=AsyncMock()) + manager.create_mcp_client = AsyncMock(return_value=_listing_client()) manager._fetch_tools_with_timeout = AsyncMock( - return_value=[MCPTool(name="turn", description="t", inputSchema={})] + return_value=ListToolsResult(tools=[MCPTool(name="turn", description="t", inputSchema={})]) ) with patch("litellm.proxy.proxy_server.prisma_client", None): @@ -8280,9 +8286,9 @@ class TestMCPServerManager: **server_auth, ) alice = UserAPIKeyAuth(api_key="sk-alice", user_id="alice") - manager.create_mcp_client = AsyncMock(return_value=AsyncMock()) + manager.create_mcp_client = AsyncMock(return_value=_listing_client()) manager._fetch_tools_with_timeout = AsyncMock( - return_value=[MCPTool(name="echo", description="listed catalog", inputSchema={})] + return_value=ListToolsResult(tools=[MCPTool(name="echo", description="listed catalog", inputSchema={})]) ) signer_headers = AsyncMock(return_value={"Authorization": "Bearer signed-jwt"}) cache_byok_credential("alice", "cc1", "BYOK-ALICE-SECRET") @@ -8496,10 +8502,10 @@ class TestMCPServerManager: ) ], } - mock_client = AsyncMock() + mock_client = _listing_client() mock_client.call_tool.return_value = MagicMock(spec=CallToolResult, content=[], isError=False) manager.create_mcp_client = AsyncMock(return_value=mock_client) - manager._fetch_tools_with_timeout = AsyncMock(side_effect=lambda client, name: catalogs[client.workspace]) + manager._fetch_tools_with_timeout = AsyncMock(side_effect=lambda client, name: ListToolsResult(tools=catalogs[client.workspace])) for workspace in ("A", "B"): manager.create_mcp_client.return_value.workspace = workspace await manager._get_tools_from_server( @@ -11787,7 +11793,7 @@ class TestMCPToolsListAuthSurfacing: manager = MCPServerManager() challenge = 'Bearer resource_metadata="https://upstream.example/.well-known/oauth-protected-resource"' client = MagicMock() - client.list_tools = AsyncMock(side_effect=_upstream_status_error(401, challenge)) + client.list_tools_result = AsyncMock(side_effect=_upstream_status_error(401, challenge)) with pytest.raises(MCPUpstreamAuthError) as exc_info: await manager._fetch_tools_with_timeout(client, "static-key-server") @@ -11805,7 +11811,7 @@ class TestMCPToolsListAuthSurfacing: manager = MCPServerManager() challenge = 'Bearer error="insufficient_scope", scope="read:tools"' client = MagicMock() - client.list_tools = AsyncMock(side_effect=_upstream_status_error(403, challenge)) + client.list_tools_result = AsyncMock(side_effect=_upstream_status_error(403, challenge)) with pytest.raises(MCPUpstreamAuthError) as exc_info: await manager._fetch_tools_with_timeout(client, "forbidden-server") @@ -11822,7 +11828,7 @@ class TestMCPToolsListAuthSurfacing: own fault ("internal").""" manager = MCPServerManager() client = MagicMock() - client.list_tools = AsyncMock(side_effect=RuntimeError("upstream 500")) + client.list_tools_result = AsyncMock(side_effect=RuntimeError("upstream 500")) with pytest.raises(MCPServerListError) as exc_info: await manager._fetch_tools_with_timeout(client, "srv") @@ -11969,7 +11975,8 @@ class TestMCPToolsListAuthSurfacing: ) upstream_challenge = 'Bearer resource_metadata="https://upstream.example/.well-known/oauth-protected-resource"' client = MagicMock() - client.list_tools = AsyncMock(side_effect=_upstream_status_error(401, upstream_challenge)) + client.discovery_auth_fingerprint = AsyncMock(return_value="test-credential-hash") + client.list_tools_result = AsyncMock(side_effect=_upstream_status_error(401, upstream_challenge)) manager.create_mcp_client = AsyncMock(return_value=client) with pytest.raises(MCPUpstreamAuthError) as exc_info: @@ -14316,7 +14323,9 @@ class TestClientForwardedDiscoveryFailureIsNotFatal: server = await self._registered(manager, auth_type, None) manager._set_oauth_discovery_deferred(server.server_id, True) manager._fetch_tools_with_timeout = AsyncMock( - return_value=[MCPTool(name="list_reports", description="d", inputSchema={"type": "object"})] + return_value=ListToolsResult( + tools=[MCPTool(name="list_reports", description="d", inputSchema={"type": "object"})] + ) ) with patch.object(manager, "_discover_oauth_metadata_for_server", new=AsyncMock(return_value=None)): @@ -15224,6 +15233,7 @@ class TestLitellmAdmissionKeyIsNeverTheSubjectToken: manager: Final = MCPServerManager() client: Final = AsyncMock() client.call_tool = AsyncMock(return_value=CallToolResult(content=[], isError=False)) + client.discovery_auth_fingerprint = AsyncMock(return_value="test-credential-hash") client.list_prompts_result = AsyncMock(return_value=ListPromptsResult(prompts=[])) client.read_resource = AsyncMock(return_value=ReadResourceResult(contents=[])) manager.create_mcp_client = AsyncMock(return_value=client) @@ -15340,7 +15350,7 @@ class TestLitellmAdmissionKeyIsNeverTheSubjectToken: @pytest.mark.asyncio async def test_tools_list_with_only_the_litellm_key_has_no_subject(self): manager: Final = self._manager_with_recording_client() - manager._fetch_tools_with_timeout = AsyncMock(return_value=[]) + manager._fetch_tools_with_timeout = AsyncMock(return_value=ListToolsResult(tools=[])) await manager._get_tools_from_server( server=self._token_exchange_server("te-list-key"), oauth2_headers={"Authorization": f"Bearer {self._ADMISSION_KEY}"}, @@ -16446,9 +16456,12 @@ async def test_health_check_logs_upstream_exchange_only_for_http_failures( class _DiscoveryUpstream: def __init__(self) -> None: self.requests: tuple[tuple[str, str], ...] = () - self.tools: tuple[dict[str, object], ...] = () + self.tools: tuple[dict[str, object], ...] = ( + {"name": "example", "description": "original", "inputSchema": {"type": "object"}}, + ) self.outcome = "supported" self.ttl_ms = 60000 + self.last_page_ttl_ms = 60000 self.entered = asyncio.Event() self.release = asyncio.Event() self.release.set() @@ -16523,7 +16536,7 @@ class _DiscoveryUpstream: "result": { **result, **continuation, - "ttlMs": self.ttl_ms, + "ttlMs": self.last_page_ttl_ms if (payload.params or {}).get("cursor") else self.ttl_ms, "cacheScope": "private", "resultType": "complete", }, @@ -16634,6 +16647,100 @@ async def test_discovery_cache_reuses_raw_results_and_expires(kind: str) -> None assert upstream.initializes == 2 +def _list_discovery_tools(manager: MCPServerManager, cursor: str | None | EllipsisType = ..., **kwargs): + from mcp.types import PaginatedRequestParams + + params: Final = None if cursor is ... else PaginatedRequestParams(cursor=cursor) + return manager.get_tools_page(_discovery_server(), params=params, **kwargs) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cursor", (..., None), ids=("full_walk", "first_page")) +async def test_tools_list_is_served_from_cache_within_the_upstream_ttl(cursor: str | None | EllipsisType) -> None: + clock: Final = _DiscoveryClock() + manager: Final = MCPServerManager(discovery_clock=clock) + upstream: Final = _DiscoveryUpstream() + upstream.ttl_ms = 30000 + with _mcp_upstream(upstream.respond): + first: Final = await _list_discovery_tools(manager, cursor) + assert [tool.name for tool in first.tools] == ["discovery-example"] + assert 29000 < first.ttl_ms <= 30000 + clock.now = 29.9 + cached: Final = await _list_discovery_tools(manager, cursor, add_prefix=False) + assert [tool.name for tool in cached.tools] == ["example"] + assert 0 < cached.ttl_ms <= 100 + assert upstream.initializes == 1 + clock.now = 30.001 + assert [tool.name for tool in (await _list_discovery_tools(manager, cursor)).tools] == ["discovery-example"] + assert upstream.initializes == 2 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cursor", (..., None), ids=("full_walk", "first_page")) +async def test_tools_list_is_not_cached_without_an_upstream_ttl(cursor: str | None | EllipsisType) -> None: + manager: Final = MCPServerManager(discovery_clock=_DiscoveryClock()) + upstream: Final = _DiscoveryUpstream() + upstream.ttl_ms = 0 + with _mcp_upstream(upstream.respond): + for _ in range(2): + assert len((await _list_discovery_tools(manager, cursor)).tools) == 1 + assert upstream.initializes == 2 + + +@pytest.mark.asyncio +async def test_tools_list_full_walk_takes_the_shortest_page_ttl_and_keys_pages_by_cursor() -> None: + clock: Final = _DiscoveryClock() + manager: Final = MCPServerManager(discovery_clock=clock) + upstream: Final = _DiscoveryUpstream() + upstream.outcome = "paged" + upstream.last_page_ttl_ms = 20000 + with _mcp_upstream(upstream.respond): + walk: Final = await _list_discovery_tools(manager) + assert len(walk.tools) == 2 + assert 19000 < walk.ttl_ms <= 20000 + clock.now = 20.001 + assert len((await _list_discovery_tools(manager)).tools) == 2 + assert upstream.initializes == 2 + first_page: Final = await _list_discovery_tools(manager, None) + last_page: Final = await _list_discovery_tools(manager, "last-page") + assert first_page.next_cursor == "last-page" + assert last_page.next_cursor is None + assert upstream.initializes == 4 + assert (await _list_discovery_tools(manager, "last-page")).next_cursor is None + assert (await _list_discovery_tools(manager, None)).next_cursor == "last-page" + assert upstream.initializes == 4 + + +@pytest.mark.asyncio +async def test_tools_list_cache_isolates_callers_and_is_invalidated_with_the_server() -> None: + manager: Final = MCPServerManager(discovery_clock=_DiscoveryClock()) + upstream: Final = _DiscoveryUpstream() + with _mcp_upstream(upstream.respond): + for credential in ("first-secret", "second-secret", "first-secret"): + assert len((await _list_discovery_tools(manager, extra_headers={"Authorization": credential})).tools) == 1 + assert upstream.initializes == 2 + assert {auth for method, auth in upstream.requests if method == "tools/list"} == { + "first-secret", + "second-secret", + } + manager.invalidate_discovery_lists("discovery") + assert len((await _list_discovery_tools(manager, extra_headers={"Authorization": "first-secret"})).tools) == 1 + assert upstream.initializes == 3 + + +@pytest.mark.asyncio +async def test_tools_list_failures_are_not_cached() -> None: + manager: Final = MCPServerManager(discovery_clock=_DiscoveryClock()) + upstream: Final = _DiscoveryUpstream() + upstream.outcome = "failure" + with _mcp_upstream(upstream.respond): + with pytest.raises(MCPServerListError): + await _list_discovery_tools(manager) + upstream.outcome = "supported" + assert len((await _list_discovery_tools(manager)).tools) == 1 + assert upstream.initializes == 2 + + @pytest.mark.asyncio @pytest.mark.parametrize("kind", ("prompts", "resources", "templates")) @pytest.mark.parametrize("outcome", ("unsupported", "rejected", "failure")) @@ -19917,8 +20024,8 @@ def catalog_guardrail(monkeypatch): def _catalog_manager(*upstream_tools: MCPTool) -> MCPServerManager: manager = MCPServerManager() - manager.create_mcp_client = AsyncMock(return_value=object()) - manager._fetch_tools_with_timeout = AsyncMock(return_value=list(upstream_tools)) + manager.create_mcp_client = AsyncMock(return_value=_listing_client()) + manager._fetch_tools_with_timeout = AsyncMock(return_value=ListToolsResult(tools=list(upstream_tools))) return manager @@ -20078,14 +20185,14 @@ class TestToolCatalogGuard: await manager._get_tools_from_server(_notes_server(), add_prefix=False, proxy_logging_obj=proxy_logging_obj) assert send_alert.await_count == 1 - manager._fetch_tools_with_timeout = AsyncMock(return_value=[LIST_NOTES]) + manager._fetch_tools_with_timeout = AsyncMock(return_value=ListToolsResult(tools=[LIST_NOTES])) recovered = await manager._get_tools_from_server( _notes_server(), add_prefix=False, proxy_logging_obj=proxy_logging_obj ) assert [tool.name for tool in recovered] == ["list_notes"] assert send_alert.await_count == 1 - manager._fetch_tools_with_timeout = AsyncMock(return_value=[LIST_NOTES, POISONED_DELETE]) + manager._fetch_tools_with_timeout = AsyncMock(return_value=ListToolsResult(tools=[LIST_NOTES, POISONED_DELETE])) await manager._get_tools_from_server(_notes_server(), add_prefix=False, proxy_logging_obj=proxy_logging_obj) assert send_alert.await_count == 2 @@ -20291,12 +20398,12 @@ class TestToolCatalogGuard: ) while send_alert.await_count == 0: await asyncio.sleep(0) - manager._fetch_tools_with_timeout = AsyncMock(return_value=[LIST_NOTES]) + manager._fetch_tools_with_timeout = AsyncMock(return_value=ListToolsResult(tools=[LIST_NOTES])) await manager._get_tools_from_server(_notes_server(), add_prefix=False, proxy_logging_obj=proxy_logging_obj) gate.set() await poisoned_listing - manager._fetch_tools_with_timeout = AsyncMock(return_value=[LIST_NOTES, POISONED_DELETE]) + manager._fetch_tools_with_timeout = AsyncMock(return_value=ListToolsResult(tools=[LIST_NOTES, POISONED_DELETE])) await manager._get_tools_from_server(_notes_server(), add_prefix=False, proxy_logging_obj=proxy_logging_obj) assert send_alert.await_count == 2 @@ -20919,6 +21026,7 @@ async def test_catalog_page_registers_bare_routes_only_for_complete_initial_disc manager.registry = {server.server_id: server, other.server_id: other} manager._create_prefixed_tools([LIST_NOTES], other) manager.create_mcp_client.return_value = SimpleNamespace( + discovery_auth_fingerprint=AsyncMock(return_value="test-credential-hash"), list_tools_page=AsyncMock(return_value=ListToolsResult(tools=[LIST_NOTES], next_cursor=next_cursor)) ) @@ -20944,7 +21052,7 @@ async def test_paginated_listing_keeps_earlier_tool_metadata_and_caller_isolatio server = MCPServer(server_id="pages", name="pages", transport=MCPTransport.http) manager = MCPServerManager() manager.registry = {server.server_id: server} - client = AsyncMock() + client = _listing_client() client._last_initialize_instructions = None client.list_tools_page.side_effect = [ ListToolsResult(tools=[first], next_cursor="next"), @@ -21102,7 +21210,7 @@ async def test_failed_aggregate_continuation_preserves_only_delivered_tool_metad servers = [MCPServer(server_id=name, name=name, transport=MCPTransport.http) for name in ("alpha", "beta")] manager = MCPServerManager() manager.registry = {server.server_id: server for server in servers} - clients = {server.server_id: AsyncMock() for server in servers} + clients = {server.server_id: _listing_client() for server in servers} for client in clients.values(): client._last_initialize_instructions = None client.list_tools_page.side_effect = [ @@ -21146,7 +21254,7 @@ async def test_aggregate_publishes_complete_bare_routes_only_after_delivering_a_ servers = [MCPServer(server_id=name, name=name, transport=MCPTransport.http) for name in ("alpha", "beta")] manager = MCPServerManager() manager.registry = {server.server_id: server for server in servers} - clients = {server.server_id: AsyncMock() for server in servers} + clients = {server.server_id: _listing_client() for server in servers} for name, client in clients.items(): client._last_initialize_instructions = None client.list_tools_page.return_value = ListToolsResult( @@ -21481,55 +21589,55 @@ class TestMCPUpstreamRateLimitRetry: manager: Final = MCPServerManager() tool: Final = MCPTool(name="ping", inputSchema={"type": "object"}) client: Final = MagicMock() - client.list_tools = AsyncMock( + client.list_tools_result = AsyncMock( side_effect=[ _upstream_http_status_error(429, {"Retry-After": "0"}), _upstream_http_status_error(429, {"Retry-After": "0"}), - [tool], + ListToolsResult(tools=[tool]), ] ) - tools: Final = await manager._fetch_tools_with_timeout(client, "flaky-server") + listing: Final = await manager._fetch_tools_with_timeout(client, "flaky-server") - assert tools == [tool] - assert client.list_tools.await_count == 3 + assert listing.tools == [tool] + assert client.list_tools_result.await_count == 3 @pytest.mark.asyncio async def test_fetch_tools_with_timeout_gives_up_after_max_retries(self) -> None: manager: Final = MCPServerManager() client: Final = MagicMock() - client.list_tools = AsyncMock(side_effect=_upstream_http_status_error(429, {"Retry-After": "0"})) + client.list_tools_result = AsyncMock(side_effect=_upstream_http_status_error(429, {"Retry-After": "0"})) expected_error: Final = pytest.raises(MCPServerListError) with expected_error as exc_info: await manager._fetch_tools_with_timeout(client, "saturated-server") assert exc_info.value.fault == ServerListFault(tag="upstream_error", status_code=429) - assert client.list_tools.await_count == 1 + MCP_UPSTREAM_RATE_LIMIT_MAX_RETRIES + assert client.list_tools_result.await_count == 1 + MCP_UPSTREAM_RATE_LIMIT_MAX_RETRIES @pytest.mark.asyncio async def test_fetch_tools_with_timeout_does_not_retry_auth_error(self) -> None: manager: Final = MCPServerManager() client: Final = MagicMock() - client.list_tools = AsyncMock(side_effect=_upstream_http_status_error(401)) + client.list_tools_result = AsyncMock(side_effect=_upstream_http_status_error(401)) with pytest.raises(MCPUpstreamAuthError): await manager._fetch_tools_with_timeout(client, "srv") - assert client.list_tools.await_count == 1 + assert client.list_tools_result.await_count == 1 @pytest.mark.asyncio async def test_fetch_tools_with_timeout_does_not_retry_server_error(self) -> None: manager: Final = MCPServerManager() client: Final = MagicMock() - client.list_tools = AsyncMock(side_effect=_upstream_http_status_error(500)) + client.list_tools_result = AsyncMock(side_effect=_upstream_http_status_error(500)) expected_error: Final = pytest.raises(MCPServerListError) with expected_error as exc_info: await manager._fetch_tools_with_timeout(client, "srv") assert exc_info.value.fault == ServerListFault(tag="upstream_error", status_code=500) - assert client.list_tools.await_count == 1 + assert client.list_tools_result.await_count == 1 @pytest.mark.asyncio async def test_fetch_tools_with_timeout_reports_429_when_backoff_would_pass_listing_timeout( @@ -21540,7 +21648,7 @@ class TestMCPUpstreamRateLimitRetry: ) manager: Final = MCPServerManager() client: Final = MagicMock() - client.list_tools = AsyncMock(side_effect=_upstream_http_status_error(429, {"Retry-After": "5"})) + client.list_tools_result = AsyncMock(side_effect=_upstream_http_status_error(429, {"Retry-After": "5"})) with anyio.fail_after(3): expected_error: Final = pytest.raises(MCPServerListError) @@ -21548,7 +21656,7 @@ class TestMCPUpstreamRateLimitRetry: await manager._fetch_tools_with_timeout(client, "srv") assert exc_info.value.fault == ServerListFault(tag="upstream_error", status_code=429) - assert client.list_tools.await_count == 1 + assert client.list_tools_result.await_count == 1 @pytest.mark.asyncio async def test_get_tools_page_retries_429_then_returns_page(self) -> None: @@ -21559,6 +21667,7 @@ class TestMCPUpstreamRateLimitRetry: manager.create_mcp_client = AsyncMock( return_value=SimpleNamespace( _last_initialize_instructions=None, + discovery_auth_fingerprint=AsyncMock(return_value="test-credential-hash"), list_tools_page=AsyncMock( side_effect=[ _upstream_http_status_error(429, {"Retry-After": "0"}), @@ -21596,7 +21705,11 @@ class TestMCPUpstreamRateLimitRetry: raise AssertionError("unreachable") list_tools_page: Final = AsyncMock(side_effect=_list_tools_page) - client: Final = SimpleNamespace(_last_initialize_instructions=None, list_tools_page=list_tools_page) + client: Final = SimpleNamespace( + _last_initialize_instructions=None, + discovery_auth_fingerprint=AsyncMock(return_value="test-credential-hash"), + list_tools_page=list_tools_page, + ) manager.create_mcp_client = AsyncMock(return_value=client) with anyio.fail_after(5): diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py index b8e28c3f9b8..6e25787d6d7 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py @@ -93,6 +93,12 @@ def cleanup_mcp_global_state(): yield +def _listing_client() -> AsyncMock: + client = AsyncMock() + client.discovery_auth_fingerprint = AsyncMock(return_value="test-credential-hash") + return client + + def _call_tool_params(name, arguments=None): from mcp.types import CallToolRequestParams @@ -7721,13 +7727,13 @@ def _worker_that_never_listed(server: MCPServer, upstream_tools: tuple[str, ...] return CallToolResult(content=[TextContent(type="text", text="ok")], is_error=False) async def fake_fetch_tools(client, server_name): - return [MCPTool(name=tool_name, inputSchema={}) for tool_name in upstream_tools] + return ListToolsResult(tools=[MCPTool(name=tool_name, inputSchema={}) for tool_name in upstream_tools]) with ( patch.object( # test-quality-ok: the upstream MCP session is the boundary; a real one needs an initialize handshake over a live server mcp_operations.global_mcp_server_manager, "create_mcp_client", - new=AsyncMock(return_value=MagicMock()), + new=AsyncMock(return_value=_listing_client()), ) as create_client, patch.object( # test-quality-ok: same boundary, this is the tools/list answer the upstream would give mcp_operations.global_mcp_server_manager, @@ -8574,7 +8580,7 @@ async def test_execute_mcp_tool_implicit_listing_before_the_first_call_hands_hoo server = _never_listed_passthrough_server() manager.registry[server.server_id] = server manager._listed_tools_by_server_id.pop(server.server_id, None) - upstream = AsyncMock() + upstream = _listing_client() upstream.call_tool.return_value = CallToolResult(content=[TextContent(type="text", text="ok")], isError=False) proxy_logging = _mock_mcp_proxy_logging() proxy_logging.create_mcp_request_object_from_kwargs = MagicMock(return_value={}) @@ -8582,7 +8588,9 @@ async def test_execute_mcp_tool_implicit_listing_before_the_first_call_hands_hoo proxy_logging.pre_call_hook = AsyncMock(return_value={}) proxy_logging.during_call_hook = AsyncMock(return_value=None) fetch_tools = AsyncMock( - return_value=[MCPTool(name="add", description="Adds. FLAGWORD", inputSchema={"type": "object"})] + return_value=ListToolsResult( + tools=[MCPTool(name="add", description="Adds. FLAGWORD", inputSchema={"type": "object"})] + ) ) with ( @@ -8629,11 +8637,13 @@ async def test_fetch_pinnable_tool_catalog_records_no_listed_catalog_for_the_adm request.client.host = "10.1.2.3" request.headers = {"x-litellm-api-key": "sk-admin"} fetch_tools = AsyncMock( - return_value=[MCPTool(name="add", description="Upstream wording", inputSchema={"type": "object"})] + return_value=ListToolsResult( + tools=[MCPTool(name="add", description="Upstream wording", inputSchema={"type": "object"})] + ) ) with ( - patch.object(manager, "create_mcp_client", new=AsyncMock(return_value=MagicMock())), + patch.object(manager, "create_mcp_client", new=AsyncMock(return_value=_listing_client())), patch.object(manager, "_fetch_tools_with_timeout", new=fetch_tools), patch("litellm.proxy.proxy_server.proxy_logging_obj", ProxyLogging(user_api_key_cache=DualCache())), ): diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_tool_search.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_tool_search.py index ea0dd70cd22..877a81892af 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_tool_search.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_tool_search.py @@ -21,7 +21,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest from pydantic import JsonValue -from mcp.types import Tool +from mcp.types import ListToolsResult, Tool import litellm from litellm.models.object_permission import LiteLLM_ObjectPermissionTable @@ -68,6 +68,10 @@ SAMPLE_TOOLS = _make_tools( ) +def _listing_client() -> SimpleNamespace: + return SimpleNamespace(discovery_auth_fingerprint=AsyncMock(return_value="test-credential-hash")) + + @pytest.mark.parametrize( ("schema", "arguments", "error"), ( @@ -1416,8 +1420,8 @@ async def test_mcp_tool_search_leaves_the_listed_tools_slot_empty(monkeypatch: p with ( patch.dict(manager.tool_name_to_mcp_server_name_mapping), patch.object(mcp_operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[server])), - patch.object(manager, "create_mcp_client", AsyncMock(return_value=object())), - patch.object(manager, "_fetch_tools_with_timeout", AsyncMock(return_value=upstream)), + patch.object(manager, "create_mcp_client", AsyncMock(return_value=_listing_client())), + patch.object(manager, "_fetch_tools_with_timeout", AsyncMock(return_value=ListToolsResult(tools=upstream))), ): try: result = await handle_mcp_tool_search(query="echo", top_k=1, user_api_key_dict=user) diff --git a/tests/unit/proxy/_experimental/mcp_server/test_operations.py b/tests/unit/proxy/_experimental/mcp_server/test_operations.py index 6f7437c90dc..5b3b02c9ec4 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_operations.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_operations.py @@ -1,11 +1,12 @@ import asyncio from collections.abc import Callable, Sequence +from types import SimpleNamespace from typing import Final, Literal from unittest.mock import AsyncMock, patch import pytest from mcp.server.context import ServerRequestContext -from mcp.types import GetPromptRequest, GetPromptRequestParams, GetPromptResult +from mcp.types import GetPromptRequest, GetPromptRequestParams, GetPromptResult, ListToolsResult from mcp.types import Tool as MCPTool import litellm @@ -44,6 +45,10 @@ async def _served_catalog_tool() -> str: return "ok" +def _listing_client() -> SimpleNamespace: + return SimpleNamespace(discovery_auth_fingerprint=AsyncMock(return_value="test-credential-hash")) + + @pytest.mark.parametrize( ("settings", "mcp_servers", "toolset_id", "auth_state", "raises"), [ @@ -1597,8 +1602,8 @@ async def test_list_mcp_tools_records_the_catalog_only_when_asked( upstream = [MCPTool(name="echo", description="Echo text back", inputSchema={"type": "object"})] with ( patch.object(operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[server])), - patch.object(manager, "create_mcp_client", AsyncMock(return_value=object())), - patch.object(manager, "_fetch_tools_with_timeout", AsyncMock(return_value=upstream)), + patch.object(manager, "create_mcp_client", AsyncMock(return_value=_listing_client())), + patch.object(manager, "_fetch_tools_with_timeout", AsyncMock(return_value=ListToolsResult(tools=upstream))), patch.dict(manager.tool_name_to_mcp_server_name_mapping), ): try: diff --git a/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py index e88d951ff09..9fe25dd3cd2 100644 --- a/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py +++ b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py @@ -1100,7 +1100,10 @@ async def test_aresponses_mcp_call_does_not_forward_admission_authorization( monkeypatch.setattr(manager, "registry", {server.server_id: server}) mcp_client: Final = AsyncMock() - mcp_client.list_tools.return_value = [MCPTool(name="whoami", inputSchema={"type": "object", "properties": {}})] + mcp_client.discovery_auth_fingerprint.return_value = "test-credential-hash" + mcp_client.list_tools_result.return_value = ListToolsResult( + tools=[MCPTool(name="whoami", inputSchema={"type": "object", "properties": {}})] + ) mcp_client.call_tool.return_value = CallToolResult( content=[TextContent(type="text", text="ok")], isError=False, diff --git a/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py b/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py index 26553558298..9499264ef56 100644 --- a/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py +++ b/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py @@ -10,7 +10,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi import HTTPException -from mcp.types import CallToolResult, TextContent +from mcp.types import CallToolResult, ListToolsResult, TextContent from mcp.types import Tool as MCPTool from openai.types.responses.tool_param import Mcp @@ -70,6 +70,10 @@ def _setup_proxy_logging(monkeypatch: pytest.MonkeyPatch) -> AsyncMock: return proxy_logging_obj.post_call_failure_hook +def _listing_client() -> types.SimpleNamespace: + return types.SimpleNamespace(discovery_auth_fingerprint=AsyncMock(return_value="test-credential-hash")) + + def test_deduplicate_mcp_tools_single_allowed_server(): tools = [{"name": "search"}, {"name": "search"}] # duplicate on purpose @@ -1371,8 +1375,8 @@ async def test_bridge_listing_leaves_the_callers_catalog_unchanged( with ( patch.dict(manager.tool_name_to_mcp_server_name_mapping), patch.object(mcp_operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[server])), - patch.object(manager, "create_mcp_client", AsyncMock(return_value=object())), - patch.object(manager, "_fetch_tools_with_timeout", AsyncMock(return_value=upstream)), + patch.object(manager, "create_mcp_client", AsyncMock(return_value=_listing_client())), + patch.object(manager, "_fetch_tools_with_timeout", AsyncMock(return_value=ListToolsResult(tools=upstream))), ): try: if real_listing: @@ -1444,9 +1448,10 @@ async def test_concurrent_bridge_calls_use_their_own_served_metadata(monkeypatch MCPTool(name="status", description="Read status", inputSchema={"type": "object"}), ] client: Final = AsyncMock() + client.discovery_auth_fingerprint = AsyncMock(return_value="test-credential-hash") client.call_tool.return_value = CallToolResult(content=[TextContent(type="text", text="ok")]) manager.create_mcp_client = AsyncMock(return_value=client) - manager._fetch_tools_with_timeout = AsyncMock(return_value=upstream) + manager._fetch_tools_with_timeout = AsyncMock(return_value=ListToolsResult(tools=upstream)) guardrail: Final = _BridgeMetadataGuardrail() logger: Final = ProxyLogging(user_api_key_cache=DualCache()) monkeypatch.setattr(litellm, "callbacks", [guardrail])