From 5767a2da0f6d28a07cf431fbd652678b3223867b Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Tue, 1 Sep 2026 14:34:14 -0700 Subject: [PATCH] fix(mcp): follow tools/list pagination from upstream servers (#39172) * fix(mcp): follow tools/list pagination from upstream servers Adopts BerriAI/litellm#32244 by Jupiter363 onto litellm_internal_staging with merge conflicts resolved * fix(mcp): degrade buggy pagination to partial results and bound the preview walk A repeated nextCursor now returns the tools collected so far instead of discarding every page with a RuntimeError, an empty-string cursor is treated as terminal, load_mcp_tools shares the same pagination walk instead of returning only the first page, and the tools/list preview is bounded by the listing timeout instead of only the per-request timeout times the page cap * fix(mcp): annotate deliberate rebind for the preview timeout scope * fix(mcp): bound the shared pagination walk with an overall listing deadline The per-request session read timeout restarts on every page, so direct SDK callers of list_tools and load_mcp_tools could run up to the page cap with no overall bound. The walk now returns the tools collected so far when max(MCP_CLIENT_TIMEOUT, MCP_TOOL_LISTING_TIMEOUT) expires * fix(mcp): let a per-server timeout extend the pagination deadline MCPClient carries a per-server timeout that can exceed the global default; list_tools now passes max(self.timeout, MCP_TOOL_LISTING_TIMEOUT) into the shared walk so a deliberately slow server is not silently truncated at the global deadline * fix(mcp): honor per-server timeouts in the preview deadline and test the walk sessionless The preview deadline now extends with the created client's own timeout, and the pagination walk's cap, repeated-cursor, and empty-cursor cases are tested directly against the helper instead of through patched SDK internals * fix(mcp): forward the preview request's per-server timeout to the temporary server model The tools preview built its temporary MCPServer without the request's timeout field, so the client factory always fell back to the global default and a per-server timeout could never extend the preview's listing deadline (or its per-request timeout). --- litellm/constants.py | 1 + litellm/experimental_mcp_client/client.py | 20 +- litellm/experimental_mcp_client/tools.py | 74 ++++- .../mcp_server/rest_endpoints.py | 32 +- tests/mcp_tests/test_mcp_client_unit.py | 78 ++++- .../experimental_mcp_client/test_tools.py | 130 ++++++++ .../mcp_server/test_rest_endpoints.py | 294 +++++++++++++++++- 7 files changed, 601 insertions(+), 28 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 9a50797f517..172ee70fd57 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -136,6 +136,7 @@ MCP_CLIENT_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_CLIENT_TIMEOUT", "60.0" MCP_TOOL_LISTING_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_TOOL_LISTING_TIMEOUT", "30.0")) MCP_METADATA_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_METADATA_TIMEOUT", "10.0")) MCP_HEALTH_CHECK_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_HEALTH_CHECK_TIMEOUT", "10.0")) +MCP_TOOL_LISTING_MAX_PAGES: Final = 1000 # Allowlist of commands permitted for MCP stdio transport. # Prevents arbitrary command execution via /mcp-rest/test/* endpoints or server creation. diff --git a/litellm/experimental_mcp_client/client.py b/litellm/experimental_mcp_client/client.py index f0a1bff8fdc..ea81e323da4 100644 --- a/litellm/experimental_mcp_client/client.py +++ b/litellm/experimental_mcp_client/client.py @@ -7,6 +7,7 @@ import base64 import os from collections.abc import Awaitable, Callable, Generator from datetime import timedelta +from functools import partial from importlib import metadata from typing import Any, Final, TypeVar @@ -47,7 +48,8 @@ from mcp.types import Tool as MCPTool from pydantic import AnyUrl from litellm._logging import verbose_logger -from litellm.constants import MCP_CLIENT_TIMEOUT, MCP_NPM_CACHE_DIR +from litellm.constants import MCP_CLIENT_TIMEOUT, MCP_NPM_CACHE_DIR, MCP_TOOL_LISTING_TIMEOUT +from litellm.experimental_mcp_client.tools import list_tools_with_pagination from litellm.llms.custom_httpx.http_handler import get_ssl_configuration from litellm.types.llms.custom_http import VerifyTypes from litellm.types.mcp import ( @@ -603,17 +605,19 @@ class MCPClient: """ verbose_logger.debug("MCP client listing tools from %s", self.server_url or "stdio") - async def _list_tools_operation(session: ClientSession): - return await session.list_tools() - try: - result: Final = await self.run_with_session(_list_tools_operation, quiet_on_error=raise_on_error) - tool_count: Final = len(result.tools) - tool_names: Final = [tool.name for tool in result.tools] + # A per-server timeout above the global default extends the whole-walk deadline + listing_deadline: Final = max(self.timeout, MCP_TOOL_LISTING_TIMEOUT) + tools: Final = await self.run_with_session( + partial(list_tools_with_pagination, listing_deadline=listing_deadline), + quiet_on_error=raise_on_error, + ) + tool_count: Final = len(tools) + tool_names: Final = tuple(tool.name for tool in tools) verbose_logger.info( "MCP client listed %s tools from %s: %s", tool_count, self.server_url or "stdio", tool_names ) - return result.tools + return tools except asyncio.CancelledError: verbose_logger.warning("MCP client list_tools was cancelled") raise diff --git a/litellm/experimental_mcp_client/tools.py b/litellm/experimental_mcp_client/tools.py index 30d50e2a74b..51d2139ef3b 100644 --- a/litellm/experimental_mcp_client/tools.py +++ b/litellm/experimental_mcp_client/tools.py @@ -1,14 +1,22 @@ import json from typing import Final, Literal +import anyio from mcp import ClientSession from mcp.types import CallToolRequestParams as MCPCallToolRequestParams from mcp.types import CallToolResult as MCPCallToolResult +from mcp.types import PaginatedRequestParams from mcp.types import Tool as MCPTool from openai.types.chat import ChatCompletionToolParam from openai.types.responses.function_tool_param import FunctionToolParam from openai.types.shared_params.function_definition import FunctionDefinition +from litellm._logging import verbose_logger +from litellm.constants import ( + MCP_CLIENT_TIMEOUT, + MCP_TOOL_LISTING_MAX_PAGES, + MCP_TOOL_LISTING_TIMEOUT, +) from litellm.types.llms.anthropic import AnthropicMessagesTool from litellm.types.utils import ChatCompletionMessageToolCall @@ -90,6 +98,64 @@ def transform_mcp_tool_to_anthropic_tool(mcp_tool: MCPTool) -> AnthropicMessages ) +async def list_tools_with_pagination( + session: ClientSession, listing_deadline: float | None = None +) -> list[MCPTool]: # mutable-ok: list return contract + """Collect tools from every tools/list page by following nextCursor. + + Stops and returns the tools collected so far when the upstream repeats a + cursor, the page cap is reached, or the whole-walk deadline expires, so a + buggy or slow upstream yields a partial catalog instead of an error. + listing_deadline overrides the default whole-walk deadline; callers with a + per-server timeout above the global default pass it through here. + """ + tools: Final[list[MCPTool]] = [] # mutable-ok: accumulates each page's tools + seen_cursors: Final[set[str]] = set() # mutable-ok: guards against cursor loops + cursor: str | None = None # rebind-ok: advances to each page's nextCursor + # The per-request session read timeout restarts on every page, so a multi-page + # walk needs its own overall deadline. max() keeps the pre-pagination guarantee + # that a single page slower than the listing timeout but within the client + # timeout still succeeds. + effective_deadline: Final = ( + listing_deadline if listing_deadline is not None else max(MCP_CLIENT_TIMEOUT, MCP_TOOL_LISTING_TIMEOUT) + ) + + with anyio.move_on_after(effective_deadline): + for _ in range(MCP_TOOL_LISTING_MAX_PAGES): + result = ( + await session.list_tools() + if cursor is None + else await session.list_tools(params=PaginatedRequestParams(cursor=cursor)) + ) + tools.extend(result.tools) + + next_cursor = getattr(result, "nextCursor", None) + if not isinstance(next_cursor, str) or not next_cursor: + return tools + if next_cursor in seen_cursors: + verbose_logger.warning( + "MCP server repeated a tools/list cursor while listing tools; returning %s tools collected so far", + len(tools), + ) + return tools + seen_cursors.add(next_cursor) + cursor = next_cursor + + verbose_logger.warning( + "MCP server tools/list pagination exceeded the maximum of %s pages; returning %s tools collected so far", + MCP_TOOL_LISTING_MAX_PAGES, + len(tools), + ) + return tools + + verbose_logger.warning( + "MCP server tools/list pagination exceeded the %s second listing deadline; returning %s tools collected so far", + effective_deadline, + len(tools), + ) + return tools + + async def load_mcp_tools( session: ClientSession, format: Literal["mcp", "openai"] = "mcp" ) -> list[MCPTool] | list[ChatCompletionToolParam]: @@ -103,10 +169,12 @@ async def load_mcp_tools( If format is set to "openai", the tools are converted to OpenAI API compatible tools. """ - tools: Final = await session.list_tools() + tools: Final = await list_tools_with_pagination(session) if format == "openai": - return [transform_mcp_tool_to_openai_tool(mcp_tool=tool) for tool in tools.tools] - return tools.tools + return [ # mutable-ok: public API returns a list + transform_mcp_tool_to_openai_tool(mcp_tool=tool) for tool in tools + ] + return tools ######################################################## diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index 3efb6429326..2b89dba0e4f 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -4,10 +4,12 @@ from collections.abc import Awaitable, Callable, Mapping from datetime import datetime from typing import TYPE_CHECKING, Any, Final, Literal +import anyio import httpx from fastapi import APIRouter, Depends, HTTPException, Query, Request, status from litellm._logging import verbose_logger +from litellm.constants import MCP_CLIENT_TIMEOUT, MCP_TOOL_LISTING_TIMEOUT from litellm.exceptions import ( BlockedPiiEntityError, GuardrailRaisedException, @@ -86,8 +88,6 @@ def _connection_error_message(exc: BaseException) -> str: if MCP_AVAILABLE: - from mcp.types import Tool as MCPTool - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES, global_mcp_server_manager, @@ -1173,6 +1173,7 @@ if MCP_AVAILABLE: transport=request.transport, auth_type=request.auth_type, mcp_info=request.mcp_info, + timeout=request.timeout, command=request.command, args=request.args, env=request.env, @@ -1402,11 +1403,28 @@ if MCP_AVAILABLE: oauth2_headers = MCPRequestHandler._get_oauth2_headers_from_headers(headers) async def _list_tools_operation(client): - async def _list_tools_session_operation(session): - return await session.list_tools() - - list_tools_response: Final = await client.run_with_session(_list_tools_session_operation) - list_tools_result: Final[list[MCPTool]] = list_tools_response.tools + # Bound the whole pagination walk: without this the preview is limited only by the + # per-request timeout times the page cap. max() keeps the pre-pagination guarantee + # that a single slow page within the client timeout still succeeds, and a + # per-server timeout above the global default extends the deadline with it. + listing_deadline: Final = max( + getattr(client, "timeout", MCP_CLIENT_TIMEOUT) or MCP_CLIENT_TIMEOUT, + MCP_TOOL_LISTING_TIMEOUT, + ) + list_tools_result = None # rebind-ok: set inside the timeout scope below + with anyio.move_on_after(listing_deadline): + list_tools_result = await client.list_tools(raise_on_error=True) # rebind-ok: fills the init above + if list_tools_result is None: + verbose_logger.warning( + "MCP tools/list preview timed out after %s seconds while paginating upstream tools", + listing_deadline, + ) + return { # mutable-ok: error response payload + "status": "error", + "error": True, + "message": f"Timed out listing tools after {listing_deadline} seconds. " + "The MCP server may be responding slowly or paginating excessively.", + } model_dumped_tools: Final[list[dict]] = [tool.model_dump() for tool in list_tools_result] return { "tools": model_dumped_tools, diff --git a/tests/mcp_tests/test_mcp_client_unit.py b/tests/mcp_tests/test_mcp_client_unit.py index aadaadd510e..6438525706a 100644 --- a/tests/mcp_tests/test_mcp_client_unit.py +++ b/tests/mcp_tests/test_mcp_client_unit.py @@ -11,7 +11,9 @@ from unittest.mock import AsyncMock, MagicMock, patch, ANY import litellm.experimental_mcp_client.client as mcp_client_module from litellm.experimental_mcp_client.client import MCPClient from litellm.types.mcp import MCPAuth, MCPTransport -from mcp.types import Tool as MCPTool, CallToolResult as MCPCallToolResult +from mcp.types import CallToolResult as MCPCallToolResult +from mcp.types import ListToolsResult, PaginatedRequestParams +from mcp.types import Tool as MCPTool def test_mcp_client_uses_configurable_default_timeout(): @@ -185,6 +187,80 @@ class TestMCPClientUnitTests: mock_session_instance.initialize.assert_called_once() mock_session_instance.list_tools.assert_called_once() + @pytest.mark.asyncio + @patch.object(mcp_client_module, "streamable_http_client") # test-quality-ok: exercises MCPClient wiring; the walk itself is covered sessionless in test_tools.py + @patch.object(mcp_client_module, "ClientSession") # test-quality-ok: exercises MCPClient wiring; the walk itself is covered sessionless in test_tools.py + async def test_list_tools_follows_next_cursor_until_exhausted( + self, + mock_session_class, + mock_transport, + ): + """Test listing tools follows MCP pagination cursors until exhausted.""" + mock_transport_ctx = AsyncMock() + mock_transport.return_value = mock_transport_ctx + mock_transport_instance = MagicMock() + mock_transport_ctx.__aenter__ = AsyncMock(return_value=mock_transport_instance) + + mock_session_ctx = AsyncMock() + mock_session_class.return_value = mock_session_ctx + mock_session_instance = AsyncMock() + mock_session_ctx.__aenter__ = AsyncMock(return_value=mock_session_instance) + + first_page_tools = [ + MCPTool(name=f"tool_{idx}", description=f"Tool {idx}", inputSchema={}) for idx in range(100) + ] + second_page_tool = MCPTool( + name="tool_100", + description="Tool 100", + inputSchema={}, + ) + mock_session_instance.list_tools.side_effect = [ + ListToolsResult(tools=first_page_tools, nextCursor="page-2"), + ListToolsResult(tools=[second_page_tool]), + ] + + client = MCPClient("http://example.com") + result = await client.list_tools() + + assert result == [*first_page_tools, second_page_tool] + assert mock_session_instance.list_tools.call_count == 2 + second_call_params = mock_session_instance.list_tools.call_args_list[1].kwargs["params"] + assert isinstance(second_call_params, PaginatedRequestParams) + assert second_call_params.cursor == "page-2" + + @pytest.mark.asyncio + @patch.object(mcp_client_module, "streamable_http_client") # test-quality-ok: exercises MCPClient wiring; the walk itself is covered sessionless in test_tools.py + @patch.object(mcp_client_module, "ClientSession") # test-quality-ok: exercises MCPClient wiring; the walk itself is covered sessionless in test_tools.py + async def test_list_tools_swallows_mid_walk_error_without_raise_on_error( + self, + mock_session_class, + mock_transport, + ): + """Test a mid-walk failure returns [] when raise_on_error is False.""" + mock_transport_ctx = AsyncMock() + mock_transport.return_value = mock_transport_ctx + mock_transport_instance = MagicMock() + mock_transport_ctx.__aenter__ = AsyncMock(return_value=mock_transport_instance) + + mock_session_ctx = AsyncMock() + mock_session_class.return_value = mock_session_ctx + mock_session_instance = AsyncMock() + mock_session_ctx.__aenter__ = AsyncMock(return_value=mock_session_instance) + + mock_session_instance.list_tools.side_effect = [ + ListToolsResult( + tools=[MCPTool(name="tool_0", description="Tool 0", inputSchema={})], + nextCursor="page-2", + ), + RuntimeError("transient upstream failure"), + ] + + client = MCPClient("http://example.com") + result = await client.list_tools() + + assert result == [] + assert mock_session_instance.list_tools.call_count == 2 + @pytest.mark.asyncio @patch.object(mcp_client_module, "streamable_http_client") @patch.object(mcp_client_module, "ClientSession") diff --git a/tests/test_litellm/experimental_mcp_client/test_tools.py b/tests/test_litellm/experimental_mcp_client/test_tools.py index 89f67452f29..6645b06664d 100644 --- a/tests/test_litellm/experimental_mcp_client/test_tools.py +++ b/tests/test_litellm/experimental_mcp_client/test_tools.py @@ -8,11 +8,13 @@ from mcp.types import ( CallToolRequestParams, CallToolResult, ListToolsResult, + PaginatedRequestParams, TextContent, ) from mcp.types import Tool as MCPTool from litellm.experimental_mcp_client.tools import ( + list_tools_with_pagination, transform_mcp_tool_to_anthropic_tool, _get_function_arguments, _normalize_mcp_input_schema, @@ -106,6 +108,134 @@ async def test_load_mcp_tools_openai_format(mock_session, mock_list_tools_result mock_session.list_tools.assert_called_once() +@pytest.mark.asyncio() +async def test_load_mcp_tools_follows_pagination(mock_session): + mock_session.list_tools.side_effect = [ + ListToolsResult( + tools=[ + MCPTool(name="tool_a", description="a", inputSchema={}), + MCPTool(name="tool_b", description="b", inputSchema={}), + ], + nextCursor="page-2", + ), + ListToolsResult(tools=[MCPTool(name="tool_c", description="c", inputSchema={})]), + ] + result = await load_mcp_tools(mock_session, format="mcp") + assert [tool.name for tool in result] == ["tool_a", "tool_b", "tool_c"] + assert mock_session.list_tools.call_count == 2 + second_call_params = mock_session.list_tools.call_args_list[1].kwargs["params"] + assert isinstance(second_call_params, PaginatedRequestParams) + assert second_call_params.cursor == "page-2" + + +@pytest.mark.asyncio() +async def test_pagination_walk_stops_at_page_cap(mock_session, monkeypatch): + monkeypatch.setattr("litellm.experimental_mcp_client.tools.MCP_TOOL_LISTING_MAX_PAGES", 2) + mock_session.list_tools.side_effect = [ + ListToolsResult( + tools=[MCPTool(name="tool_0", description="0", inputSchema={})], + nextCursor="page-2", + ), + ListToolsResult( + tools=[MCPTool(name="tool_1", description="1", inputSchema={})], + nextCursor="page-3", + ), + ListToolsResult(tools=[MCPTool(name="tool_2", description="2", inputSchema={})]), + ] + result = await list_tools_with_pagination(mock_session) + assert [tool.name for tool in result] == ["tool_0", "tool_1"] + assert mock_session.list_tools.call_count == 2 + + +@pytest.mark.asyncio() +async def test_pagination_walk_stops_on_repeated_cursor(mock_session): + mock_session.list_tools.side_effect = [ + ListToolsResult( + tools=[MCPTool(name="tool_0", description="0", inputSchema={})], + nextCursor="same-cursor", + ), + ListToolsResult( + tools=[MCPTool(name="tool_1", description="1", inputSchema={})], + nextCursor="same-cursor", + ), + ] + result = await list_tools_with_pagination(mock_session) + assert [tool.name for tool in result] == ["tool_0", "tool_1"] + assert mock_session.list_tools.call_count == 2 + + +@pytest.mark.asyncio() +async def test_pagination_walk_treats_empty_cursor_as_terminal(mock_session): + mock_session.list_tools.side_effect = [ + ListToolsResult( + tools=[MCPTool(name="tool_0", description="0", inputSchema={})], + nextCursor="", + ), + ] + result = await list_tools_with_pagination(mock_session) + assert [tool.name for tool in result] == ["tool_0"] + mock_session.list_tools.assert_called_once() + + +@pytest.mark.asyncio() +async def test_pagination_walk_stops_at_whole_walk_deadline(mock_session, monkeypatch): + import anyio + + from litellm.experimental_mcp_client.tools import list_tools_with_pagination + + monkeypatch.setattr("litellm.experimental_mcp_client.tools.MCP_CLIENT_TIMEOUT", 0.2) + monkeypatch.setattr("litellm.experimental_mcp_client.tools.MCP_TOOL_LISTING_TIMEOUT", 0.2) + + async def slow_page(params=None): + await anyio.sleep(0.15) + idx = int(params.cursor) if params is not None else 0 + return ListToolsResult( + tools=[MCPTool(name=f"tool_{idx}", description=str(idx), inputSchema={})], + nextCursor=str(idx + 1), + ) + + mock_session.list_tools = slow_page + result = await list_tools_with_pagination(mock_session) + + assert [tool.name for tool in result] == ["tool_0"] + + +@pytest.mark.asyncio() +async def test_pagination_walk_honors_explicit_deadline_over_globals(mock_session, monkeypatch): + import anyio + + from litellm.experimental_mcp_client.tools import list_tools_with_pagination + + monkeypatch.setattr("litellm.experimental_mcp_client.tools.MCP_CLIENT_TIMEOUT", 0.1) + monkeypatch.setattr("litellm.experimental_mcp_client.tools.MCP_TOOL_LISTING_TIMEOUT", 0.1) + + async def slow_page(params=None): + await anyio.sleep(0.15) + idx = int(params.cursor) if params is not None else 0 + tools = [MCPTool(name=f"tool_{idx}", description=str(idx), inputSchema={})] + if idx == 0: + return ListToolsResult(tools=tools, nextCursor="1") + return ListToolsResult(tools=tools) + + mock_session.list_tools = slow_page + result = await list_tools_with_pagination(mock_session, listing_deadline=2.0) + + assert [tool.name for tool in result] == ["tool_0", "tool_1"] + + +@pytest.mark.asyncio() +async def test_load_mcp_tools_openai_format_spans_pages(mock_session): + mock_session.list_tools.side_effect = [ + ListToolsResult( + tools=[MCPTool(name="tool_a", description="a", inputSchema={})], + nextCursor="page-2", + ), + ListToolsResult(tools=[MCPTool(name="tool_b", description="b", inputSchema={})]), + ] + result = await load_mcp_tools(mock_session, format="openai") + assert [t["function"]["name"] for t in result] == ["tool_a", "tool_b"] + + def test_get_function_arguments(): # Test with string arguments function = {"arguments": '{"test": "value"}'} diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py index ef5631218f3..e36bef229f6 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py @@ -214,6 +214,46 @@ class TestExecuteWithMcpClient: assert server.scopes == ["read", "write"] assert server.has_client_credentials is True + async def test_preview_forwards_per_server_timeout_to_client_factory(self, monkeypatch): + """The request's per-server timeout must reach the temporary MCPServer model: + the client factory reads ``server.timeout`` for both the per-request timeout + and the preview's whole-walk listing deadline.""" + captured: dict = {} + + def fake_build_stdio_env(server, raw_headers): + return None + + async def fake_create_client(*args, **kwargs): + captured["server"] = kwargs.get("server") + return object() + + monkeypatch.setattr( + rest_endpoints.global_mcp_server_manager, + "_build_stdio_env", + fake_build_stdio_env, + raising=False, + ) + monkeypatch.setattr( + rest_endpoints.global_mcp_server_manager, + "_create_mcp_client", + fake_create_client, + raising=False, + ) + + async def ok_operation(client): + return {"status": "ok"} + + payload = NewMCPServerRequest( + server_name="slow-catalog-server", + url="https://example.com", + timeout=120.5, + ) + + result = await rest_endpoints._execute_with_mcp_client(payload, ok_operation) + + assert result["status"] == "ok" + assert captured["server"].timeout == 120.5 + @pytest.mark.asyncio async def test_m2m_drops_incoming_oauth2_headers(self, monkeypatch): """For M2M OAuth servers the incoming Authorization header (which carries @@ -524,6 +564,131 @@ class TestTestToolsList: assert captured["oauth2_headers"] is None assert oauth_call_counter["count"] == 0 + async def test_preview_tools_list_times_out_on_slow_pagination(self, monkeypatch): + """A preview whose upstream paginates past the listing deadline returns a + timeout error instead of holding the request open.""" + monkeypatch.setattr(rest_endpoints, "MCP_CLIENT_TIMEOUT", 0.05, raising=False) + monkeypatch.setattr(rest_endpoints, "MCP_TOOL_LISTING_TIMEOUT", 0.05, raising=False) + + class SlowClient: + async def list_tools(self, raise_on_error=False): + await asyncio.sleep(1) + return [] + + async def fake_execute( + request, + operation, + mcp_auth_header=None, + oauth2_headers=None, + raw_headers=None, + ): + return await operation(SlowClient()) + + monkeypatch.setattr(rest_endpoints, "_execute_with_mcp_client", fake_execute, raising=False) + + from litellm.proxy._types import LitellmUserRoles + + request = _build_request() + payload = NewMCPServerRequest( + server_name="example", + url="https://example.com", + auth_type=MCPAuth.api_key, + credentials={"auth_value": "secret-key"}, + ) + + result = await rest_endpoints.test_tools_list( + request, + payload, + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + assert result["status"] == "error" + assert result["error"] is True + assert "Timed out listing tools" in result["message"] + + async def test_preview_tools_list_succeeds_within_deadline(self, monkeypatch): + """The preview timeout scope passes a fast listing through untouched.""" + from mcp.types import Tool as MCPTool + + class QuickClient: + async def list_tools(self, raise_on_error=False): + return [MCPTool(name="quick_tool", description="q", inputSchema={})] + + async def fake_execute( + request, + operation, + mcp_auth_header=None, + oauth2_headers=None, + raw_headers=None, + ): + return await operation(QuickClient()) + + monkeypatch.setattr(rest_endpoints, "_execute_with_mcp_client", fake_execute, raising=False) + + from litellm.proxy._types import LitellmUserRoles + + request = _build_request() + payload = NewMCPServerRequest( + server_name="example", + url="https://example.com", + auth_type=MCPAuth.api_key, + credentials={"auth_value": "secret-key"}, + ) + + result = await rest_endpoints.test_tools_list( + request, + payload, + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + assert result["error"] is None + assert result["message"] == "Successfully retrieved tools" + assert [tool["name"] for tool in result["tools"]] == ["quick_tool"] + + async def test_preview_tools_list_honors_per_server_timeout(self, monkeypatch): + """A per-server timeout above the global default extends the preview deadline.""" + monkeypatch.setattr(rest_endpoints, "MCP_CLIENT_TIMEOUT", 0.05, raising=False) + monkeypatch.setattr(rest_endpoints, "MCP_TOOL_LISTING_TIMEOUT", 0.05, raising=False) + + from mcp.types import Tool as MCPTool + + class SlowConfiguredClient: + timeout = 1.0 + + async def list_tools(self, raise_on_error=False): + await asyncio.sleep(0.2) + return [MCPTool(name="slow_tool", description="s", inputSchema={})] + + async def fake_execute( + request, + operation, + mcp_auth_header=None, + oauth2_headers=None, + raw_headers=None, + ): + return await operation(SlowConfiguredClient()) + + monkeypatch.setattr(rest_endpoints, "_execute_with_mcp_client", fake_execute, raising=False) + + from litellm.proxy._types import LitellmUserRoles + + request = _build_request() + payload = NewMCPServerRequest( + server_name="example", + url="https://example.com", + auth_type=MCPAuth.api_key, + credentials={"auth_value": "secret-key"}, + ) + + result = await rest_endpoints.test_tools_list( + request, + payload, + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + assert result["error"] is None + assert [tool["name"] for tool in result["tools"]] == ["slow_tool"] + async def test_extracts_oauth2_headers(self, monkeypatch): """Ensure oauth2 auth type pulls oauth headers and omits MCP auth header.""" @@ -786,9 +951,7 @@ class TestListToolsRestAPI: they do for a gateway session, never to the bare session key.""" from litellm.constants import UI_SESSION_TOKEN_TEAM_ID - session_auth = UserAPIKeyAuth( - team_id=UI_SESSION_TOKEN_TEAM_ID, user_id="grant-user", user_role="internal_user" - ) + session_auth = UserAPIKeyAuth(team_id=UI_SESSION_TOKEN_TEAM_ID, user_id="grant-user", user_role="internal_user") admitted_auth = UserAPIKeyAuth(user_id="grant-user", org_id="admitted-org") async def fake_reload(user_id): @@ -868,9 +1031,7 @@ class TestListToolsRestAPI: from litellm.constants import UI_SESSION_TOKEN_TEAM_ID from litellm.proxy._types import LiteLLM_ObjectPermissionTable - session_auth = UserAPIKeyAuth( - team_id=UI_SESSION_TOKEN_TEAM_ID, user_id="grant-user", user_role="internal_user" - ) + session_auth = UserAPIKeyAuth(team_id=UI_SESSION_TOKEN_TEAM_ID, user_id="grant-user", user_role="internal_user") scoped_auth = UserAPIKeyAuth( object_permission=LiteLLM_ObjectPermissionTable( object_permission_id="toolset-scope", @@ -952,6 +1113,123 @@ class TestListToolsRestAPI: assert scope_inputs == [session_auth] assert reload_calls == [] + async def test_single_server_response_includes_paginated_upstream_tools( + self, + monkeypatch, + ): + """The REST tools/list path should include tools beyond the upstream first page.""" + import litellm.experimental_mcp_client.client as mcp_client_module + from mcp.types import ListToolsResult, PaginatedRequestParams + from mcp.types import Tool as MCPTool + + from litellm.proxy._experimental.mcp_server.server import MCPServer + from litellm.types.mcp import MCPTransport + + async def fake_contexts(user_api_key_auth): + return [user_api_key_auth] + + async def fake_get_allowed_mcp_servers(*args, **kwargs): + return ["server-1"] + + stub_server = MCPServer( + server_id="server-1", + name="stub", + server_name="stub", + alias="stub", + url="https://example.com/mcp", + transport=MCPTransport.http, + mcp_info={"server_name": "stub"}, + ) + stub_server.available_on_public_internet = True + + mock_transport_ctx = AsyncMock() + mock_transport_ctx.__aenter__ = AsyncMock(return_value=(MagicMock(), MagicMock())) + mock_transport_ctx.__aexit__ = AsyncMock(return_value=None) + monkeypatch.setattr( + mcp_client_module, + "streamable_http_client", + MagicMock(return_value=mock_transport_ctx), + raising=False, + ) + + mock_session_ctx = AsyncMock() + mock_session_instance = AsyncMock() + mock_session_instance.initialize = AsyncMock(return_value=None) + mock_session_instance.list_tools.side_effect = [ + ListToolsResult( + tools=[ + MCPTool( + name="first_page_tool", + description="First page tool", + inputSchema={}, + ) + ], + nextCursor="page-2", + ), + ListToolsResult( + tools=[ + MCPTool( + name="second_page_tool", + description="Second page tool", + inputSchema={}, + ) + ] + ), + ] + mock_session_ctx.__aenter__ = AsyncMock(return_value=mock_session_instance) + mock_session_ctx.__aexit__ = AsyncMock(return_value=None) + monkeypatch.setattr( + mcp_client_module, + "ClientSession", + MagicMock(return_value=mock_session_ctx), + raising=False, + ) + + monkeypatch.setattr( + rest_endpoints, + "build_effective_auth_contexts", + fake_contexts, + raising=False, + ) + monkeypatch.setattr( + rest_endpoints.global_mcp_server_manager, + "get_allowed_mcp_servers", + fake_get_allowed_mcp_servers, + raising=False, + ) + monkeypatch.setattr( + rest_endpoints.global_mcp_server_manager, + "filter_server_ids_by_ip_with_info", + lambda server_ids, client_ip: (server_ids, 0), + raising=False, + ) + monkeypatch.setattr( + rest_endpoints.global_mcp_server_manager, + "get_mcp_server_by_id", + lambda server_id: stub_server if server_id == "server-1" else None, + raising=False, + ) + + request = _build_request(path="/mcp-rest/tools/list", method="GET") + result = await rest_endpoints.list_tool_rest_api( + request, + server_id="server-1", + user_api_key_dict=UserAPIKeyAuth(), + ) + + assert set(result.keys()) == {"tools", "error", "message"} + assert [tool.name for tool in result["tools"]] == [ + "first_page_tool", + "second_page_tool", + ] + assert result["error"] is None + assert result["message"] == "Successfully retrieved tools" + + assert mock_session_instance.list_tools.call_count == 2 + second_call_params = mock_session_instance.list_tools.call_args_list[1].kwargs["params"] + assert isinstance(second_call_params, PaginatedRequestParams) + assert second_call_params.cursor == "page-2" + async def test_include_disabled_tools_is_admin_only(self, monkeypatch): """include_disabled_tools skips the allowlist filter only for PROXY_ADMIN; a non-admin passing it stays filtered so the REST endpoint can't be used @@ -3021,9 +3299,7 @@ class TestRestListToolsetFiltering: mock_manager = MagicMock() mock_manager.expand_tool_permissions = MagicMock(side_effect=lambda perms: perms or {}) - mock_manager.resolve_toolset_tool_permissions = AsyncMock( - return_value={"server-a": ["lookup_status"]} - ) + mock_manager.resolve_toolset_tool_permissions = AsyncMock(return_value={"server-a": ["lookup_status"]}) monkeypatch.setattr( rest_endpoints.global_mcp_server_manager,