From 4a6a387ca1e511e35858fee0c92fe3e3415d03ee Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 20:48:22 +0000 Subject: [PATCH 1/4] fix(mcp): follow nextCursor on paginated tools/prompts/resources list operations Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 1 + litellm/experimental_mcp_client/client.py | 56 ++++++----- litellm/experimental_mcp_client/pagination.py | 92 +++++++++++++++++++ litellm/experimental_mcp_client/tools.py | 7 +- .../mcp_server/rest_endpoints.py | 6 +- .../test_mcp_client.py | 52 ++++++++++- .../test_pagination.py | 80 ++++++++++++++++ 7 files changed, 258 insertions(+), 36 deletions(-) create mode 100644 litellm/experimental_mcp_client/pagination.py create mode 100644 tests/test_litellm/experimental_mcp_client/test_pagination.py diff --git a/litellm/constants.py b/litellm/constants.py index 9a50797f517..11f35177636 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -136,6 +136,7 @@ MCP_CLIENT_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_CLIENT_TIMEOUT", "60.0" MCP_TOOL_LISTING_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_TOOL_LISTING_TIMEOUT", "30.0")) MCP_METADATA_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_METADATA_TIMEOUT", "10.0")) MCP_HEALTH_CHECK_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_HEALTH_CHECK_TIMEOUT", "10.0")) +MCP_LIST_MAX_PAGES: Final = int(os.getenv("LITELLM_MCP_LIST_MAX_PAGES", "100")) # Allowlist of commands permitted for MCP stdio transport. # Prevents arbitrary command execution via /mcp-rest/test/* endpoints or server creation. diff --git a/litellm/experimental_mcp_client/client.py b/litellm/experimental_mcp_client/client.py index f0a1bff8fdc..814074d35a4 100644 --- a/litellm/experimental_mcp_client/client.py +++ b/litellm/experimental_mcp_client/client.py @@ -48,6 +48,12 @@ from pydantic import AnyUrl from litellm._logging import verbose_logger from litellm.constants import MCP_CLIENT_TIMEOUT, MCP_NPM_CACHE_DIR +from litellm.experimental_mcp_client.pagination import ( + list_all_prompts, + list_all_resource_templates, + list_all_resources, + list_all_tools, +) from litellm.llms.custom_httpx.http_handler import get_ssl_configuration from litellm.types.llms.custom_http import VerifyTypes from litellm.types.mcp import ( @@ -603,17 +609,17 @@ class MCPClient: """ verbose_logger.debug("MCP client listing tools from %s", self.server_url or "stdio") - async def _list_tools_operation(session: ClientSession): - return await session.list_tools() + async def _list_tools_operation(session: ClientSession) -> tuple[MCPTool, ...]: + return await list_all_tools(session, self.server_url or "stdio") try: - result: Final = await self.run_with_session(_list_tools_operation, quiet_on_error=raise_on_error) - tool_count: Final = len(result.tools) - tool_names: Final = [tool.name for tool in result.tools] + tools: Final = await self.run_with_session(_list_tools_operation, quiet_on_error=raise_on_error) + tool_count: Final = len(tools) + tool_names: Final = [tool.name for tool in tools] verbose_logger.info( "MCP client listed %s tools from %s: %s", tool_count, self.server_url or "stdio", tool_names ) - return result.tools + return list(tools) except asyncio.CancelledError: verbose_logger.warning("MCP client list_tools was cancelled") raise @@ -734,17 +740,17 @@ class MCPClient: """List available prompts from the server.""" verbose_logger.debug("MCP client listing tools from %s", self.server_url or "stdio") - async def _list_prompts_operation(session: ClientSession): - return await session.list_prompts() + async def _list_prompts_operation(session: ClientSession) -> tuple[Prompt, ...]: + return await list_all_prompts(session, self.server_url or "stdio") try: - result: Final = await self.run_with_session(_list_prompts_operation) - prompt_count: Final = len(result.prompts) - prompt_names: Final = [prompt.name for prompt in result.prompts] + prompts: Final = await self.run_with_session(_list_prompts_operation) + prompt_count: Final = len(prompts) + prompt_names: Final = [prompt.name for prompt in prompts] verbose_logger.info( - "MCP client listed %s tools from %s: %s", prompt_count, self.server_url or "stdio", prompt_names + "MCP client listed %s prompts from %s: %s", prompt_count, self.server_url or "stdio", prompt_names ) - return result.prompts + return list(prompts) except asyncio.CancelledError: verbose_logger.warning("MCP client list_prompts was cancelled") raise @@ -811,17 +817,17 @@ class MCPClient: """List available resources from the server.""" verbose_logger.debug("MCP client listing resources from %s", self.server_url or "stdio") - async def _list_resources_operation(session: ClientSession): - return await session.list_resources() + async def _list_resources_operation(session: ClientSession) -> tuple[Resource, ...]: + return await list_all_resources(session, self.server_url or "stdio") try: - result: Final = await self.run_with_session(_list_resources_operation) - resource_count: Final = len(result.resources) - resource_names: Final = [resource.name for resource in result.resources] + resources: Final = await self.run_with_session(_list_resources_operation) + resource_count: Final = len(resources) + resource_names: Final = [resource.name for resource in resources] verbose_logger.info( "MCP client listed %s resources from %s: %s", resource_count, self.server_url or "stdio", resource_names ) - return result.resources + return list(resources) except asyncio.CancelledError: verbose_logger.warning("MCP client list_resources was cancelled") raise @@ -847,20 +853,20 @@ class MCPClient: """List available resource templates from the server.""" verbose_logger.debug("MCP client listing resource templates from %s", self.server_url or "stdio") - async def _list_resource_templates_operation(session: ClientSession): - return await session.list_resource_templates() + async def _list_resource_templates_operation(session: ClientSession) -> tuple[ResourceTemplate, ...]: + return await list_all_resource_templates(session, self.server_url or "stdio") try: - result: Final = await self.run_with_session(_list_resource_templates_operation) - resource_template_count: Final = len(result.resourceTemplates) - resource_template_names: Final = [resourceTemplate.name for resourceTemplate in result.resourceTemplates] + resource_templates: Final = await self.run_with_session(_list_resource_templates_operation) + resource_template_count: Final = len(resource_templates) + resource_template_names: Final = [resource_template.name for resource_template in resource_templates] verbose_logger.info( "MCP client listed %s resource templates from %s: %s", resource_template_count, self.server_url or "stdio", resource_template_names, ) - return result.resourceTemplates + return list(resource_templates) except asyncio.CancelledError: verbose_logger.warning("MCP client list_resource_templates was cancelled") raise diff --git a/litellm/experimental_mcp_client/pagination.py b/litellm/experimental_mcp_client/pagination.py new file mode 100644 index 00000000000..8852715aba5 --- /dev/null +++ b/litellm/experimental_mcp_client/pagination.py @@ -0,0 +1,92 @@ +""" +Follows ``nextCursor`` on the paginated MCP list operations so a multi-page catalog is read in full. +""" + +from collections.abc import Awaitable, Callable, Sequence +from typing import Final, TypeVar + +from mcp import ClientSession, Resource +from mcp.types import PaginatedRequestParams, PaginatedResult, Prompt, ResourceTemplate +from mcp.types import Tool as MCPTool + +from litellm._logging import verbose_logger +from litellm.constants import MCP_LIST_MAX_PAGES + +TPage = TypeVar("TPage", bound=PaginatedResult) +TItem = TypeVar("TItem") + + +async def collect_pages( + fetch_page: Callable[[PaginatedRequestParams | None], Awaitable[TPage]], + items_of: Callable[[TPage], Sequence[TItem]], + *, + method: str, + server: str, + cursor: str | None = None, + seen_cursors: frozenset[str] = frozenset(), +) -> tuple[TItem, ...]: + page: Final = await fetch_page(None if cursor is None else PaginatedRequestParams(cursor=cursor)) + items: Final = tuple(items_of(page)) + next_cursor: Final = page.nextCursor + pages_read: Final = len(seen_cursors) + 1 + if next_cursor is None: + return items + if next_cursor in seen_cursors: + verbose_logger.warning( + "MCP %s from %s repeated cursor %r; returning the %s page(s) read so far", + method, + server, + next_cursor, + pages_read, + ) + return items + if pages_read >= MCP_LIST_MAX_PAGES: + verbose_logger.warning( + "MCP %s from %s still paginating after %s pages (LITELLM_MCP_LIST_MAX_PAGES); returning what was read", + method, + server, + pages_read, + ) + return items + rest: Final = await collect_pages( + fetch_page, + items_of, + method=method, + server=server, + cursor=next_cursor, + seen_cursors=seen_cursors | frozenset((next_cursor,)), + ) + return items + rest + + +async def list_all_tools(session: ClientSession, server: str) -> tuple[MCPTool, ...]: + return await collect_pages( + lambda params: session.list_tools(params=params), lambda page: page.tools, method="tools/list", server=server + ) + + +async def list_all_prompts(session: ClientSession, server: str) -> tuple[Prompt, ...]: + return await collect_pages( + lambda params: session.list_prompts(params=params), + lambda page: page.prompts, + method="prompts/list", + server=server, + ) + + +async def list_all_resources(session: ClientSession, server: str) -> tuple[Resource, ...]: + return await collect_pages( + lambda params: session.list_resources(params=params), + lambda page: page.resources, + method="resources/list", + server=server, + ) + + +async def list_all_resource_templates(session: ClientSession, server: str) -> tuple[ResourceTemplate, ...]: + return await collect_pages( + lambda params: session.list_resource_templates(params=params), + lambda page: page.resourceTemplates, + method="resources/templates/list", + server=server, + ) diff --git a/litellm/experimental_mcp_client/tools.py b/litellm/experimental_mcp_client/tools.py index 30d50e2a74b..adaca0888aa 100644 --- a/litellm/experimental_mcp_client/tools.py +++ b/litellm/experimental_mcp_client/tools.py @@ -9,6 +9,7 @@ from openai.types.chat import ChatCompletionToolParam from openai.types.responses.function_tool_param import FunctionToolParam from openai.types.shared_params.function_definition import FunctionDefinition +from litellm.experimental_mcp_client.pagination import list_all_tools from litellm.types.llms.anthropic import AnthropicMessagesTool from litellm.types.utils import ChatCompletionMessageToolCall @@ -103,10 +104,10 @@ async def load_mcp_tools( If format is set to "openai", the tools are converted to OpenAI API compatible tools. """ - tools: Final = await session.list_tools() + tools: Final = await list_all_tools(session, "upstream") if format == "openai": - return [transform_mcp_tool_to_openai_tool(mcp_tool=tool) for tool in tools.tools] - return tools.tools + return [transform_mcp_tool_to_openai_tool(mcp_tool=tool) for tool in tools] + return list(tools) ######################################################## diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index 3efb6429326..3ca6b6c5f90 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -1402,11 +1402,7 @@ if MCP_AVAILABLE: oauth2_headers = MCPRequestHandler._get_oauth2_headers_from_headers(headers) async def _list_tools_operation(client): - async def _list_tools_session_operation(session): - return await session.list_tools() - - list_tools_response: Final = await client.run_with_session(_list_tools_session_operation) - list_tools_result: Final[list[MCPTool]] = list_tools_response.tools + list_tools_result: Final[list[MCPTool]] = await client.list_tools(raise_on_error=True) model_dumped_tools: Final[list[dict]] = [tool.model_dump() for tool in list_tools_result] return { "tools": model_dumped_tools, diff --git a/tests/test_litellm/experimental_mcp_client/test_mcp_client.py b/tests/test_litellm/experimental_mcp_client/test_mcp_client.py index fd7ab3afdab..3f501d3859a 100644 --- a/tests/test_litellm/experimental_mcp_client/test_mcp_client.py +++ b/tests/test_litellm/experimental_mcp_client/test_mcp_client.py @@ -20,8 +20,10 @@ from mcp.types import ( JSONRPCError, JSONRPCMessage, JSONRPCResponse, + ListToolsResult, ServerCapabilities, ) +from mcp.types import Tool as MCPTool # Add the parent directory to the path so we can import litellm @@ -740,8 +742,14 @@ class _ScriptedUpstream: error, the shape an upstream application uses to report its own failure. """ - def __init__(self, tools_list_error: ErrorData | None = None): + def __init__( + self, + tools_list_error: ErrorData | None = None, + tool_pages: tuple[tuple[MCPTool, ...], ...] = (), + ): self._tools_list_error = tools_list_error + self._tool_pages = tool_pages + self.tools_list_cursors: list[str | None] = [] self._to_client_tx, self._to_client_rx = anyio.create_memory_object_stream(10) self._from_client_tx, self._from_client_rx = anyio.create_memory_object_stream(10) self._task_group = None @@ -778,15 +786,37 @@ class _ScriptedUpstream: ) elif method == "tools/list" and self._tools_list_error is not None: await self._send(JSONRPCError(jsonrpc="2.0", id=request.id, error=self._tools_list_error)) + elif method == "tools/list" and self._tool_pages: + cursor = (request.params or {}).get("cursor") + self.tools_list_cursors.append(cursor) + page_index = int(cursor) if cursor else 0 + has_more = page_index + 1 < len(self._tool_pages) + page = ListToolsResult( + tools=list(self._tool_pages[page_index]), + nextCursor=str(page_index + 1) if has_more else None, + ) + await self._send( + JSONRPCResponse( + jsonrpc="2.0", + id=request.id, + result=page.model_dump(by_alias=True, mode="json", exclude_none=True), + ) + ) class _ScriptedClient(MCPClient): """An MCPClient whose transport is a scripted in-memory upstream instead of a real connection, so the real ``ClientSession`` and its real timeout machinery are what run.""" - def __init__(self, *, timeout: float, tools_list_error: ErrorData | None = None): + def __init__( + self, + *, + timeout: float, + tools_list_error: ErrorData | None = None, + tool_pages: tuple[tuple[MCPTool, ...], ...] = (), + ): super().__init__(server_url="http://upstream.local/mcp", timeout=timeout) - self._upstream = _ScriptedUpstream(tools_list_error=tools_list_error) + self._upstream = _ScriptedUpstream(tools_list_error=tools_list_error, tool_pages=tool_pages) def _create_transport_context(self): return self._upstream, None @@ -821,6 +851,22 @@ async def test_list_tools_fails_on_its_own_timeout_when_the_upstream_never_answe assert list_fault_http_status(fault) == 504 +@pytest.mark.asyncio +async def test_list_tools_follows_tools_list_pagination_across_the_whole_catalog(): + """An upstream that pages tools/list (72 tools, 30 per page) must have every page read within the + one session, each request carrying the cursor the previous page returned. Reading only the first + page made 42 tools invisible to the proxy and every call to them fail as unknown.""" + tools = tuple( + MCPTool(name=f"tool_{i:02d}", inputSchema={"type": "object", "properties": {}}) for i in range(72) + ) + client = _ScriptedClient(timeout=30, tool_pages=(tools[:30], tools[30:60], tools[60:])) + + listed = await asyncio.wait_for(client.list_tools(raise_on_error=True), timeout=10) + + assert [tool.name for tool in listed] == [tool.name for tool in tools] + assert client._upstream.tools_list_cursors == [None, "1", "2"] + + @pytest.mark.asyncio async def test_upstream_json_rpc_error_408_is_not_reported_as_a_client_timeout(): """The SDK reports its own elapsed read timeout and relays an upstream JSON-RPC error through diff --git a/tests/test_litellm/experimental_mcp_client/test_pagination.py b/tests/test_litellm/experimental_mcp_client/test_pagination.py new file mode 100644 index 00000000000..f588fdd1eee --- /dev/null +++ b/tests/test_litellm/experimental_mcp_client/test_pagination.py @@ -0,0 +1,80 @@ +import logging + +import pytest +from mcp.types import ListToolsResult, PaginatedRequestParams +from mcp.types import Tool as MCPTool + +import litellm.experimental_mcp_client.pagination as pagination_module +from litellm.experimental_mcp_client.pagination import collect_pages + + +def _tool(index: int) -> MCPTool: + return MCPTool(name=f"tool_{index:02d}", inputSchema={"type": "object", "properties": {}}) + + +class _PagedTools: + """A tools/list upstream serving ``total`` tools ``page_size`` at a time, cursors being offsets.""" + + def __init__(self, total: int, page_size: int): + self._tools = tuple(_tool(i) for i in range(total)) + self._page_size = page_size + self.cursors_seen: list[str | None] = [] + + async def fetch(self, params: PaginatedRequestParams | None) -> ListToolsResult: + cursor = params.cursor if params is not None else None + self.cursors_seen.append(cursor) + start = int(cursor) if cursor else 0 + end = start + self._page_size + return ListToolsResult( + tools=list(self._tools[start:end]), + nextCursor=str(end) if end < len(self._tools) else None, + ) + + +@pytest.mark.asyncio +async def test_collect_pages_follows_next_cursor_until_exhausted(): + upstream = _PagedTools(total=72, page_size=30) + + tools = await collect_pages(upstream.fetch, lambda page: page.tools, method="tools/list", server="s") + + assert [t.name for t in tools] == [f"tool_{i:02d}" for i in range(72)] + assert upstream.cursors_seen == [None, "30", "60"], "each page must be requested with the cursor the previous one returned" + + +@pytest.mark.asyncio +async def test_collect_pages_single_page_makes_one_request(): + upstream = _PagedTools(total=5, page_size=30) + + tools = await collect_pages(upstream.fetch, lambda page: page.tools, method="tools/list", server="s") + + assert len(tools) == 5 + assert upstream.cursors_seen == [None] + + +@pytest.mark.asyncio +async def test_collect_pages_stops_on_a_repeated_cursor_and_keeps_what_it_read(caplog): + calls: list[str | None] = [] + + async def fetch(params: PaginatedRequestParams | None) -> ListToolsResult: + calls.append(params.cursor if params else None) + return ListToolsResult(tools=[_tool(len(calls))], nextCursor="same") + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + tools = await collect_pages(fetch, lambda page: page.tools, method="tools/list", server="s") + + assert calls == [None, "same"], "the cursor must be followed once and refused the second time it comes back" + assert len(tools) == 2 + assert any("repeated cursor" in record.getMessage() for record in caplog.records) + + +@pytest.mark.asyncio +async def test_collect_pages_honors_the_page_cap(monkeypatch, caplog): + monkeypatch.setattr(pagination_module, "MCP_LIST_MAX_PAGES", 3) + upstream = _PagedTools(total=1000, page_size=10) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + tools = await collect_pages(upstream.fetch, lambda page: page.tools, method="tools/list", server="s") + + assert len(upstream.cursors_seen) == 3 + assert len(tools) == 30 + assert any("LITELLM_MCP_LIST_MAX_PAGES" in record.getMessage() for record in caplog.records) From 26b48d58919e5021b9b251339bf5c720a3e3649e Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 21:00:27 +0000 Subject: [PATCH 2/4] refactor(mcp): keep list pagination within type-discipline budget and ratchet LIT001 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/experimental_mcp_client/client.py | 56 ++++++++++--------- .../mcp_server/rest_endpoints.py | 4 +- .../test_pagination.py | 4 +- type-discipline-budget.json | 2 +- 4 files changed, 35 insertions(+), 31 deletions(-) diff --git a/litellm/experimental_mcp_client/client.py b/litellm/experimental_mcp_client/client.py index 814074d35a4..46d8823e439 100644 --- a/litellm/experimental_mcp_client/client.py +++ b/litellm/experimental_mcp_client/client.py @@ -377,29 +377,31 @@ class MCPClient: return provided_env # Minimal allowlist of safe/standard environment variables - safe_keys: Final = { - "PATH", - "HOME", - "USER", - "LOGNAME", - "TMPDIR", - "TMP", - "TEMP", - "SHELL", - "LANG", - "LC_ALL", - # Node/Package manager caches - "NPM_CONFIG_CACHE", - "PNPM_HOME", - "XDG_CACHE_HOME", - "XDG_CONFIG_HOME", - "XDG_DATA_HOME", - # System info - "SYSTEMROOT", - "COMSPEC", - "PATHEXT", - "WINDIR", - } + safe_keys: Final = frozenset( + { + "PATH", + "HOME", + "USER", + "LOGNAME", + "TMPDIR", + "TMP", + "TEMP", + "SHELL", + "LANG", + "LC_ALL", + # Node/Package manager caches + "NPM_CONFIG_CACHE", + "PNPM_HOME", + "XDG_CACHE_HOME", + "XDG_CONFIG_HOME", + "XDG_DATA_HOME", + # System info + "SYSTEMROOT", + "COMSPEC", + "PATHEXT", + "WINDIR", + } + ) safe_env: Final = {} for key in safe_keys: @@ -615,7 +617,7 @@ class MCPClient: try: tools: Final = await self.run_with_session(_list_tools_operation, quiet_on_error=raise_on_error) tool_count: Final = len(tools) - tool_names: Final = [tool.name for tool in tools] + tool_names: Final = tuple(tool.name for tool in tools) verbose_logger.info( "MCP client listed %s tools from %s: %s", tool_count, self.server_url or "stdio", tool_names ) @@ -746,7 +748,7 @@ class MCPClient: try: prompts: Final = await self.run_with_session(_list_prompts_operation) prompt_count: Final = len(prompts) - prompt_names: Final = [prompt.name for prompt in prompts] + prompt_names: Final = tuple(prompt.name for prompt in prompts) verbose_logger.info( "MCP client listed %s prompts from %s: %s", prompt_count, self.server_url or "stdio", prompt_names ) @@ -823,7 +825,7 @@ class MCPClient: try: resources: Final = await self.run_with_session(_list_resources_operation) resource_count: Final = len(resources) - resource_names: Final = [resource.name for resource in resources] + resource_names: Final = tuple(resource.name for resource in resources) verbose_logger.info( "MCP client listed %s resources from %s: %s", resource_count, self.server_url or "stdio", resource_names ) @@ -859,7 +861,7 @@ class MCPClient: try: resource_templates: Final = await self.run_with_session(_list_resource_templates_operation) resource_template_count: Final = len(resource_templates) - resource_template_names: Final = [resource_template.name for resource_template in resource_templates] + resource_template_names: Final = tuple(resource_template.name for resource_template in resource_templates) verbose_logger.info( "MCP client listed %s resource templates from %s: %s", resource_template_count, diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index 3ca6b6c5f90..beee7c1db78 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -1,6 +1,6 @@ import asyncio import importlib -from collections.abc import Awaitable, Callable, Mapping +from collections.abc import Awaitable, Callable, Mapping, Sequence from datetime import datetime from typing import TYPE_CHECKING, Any, Final, Literal @@ -1402,7 +1402,7 @@ if MCP_AVAILABLE: oauth2_headers = MCPRequestHandler._get_oauth2_headers_from_headers(headers) async def _list_tools_operation(client): - list_tools_result: Final[list[MCPTool]] = await client.list_tools(raise_on_error=True) + list_tools_result: Final[Sequence[MCPTool]] = await client.list_tools(raise_on_error=True) model_dumped_tools: Final[list[dict]] = [tool.model_dump() for tool in list_tools_result] return { "tools": model_dumped_tools, diff --git a/tests/test_litellm/experimental_mcp_client/test_pagination.py b/tests/test_litellm/experimental_mcp_client/test_pagination.py index f588fdd1eee..a76f410ac3c 100644 --- a/tests/test_litellm/experimental_mcp_client/test_pagination.py +++ b/tests/test_litellm/experimental_mcp_client/test_pagination.py @@ -38,7 +38,9 @@ async def test_collect_pages_follows_next_cursor_until_exhausted(): tools = await collect_pages(upstream.fetch, lambda page: page.tools, method="tools/list", server="s") assert [t.name for t in tools] == [f"tool_{i:02d}" for i in range(72)] - assert upstream.cursors_seen == [None, "30", "60"], "each page must be requested with the cursor the previous one returned" + assert upstream.cursors_seen == [None, "30", "60"], ( + "each page must be requested with the cursor the previous one returned" + ) @pytest.mark.asyncio diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 3d2e97d55a5..b0c3cc7f9fd 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,6 +1,6 @@ { "LIT001": { - "limit": 22367 + "limit": 22366 }, "LIT002": { "limit": 26777 From d32f8a07c88ed165e55ce944026504a1cd15a327 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 21:15:10 +0000 Subject: [PATCH 3/4] fix(mcp): make list page cap a plain constant and use a real ListToolsResult in the unit mock Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 2 +- litellm/experimental_mcp_client/pagination.py | 2 +- tests/mcp_tests/test_mcp_client_unit.py | 6 ++---- .../test_litellm/experimental_mcp_client/test_pagination.py | 2 +- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 11f35177636..07914934495 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -136,7 +136,7 @@ MCP_CLIENT_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_CLIENT_TIMEOUT", "60.0" MCP_TOOL_LISTING_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_TOOL_LISTING_TIMEOUT", "30.0")) MCP_METADATA_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_METADATA_TIMEOUT", "10.0")) MCP_HEALTH_CHECK_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_HEALTH_CHECK_TIMEOUT", "10.0")) -MCP_LIST_MAX_PAGES: Final = int(os.getenv("LITELLM_MCP_LIST_MAX_PAGES", "100")) +MCP_LIST_MAX_PAGES: Final = 100 # Allowlist of commands permitted for MCP stdio transport. # Prevents arbitrary command execution via /mcp-rest/test/* endpoints or server creation. diff --git a/litellm/experimental_mcp_client/pagination.py b/litellm/experimental_mcp_client/pagination.py index 8852715aba5..85f46268f92 100644 --- a/litellm/experimental_mcp_client/pagination.py +++ b/litellm/experimental_mcp_client/pagination.py @@ -42,7 +42,7 @@ async def collect_pages( return items if pages_read >= MCP_LIST_MAX_PAGES: verbose_logger.warning( - "MCP %s from %s still paginating after %s pages (LITELLM_MCP_LIST_MAX_PAGES); returning what was read", + "MCP %s from %s still paginating after %s pages (MCP_LIST_MAX_PAGES); returning what was read", method, server, pages_read, diff --git a/tests/mcp_tests/test_mcp_client_unit.py b/tests/mcp_tests/test_mcp_client_unit.py index aadaadd510e..ef4231fe1d9 100644 --- a/tests/mcp_tests/test_mcp_client_unit.py +++ b/tests/mcp_tests/test_mcp_client_unit.py @@ -11,7 +11,7 @@ from unittest.mock import AsyncMock, MagicMock, patch, ANY import litellm.experimental_mcp_client.client as mcp_client_module from litellm.experimental_mcp_client.client import MCPClient from litellm.types.mcp import MCPAuth, MCPTransport -from mcp.types import Tool as MCPTool, CallToolResult as MCPCallToolResult +from mcp.types import Tool as MCPTool, CallToolResult as MCPCallToolResult, ListToolsResult def test_mcp_client_uses_configurable_default_timeout(): @@ -174,9 +174,7 @@ class TestMCPClientUnitTests: }, ) ] - mock_result = MagicMock() - mock_result.tools = mock_tools - mock_session_instance.list_tools.return_value = mock_result + mock_session_instance.list_tools.return_value = ListToolsResult(tools=mock_tools) client = MCPClient("http://example.com") result = await client.list_tools() diff --git a/tests/test_litellm/experimental_mcp_client/test_pagination.py b/tests/test_litellm/experimental_mcp_client/test_pagination.py index a76f410ac3c..93952c176a8 100644 --- a/tests/test_litellm/experimental_mcp_client/test_pagination.py +++ b/tests/test_litellm/experimental_mcp_client/test_pagination.py @@ -79,4 +79,4 @@ async def test_collect_pages_honors_the_page_cap(monkeypatch, caplog): assert len(upstream.cursors_seen) == 3 assert len(tools) == 30 - assert any("LITELLM_MCP_LIST_MAX_PAGES" in record.getMessage() for record in caplog.records) + assert any("MCP_LIST_MAX_PAGES" in record.getMessage() for record in caplog.records) From 875f015e24219110dbad35691d80acd1c1a3c375 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 19 Sep 2026 18:33:56 -0700 Subject: [PATCH 4/4] fix(token_counter): count replayed redacted_thinking blocks so prompt_caching keeps pinning A conversation that replays a redacted_thinking block (Anthropic redacted reasoning, or the /v1/messages bridge's stand-in for a reasoning item that carries no summary) made _count_content_list raise, is_prompt_caching_valid_prompt swallowed that to False, and the prompt_caching pre-call check neither recorded nor pinned the serving deployment, so the conversation bounced across the group and paid a cache write on every deployment. The block now counts like a thinking block with no text: zero tokens for the encrypted payload. --- litellm/litellm_core_utils/token_counter.py | 11 ++-- .../litellm_core_utils/test_token_counter.py | 19 +++++++ .../test_prompt_caching_deployment_check.py | 52 +++++++++++++++++++ 3 files changed, 79 insertions(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 6c1b7946394..bf37b1be2e4 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -46,6 +46,8 @@ from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionDocumentObject, ChatCompletionNamedToolChoiceParam, + ChatCompletionRedactedThinkingBlock, + ChatCompletionThinkingBlock, ChatCompletionToolParam, OpenAIMessageContentListBlock, ) @@ -854,6 +856,8 @@ def _count_content_list( content_list: str | Iterable[ OpenAIMessageContentListBlock + | ChatCompletionThinkingBlock + | ChatCompletionRedactedThinkingBlock | AnthropicMessagesTextParam | AnthropicMessagesImageParam | AnthropicMessagesDocumentParam @@ -898,9 +902,9 @@ def _count_content_list( use_default_image_token_count, default_token_count, ) - elif c["type"] == "thinking": + elif c["type"] in ("thinking", "redacted_thinking"): # Claude extended thinking content block - # Count the thinking text and skip signature (opaque signature blob) + # Count the thinking text and skip the opaque blobs (signature, redacted data) thinking_text = str(c.get("thinking", "")) if thinking_text: num_tokens += count_function(thinking_text) @@ -920,7 +924,8 @@ def _count_content_list( raise ValueError( f"Invalid content item type: {content_type}. " f"Expected str or dict with 'type' field " - f"(text, image_url, image, document, file, tool_use, tool_result, thinking, tool_reference)." + f"(text, image_url, image, document, file, tool_use, tool_result, thinking, redacted_thinking, " + f"tool_reference)." ) return num_tokens except Exception as e: diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index ba3a6be609f..f19a8891609 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -1257,6 +1257,25 @@ def test_token_counter_with_thinking_content(): ), f"Expected minimal token count for empty thinking block, got {tokens_no_thinking}" + +def test_token_counter_with_redacted_thinking_content(): + """ + A replayed redacted_thinking block (Anthropic redacted reasoning, or the /v1/messages bridge's stand-in + for a reasoning item with no summary) counts zero tokens for its encrypted payload, like a thinking + block with no text. It used to raise, which made is_prompt_caching_valid_prompt return False and the + prompt_caching pre-call check stop pinning the deployment that held the cached prefix. + """ + model = "anthropic/claude-sonnet-4-5-20250929" + reply = {"type": "text", "text": "Draw from the box labeled Mixed, because that label must be wrong."} + redacted_block = {"type": "redacted_thinking", "data": "EqQBCkYIBRgCKkBjZ2xhc3M" * 30} + user_turn = {"role": "user", "content": [{"type": "text", "text": "Which box do you draw from?"}]} + follow_up = {"role": "user", "content": [{"type": "text", "text": "Restate that in one sentence."}]} + + without_block = [user_turn, {"role": "assistant", "content": [reply]}, follow_up] + with_block = [user_turn, {"role": "assistant", "content": [redacted_block, reply]}, follow_up] + + assert token_counter(model=model, messages=with_block) == token_counter(model=model, messages=without_block) + def test_token_counter_with_tool_reference_block(): """ Regression test: a message containing an Anthropic tool-search diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py b/tests/test_litellm/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py index 333e7b2ff31..267109c9164 100644 --- a/tests/test_litellm/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py +++ b/tests/test_litellm/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py @@ -197,6 +197,58 @@ async def test_async_filter_deployments_narrows_for_group_whose_model_minimum_is AUTO_CACHING_MODEL = "anthropic/claude-sonnet-4-5" +@pytest.mark.asyncio +async def test_replayed_redacted_thinking_block_still_records_and_pins(): + """ + A model that returns no reasoning summary (gpt-5.x through the /v1/messages bridge, Anthropic with + redacted reasoning) hands the client a `redacted_thinking` block, and the client replays it on every + later turn. The token count behind `is_prompt_caching_valid_prompt` raised on that block, the helper + swallowed it to False, and the check neither recorded the serving deployment nor pinned it, so the + conversation bounced across the group and paid a cache write on each deployment. + """ + cache = DualCache() + check = PromptCachingDeploymentCheck(cache=cache) + model = "openai/gpt-5.6-sol" + deployments = _deployments(model, model, model) + messages = cast( + List[AllMessageValues], + [ + *_messages(word_count=3000), + { + "role": "assistant", + "content": [ + {"type": "redacted_thinking", "data": "litellm_encrypted_reasoning:" + "Z" * 400}, + {"type": "text", "text": "Draw from the box labeled Mixed."}, + ], + }, + {"role": "user", "content": "Restate that in one sentence."}, + ], + ) + + assert is_prompt_caching_valid_prompt(model=model, messages=messages) is True + + await check.async_log_success_event( + kwargs={ + "standard_logging_object": { + "call_type": "anthropic_messages", + "model": model, + "messages": messages, + "model_id": "dep-2", + } + }, + response_obj=None, + start_time=None, + end_time=None, + ) + filtered = await check.async_filter_deployments( + model=MODEL_GROUP_ALIAS, + healthy_deployments=deployments, + messages=messages, + ) + + assert filtered == [deployments[1]] + + def _auto_caching_messages() -> List[AllMessageValues]: """A prompt over the model minimum that carries no client cache_control.""" return cast(