From fc1582872574c2c1241a607fb1163299f3d16ad9 Mon Sep 17 00:00:00 2001 From: yassin Date: Wed, 21 Oct 2026 10:11:04 +0000 Subject: [PATCH 1/5] fix(mcp): count initialize and tools/list against the key rpm limit Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/_experimental/mcp_server/server.py | 42 +++++-- tests/integration/mcp/test_mcp_rate_limits.py | 80 +++++++++++++ .../mcp_server/test_mcp_server.py | 113 ++++++++++++++++++ 3 files changed, 222 insertions(+), 13 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index b3426089375..ffe9a6070af 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -115,6 +115,7 @@ if TYPE_CHECKING: _redact_mcp_resource_url: Final = redact_mcp_resource_url +_MCP_SETUP_REQUEST_METHODS: Final = frozenset({"initialize", "tools/list"}) _STATEFUL_SESSION_IDLE_TIMEOUT_SECONDS: Final = 30 * 60 # Upper bound on concurrent stateful sessions a single caller may hold. Each # `initialize` creates a session that survives until the idle timeout, so @@ -135,6 +136,22 @@ _MCP_PROTOCOL_VERSION_HEADER: Final = b"mcp-protocol-version" _DECODED_JSON: Final = TypeAdapter(object) +def _mcp_setup_request_capacity( + method: str | None, + user_api_key_auth: UserAPIKeyAuth | None, +) -> contextlib.AbstractAsyncContextManager[None]: + if method not in _MCP_SETUP_REQUEST_METHODS or user_api_key_auth is None: + return contextlib.nullcontext() + + from litellm.proxy.hooks.parallel_request_limiter_v3 import PROXY_MaxParallelRequestsHandler_v3 + from litellm.proxy.proxy_server import proxy_logging_obj + + limiter: Final = proxy_logging_obj.get_proxy_hook("parallel_request_limiter") + if not isinstance(limiter, PROXY_MaxParallelRequestsHandler_v3): + return contextlib.nullcontext() + return limiter.request_capacity(user_api_key_auth, "mcp-tool-call", request_data={}) + + def reject_disallowed_mcp_origin(request: StarletteRequest) -> None: from litellm.proxy.proxy_server import origins # noqa: PLC0415 # proxy imports this module during startup @@ -1393,18 +1410,15 @@ if MCP_AVAILABLE: return b"{}" return json.dumps({"method": method}).encode() - def _is_initialize_request(body: bytes) -> bool: - """ - Check if the request body is a JSON-RPC initialize method. - Returns True if method is "initialize", False otherwise or on parse error. - """ + def _mcp_request_method(body: bytes) -> str | None: if not body: - return False + return None try: data: Final = as_str_mapping(_DECODED_JSON.validate_python(json.loads(body))) - return data is not None and data.get("method") == "initialize" + method: Final = data.get("method") if data is not None else None + return method if isinstance(method, str) else None except (json.JSONDecodeError, UnicodeDecodeError, TypeError): - return False + return None def _jsonrpc_request_method(body: bytes) -> str | None: try: @@ -2423,7 +2437,8 @@ if MCP_AVAILABLE: ) = await extract_mcp_auth_context(scope, path) admit_mcp_request(scope, user_api_key_auth, mcp_servers) body: Final = await peek.body() if peek is not None else b"" - is_initialize: Final = _is_initialize_request(body) + mcp_method: Final = _mcp_request_method(body) + is_initialize: Final = mcp_method == "initialize" scoped_server_endpoint: Final = len(_get_mcp_servers_in_path(path) or []) == 1 request_headers: Final = StarletteRequest(scope).headers defer_upstream_probes: Final = request_headers.get( @@ -2705,11 +2720,12 @@ if MCP_AVAILABLE: if use_stateful and session_id and scope.get("method") == "DELETE": _remove_stateful_session_tracking(session_id) + request_capacity: Final = _mcp_setup_request_capacity(mcp_method, user_api_key_auth) try: - if session_lock is not None: - async with session_lock: - await _dispatch() - else: + async with ( + session_lock if session_lock is not None else contextlib.nullcontext(), + request_capacity, + ): await _dispatch() finally: for active_request_session_id in active_request_session_ids: diff --git a/tests/integration/mcp/test_mcp_rate_limits.py b/tests/integration/mcp/test_mcp_rate_limits.py index 273419d624f..b872bd5c740 100644 --- a/tests/integration/mcp/test_mcp_rate_limits.py +++ b/tests/integration/mcp/test_mcp_rate_limits.py @@ -269,3 +269,83 @@ def test_mcp_key_team_and_server_rpm_limits_share_redis(tmp_path: Path, monkeypa server_rejected: Final = _call_tool(second_replica, key_three, server_id, "rpm-add") _assert_rate_limit(server_rejected, "mcp_server") assert tool_calls(peer.drain()) == () + + +def test_streamable_mcp_setup_requests_use_virtual_key_rpm_limit( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + results_directory: Final = tmp_path / "results" + results_directory.mkdir() + monkeypatch.setenv("INTEGRATION_RESULTS_DIR", str(results_directory)) + assert os.environ.get("DATABASE_URL"), "This integration case requires disposable-database access" + + with ( + owned_redis(tmp_path) as redis, + scratch_database() as database_url, + mcp_peer() as peer, + httpx.Client() as client, + ): + monkeypatch.setenv("DATABASE_URL", database_url) + seed: Final = Gateway(client, "sk-mcp-streamable-setup-rate-limit", peer.url) + config: Final = _config_file( + tmp_path, + seed.key, + redis, + upstream=peer.url, + store_model_in_db=True, + ) + environment: Final = { + "DATABASE_URL": database_url, + "LITELLM_SALT_KEY": "mcp-streamable-setup-rate-limit", + "LITELLM_RATE_LIMIT_WINDOW_SIZE": "30", + } + options: Final = { + "config": config, + "remove_environment": ("DATABASE_URL_READ_REPLICA", "LITELLM_LICENSE", "LITELLM_LICENSE_PATH"), + } + with owned_proxy(seed, tmp_path / "proxy", environment, **options) as gateway, gateway.scenario() as scenario: + server_id: Final = register_mcp(scenario, peer, "rpm", allowed_tools=["add"]) + key: Final = scenario.key(rpm_limit=2, object_permission={"mcp_servers": [server_id]}) + + def post_mcp(payload: dict[str, object], session_id: str | None = None) -> httpx.Response: + headers: Final = { + "Authorization": f"Bearer {key}", + "Accept": "application/json, text/event-stream", + **({"mcp-session-id": session_id} if session_id is not None else {}), + } + return gateway.client.post("/mcp/", headers=headers, json=payload) + + initialize: Final = post_mcp( + { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-03-26", + "capabilities": {}, + "clientInfo": {"name": "litellm-test", "version": "1"}, + }, + } + ) + assert initialize.status_code == 200, initialize.text + session_id: Final = initialize.headers["mcp-session-id"] + + initialized: Final = post_mcp( + {"jsonrpc": "2.0", "method": "notifications/initialized"}, + session_id, + ) + assert initialized.status_code == 202, initialized.text + + first_list: Final = post_mcp( + {"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}}, + session_id, + ) + assert first_list.status_code == 200, first_list.text + + second_list: Final = post_mcp( + {"jsonrpc": "2.0", "id": 3, "method": "tools/list", "params": {}}, + session_id, + ) + assert second_list.status_code == 429, second_list.text + assert second_list.headers.get("retry-after") is not None diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py index de711d737d0..48c022246a3 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py @@ -2,6 +2,7 @@ import asyncio import json import os +from collections.abc import AsyncIterator, Awaitable, Callable from typing import Final import pytest @@ -17,6 +18,7 @@ from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( MCPTransport, ) from litellm.proxy._types import LiteLLM_ObjectPermissionTable +from litellm.proxy._types import UserAPIKeyAuth from mcp.types import Tool as MCPTool, CallToolResult, ListToolsResult from mcp.types import TextContent @@ -3259,6 +3261,117 @@ async def test_mcp_routing_batch_body_is_not_stashed_for_auth(): assert await _read_request_body(_admission_request(scope)) == {} +@asynccontextmanager +async def _streamable_mcp_request_runner( + monkeypatch: pytest.MonkeyPatch, + auth: UserAPIKeyAuth, +) -> AsyncIterator[tuple[Callable[[str], Awaitable[None]], AsyncMock]]: + from litellm.caching.dual_cache import DualCache + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server import server + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.hooks.parallel_request_limiter_v3 import PROXY_MaxParallelRequestsHandler_v3 + from litellm.proxy.utils import InternalUsageCache, ProxyLogging + + proxy_logging: Final = ProxyLogging(user_api_key_cache=UserApiKeyCache()) + limiter: Final = PROXY_MaxParallelRequestsHandler_v3( + internal_usage_cache=InternalUsageCache(dual_cache=DualCache()) + ) + proxy_logging.proxy_hook_mapping["parallel_request_limiter"] = limiter + monkeypatch.setattr(proxy_server, "proxy_logging_obj", proxy_logging) + + dispatch: Final = AsyncMock() + session_manager: Final = MagicMock() + session_manager.handle_request = dispatch + monkeypatch.setattr(server, "_SESSION_MANAGERS_INITIALIZED", True) + monkeypatch.setattr(server, "session_manager_stateless", session_manager) + monkeypatch.setattr(server, "session_manager_stateful", session_manager) + monkeypatch.setattr( + server, + "extract_mcp_auth_context", + AsyncMock(return_value=(auth, None, ["echo_server"], {}, {}, {})), + ) + + @asynccontextmanager + async def no_initialize_scope(*_args: object, **_kwargs: object) -> AsyncIterator[None]: + yield + + monkeypatch.setattr(server, "_gateway_initialize_instructions_request_scope", no_initialize_scope) + + async def request(method: str) -> None: + body: Final = json.dumps( + { + "jsonrpc": "2.0", + "method": method, + **({"id": 1} if method != "notifications/initialized" else {}), + } + ).encode() + scope: Final[Scope] = { + "type": "http", + "method": "POST", + "path": "/mcp/echo_server", + "headers": [(b"content-type", b"application/json")], + "query_string": b"", + "server": ("localhost", 4000), + "scheme": "http", + } + receive: Final[Receive] = AsyncMock(return_value={"type": "http.request", "body": body, "more_body": False}) + send: Final[Send] = AsyncMock() + await server.handle_streamable_http_mcp(scope, receive, send) + + yield request, dispatch + + +@pytest.mark.asyncio +async def test_streamable_mcp_setup_requests_obey_virtual_key_rpm_limit(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + + auth: Final = UserAPIKeyAuth(api_key="mcp-setup-rpm-key", rpm_limit=2) + async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch): + await request("initialize") + await request("tools/list") + with pytest.raises(ProxyRateLimitError) as rate_limit_error: + await request("tools/list") + + assert rate_limit_error.value.status_code == 429 + assert rate_limit_error.value.headers.get("retry-after") is not None + assert dispatch.await_count == 2 + + +@pytest.mark.asyncio +async def test_streamable_mcp_tools_call_is_not_charged_by_setup_limiter(monkeypatch: pytest.MonkeyPatch) -> None: + auth: Final = UserAPIKeyAuth(api_key="mcp-tools-call-rpm-key", rpm_limit=1) + async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch): + await request("tools/call") + await request("initialize") + + assert dispatch.await_count == 2 + + +@pytest.mark.asyncio +async def test_streamable_mcp_initialized_notification_does_not_consume_rpm( + monkeypatch: pytest.MonkeyPatch, +) -> None: + auth: Final = UserAPIKeyAuth(api_key="mcp-initialized-rpm-key", rpm_limit=1) + async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch): + await request("notifications/initialized") + await request("initialize") + + assert dispatch.await_count == 2 + + +@pytest.mark.asyncio +async def test_streamable_mcp_setup_request_releases_parallel_request_slot( + monkeypatch: pytest.MonkeyPatch, +) -> None: + auth: Final = UserAPIKeyAuth(api_key="mcp-setup-parallel-key", max_parallel_requests=1) + async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch): + await request("initialize") + await request("tools/list") + + assert dispatch.await_count == 2 + + @pytest.mark.asyncio async def test_mcp_routing_truncated_json_body_is_not_stashed_for_auth(): try: From a59a9320b198e56a4cd924ecd2a5d667840ad951 Mon Sep 17 00:00:00 2001 From: yassin Date: Wed, 21 Oct 2026 10:51:39 +0000 Subject: [PATCH 2/5] fix(mcp): charge oversized setup requests from the full body --- .../proxy/_experimental/mcp_server/server.py | 7 ++- .../mcp_server/test_mcp_server.py | 46 +++++++++++++++---- .../test_mcp_server_tool_calls_and_headers.py | 27 +++++------ 3 files changed, 56 insertions(+), 24 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index ffe9a6070af..f9e4da27079 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -2720,7 +2720,12 @@ if MCP_AVAILABLE: if use_stateful and session_id and scope.get("method") == "DELETE": _remove_stateful_session_tracking(session_id) - request_capacity: Final = _mcp_setup_request_capacity(mcp_method, user_api_key_auth) + charged_method: Final = ( + _jsonrpc_request_method(await peek.full_body()) + if mcp_method is None and peek is not None and len(body) >= _MCP_ROUTING_PEEK_MAX_BYTES + else mcp_method + ) + request_capacity: Final = _mcp_setup_request_capacity(charged_method, user_api_key_auth) try: async with ( session_lock if session_lock is not None else contextlib.nullcontext(), diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py index 48c022246a3..9ba743a2f2d 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py @@ -3282,6 +3282,7 @@ async def _streamable_mcp_request_runner( dispatch: Final = AsyncMock() session_manager: Final = MagicMock() + session_manager.max_request_body_size = server.session_manager_stateless.max_request_body_size session_manager.handle_request = dispatch monkeypatch.setattr(server, "_SESSION_MANAGERS_INITIALIZED", True) monkeypatch.setattr(server, "session_manager_stateless", session_manager) @@ -3298,14 +3299,18 @@ async def _streamable_mcp_request_runner( monkeypatch.setattr(server, "_gateway_initialize_instructions_request_scope", no_initialize_scope) - async def request(method: str) -> None: - body: Final = json.dumps( - { - "jsonrpc": "2.0", - "method": method, - **({"id": 1} if method != "notifications/initialized" else {}), - } - ).encode() + async def request(method: str, request_body: bytes | None = None) -> None: + body: Final = ( + request_body + if request_body is not None + else json.dumps( + { + "jsonrpc": "2.0", + "method": method, + **({"id": 1} if method != "notifications/initialized" else {}), + } + ).encode() + ) scope: Final[Scope] = { "type": "http", "method": "POST", @@ -3338,6 +3343,31 @@ async def test_streamable_mcp_setup_requests_obey_virtual_key_rpm_limit(monkeypa assert dispatch.await_count == 2 +@pytest.mark.asyncio +async def test_streamable_mcp_oversized_setup_request_obeys_virtual_key_rpm_limit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + + auth: Final = UserAPIKeyAuth(api_key="mcp-oversized-setup-rpm-key", rpm_limit=1) + async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch): + await request("tools/list") + oversized_body: Final = json.dumps( + { + "jsonrpc": "2.0", + "id": 2, + "params": {"_meta": {"pad": "x" * 5000}}, + "method": "tools/list", + } + ).encode() + with pytest.raises(ProxyRateLimitError) as rate_limit_error: + await request("tools/list", oversized_body) + + assert rate_limit_error.value.status_code == 429 + assert rate_limit_error.value.headers.get("retry-after") is not None + assert dispatch.await_count == 1 + + @pytest.mark.asyncio async def test_streamable_mcp_tools_call_is_not_charged_by_setup_limiter(monkeypatch: pytest.MonkeyPatch) -> None: auth: Final = UserAPIKeyAuth(api_key="mcp-tools-call-rpm-key", rpm_limit=1) diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py index b8e28c3f9b8..e644f6781f2 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py @@ -2388,13 +2388,10 @@ async def test_mcp_routing_chunked_initialize_to_stateful(): @pytest.mark.asyncio -async def test_mcp_routing_caps_body_peek_for_oversized_chunked_body(): +async def test_mcp_routing_full_body_fallback_replays_oversized_chunked_body(): """ - A no-session-id POST with a very large chunked body should not force - the proxy to buffer the entire body just to decide routing — the peek - should stop once ``_MCP_ROUTING_PEEK_MAX_BYTES`` worth of body has been - consumed, and the remaining chunks should stream through the original - receive into the downstream handler. + A no-session-id POST with a truncated routing peek should read the full + body for charge resolution, then replay every chunk to the downstream handler. """ try: from litellm.proxy._experimental.mcp_server import server as mcp_server @@ -2471,12 +2468,8 @@ async def test_mcp_routing_caps_body_peek_for_oversized_chunked_body(): ): await handle_streamable_http_mcp(scope, receive, send) - # The routing peek must stop pulling from the wire once the cap is reached. - # Without the cap fix, every chunk would have been pulled before dispatch, - # so this assertion guards against unbounded pre-dispatch buffering. - assert receive_count_at_dispatch["value"] == 1, ( - "routing should stop reading after the peek cap is filled, " - f"but consumed {receive_count_at_dispatch['value']} chunks before dispatching" + assert receive_count_at_dispatch["value"] == len(messages), ( + "charge resolution should read every chunk after the routing peek is truncated" ) # All chunks must still reach the downstream handler via replay+stream. total_streamed = sum(len(b) for b in stateless_received_chunks) @@ -2521,7 +2514,7 @@ async def test_mcp_routing_peek_survives_multibyte_char_split_at_cap( received: Final[asyncio.Future[bytes]] = asyncio.get_running_loop().create_future() async def handle_request(incoming: Scope, downstream_receive: Receive, outgoing: Send) -> None: - assert receive.await_count == (2 if chunked else 1) + assert receive.await_count == len(messages) received.set_result(await _drain_body(downstream_receive)) async def dispatch(context): @@ -2533,12 +2526,16 @@ async def test_mcp_routing_peek_survives_multibyte_char_split_at_cap( stateless_handle: Final = AsyncMock(side_effect=handle_request) stateful_handle: Final = AsyncMock() + max_request_body_size: Final = mcp_module.session_manager_stateless.max_request_body_size scope: Final[Scope] = {"type": "http", "method": "POST", "path": "/mcp", "headers": []} with ( _client_allowlist_patches({}, None), patch( "litellm.proxy._experimental.mcp_server.server.session_manager_stateless", - SimpleNamespace(handle_request=stateless_handle), + SimpleNamespace( + handle_request=stateless_handle, + max_request_body_size=max_request_body_size, + ), ), patch( "litellm.proxy._experimental.mcp_server.server.session_manager_stateful", @@ -3006,7 +3003,7 @@ def test_oversized_initialize_peek_neither_routes_stateful_nor_attributes_client peeked = full_body[: mcp_server._MCP_ROUTING_PEEK_MAX_BYTES] assert mcp_server._extract_initialize_client_info(full_body) is not None - assert mcp_server._is_initialize_request(peeked) is False + assert mcp_server._mcp_request_method(peeked) is None assert mcp_server._extract_initialize_client_info(peeked) is None From 179176d68530914c5eff085f60a6c80574c7b442 Mon Sep 17 00:00:00 2001 From: yassin Date: Wed, 21 Oct 2026 11:14:32 +0000 Subject: [PATCH 3/5] fix(mcp): read the full body only for rate-limited setup requests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/_experimental/mcp_server/server.py | 82 +++++++++++++++++-- .../mcp_server/test_mcp_server.py | 19 +++++ .../test_mcp_server_tool_calls_and_headers.py | 25 +++--- 3 files changed, 110 insertions(+), 16 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index f9e4da27079..77c8d260511 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -136,17 +136,78 @@ _MCP_PROTOCOL_VERSION_HEADER: Final = b"mcp-protocol-version" _DECODED_JSON: Final = TypeAdapter(object) +def _mcp_setup_request_limiter(user_api_key_auth: UserAPIKeyAuth | None) -> object | None: + if user_api_key_auth is None: + return None + + from litellm.proxy.hooks.parallel_request_limiter_v3 import ( + PROXY_MaxParallelRequestsHandler_v3, + RateLimitedModel, + ) + from litellm.proxy.proxy_server import llm_router, proxy_logging_obj + from litellm.router_utils.common_utils import resolve_model_group_alias + + limiter: Final = proxy_logging_obj.get_proxy_hook("parallel_request_limiter") + if not isinstance(limiter, PROXY_MaxParallelRequestsHandler_v3): + return None + + if user_api_key_auth.api_key and ( + user_api_key_auth.rpm_limit is not None or user_api_key_auth.max_parallel_requests is not None + ): + return limiter + if user_api_key_auth.user_id and user_api_key_auth.user_rpm_limit is not None: + return limiter + if user_api_key_auth.team_id and user_api_key_auth.team_rpm_limit is not None: + return limiter + if user_api_key_auth.user_id and user_api_key_auth.team_member_rpm_limit is not None: + return limiter + if user_api_key_auth.org_id and user_api_key_auth.organization_rpm_limit is not None: + return limiter + if user_api_key_auth.end_user_id and user_api_key_auth.end_user_rpm_limit is not None: + return limiter + + from litellm.proxy.auth.auth_utils import get_key_model_rpm_limit, get_model_rate_limit_from_metadata + + model_name: Final = "mcp-tool-call" + model_group: Final = ( + resolve_model_group_alias(llm_router.model_group_alias, model_name) if llm_router is not None else None + ) + model: Final = RateLimitedModel(requested=model_name, group=model_group or model_name) + key_model_rpm_limit: Final = model.limit_in(get_key_model_rpm_limit(user_api_key_auth, model_name=model.group)) + if key_model_rpm_limit is not None: + return limiter + + team_model_rpm_limit: Final = model.limit_in( + get_model_rate_limit_from_metadata(user_api_key_auth, "team_metadata", "model_rpm_limit") + ) + if team_model_rpm_limit is not None: + return limiter + + organization_model_rpm_limit: Final = model.limit_in( + get_model_rate_limit_from_metadata(user_api_key_auth, "organization_metadata", "model_rpm_limit") + ) + if organization_model_rpm_limit is not None: + return limiter + + project_model_rpm_limit: Final = model.limit_in( + get_model_rate_limit_from_metadata(user_api_key_auth, "project_metadata", "model_rpm_limit") + ) + if project_model_rpm_limit is not None: + return limiter + + return None + + def _mcp_setup_request_capacity( method: str | None, user_api_key_auth: UserAPIKeyAuth | None, + limiter: object | None, ) -> contextlib.AbstractAsyncContextManager[None]: - if method not in _MCP_SETUP_REQUEST_METHODS or user_api_key_auth is None: + if method not in _MCP_SETUP_REQUEST_METHODS or user_api_key_auth is None or limiter is None: return contextlib.nullcontext() from litellm.proxy.hooks.parallel_request_limiter_v3 import PROXY_MaxParallelRequestsHandler_v3 - from litellm.proxy.proxy_server import proxy_logging_obj - limiter: Final = proxy_logging_obj.get_proxy_hook("parallel_request_limiter") if not isinstance(limiter, PROXY_MaxParallelRequestsHandler_v3): return contextlib.nullcontext() return limiter.request_capacity(user_api_key_auth, "mcp-tool-call", request_data={}) @@ -2720,12 +2781,23 @@ if MCP_AVAILABLE: if use_stateful and session_id and scope.get("method") == "DELETE": _remove_stateful_session_tracking(session_id) + setup_request_limiter: Final = ( + _mcp_setup_request_limiter(user_api_key_auth) + if mcp_method in _MCP_SETUP_REQUEST_METHODS + or (mcp_method is None and peek is not None and len(body) >= _MCP_ROUTING_PEEK_MAX_BYTES) + else None + ) charged_method: Final = ( _jsonrpc_request_method(await peek.full_body()) - if mcp_method is None and peek is not None and len(body) >= _MCP_ROUTING_PEEK_MAX_BYTES + if mcp_method is None + and peek is not None + and len(body) >= _MCP_ROUTING_PEEK_MAX_BYTES + and setup_request_limiter is not None else mcp_method ) - request_capacity: Final = _mcp_setup_request_capacity(charged_method, user_api_key_auth) + request_capacity: Final = _mcp_setup_request_capacity( + charged_method, user_api_key_auth, setup_request_limiter + ) try: async with ( session_lock if session_lock is not None else contextlib.nullcontext(), diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py index 9ba743a2f2d..d33dc412b4c 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py @@ -3368,6 +3368,25 @@ async def test_streamable_mcp_oversized_setup_request_obeys_virtual_key_rpm_limi assert dispatch.await_count == 1 +@pytest.mark.asyncio +async def test_streamable_mcp_oversized_setup_request_dispatches_without_virtual_key_limits( + monkeypatch: pytest.MonkeyPatch, +) -> None: + auth: Final = UserAPIKeyAuth(api_key="mcp-oversized-unlimited-key") + async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch): + oversized_body: Final = json.dumps( + { + "jsonrpc": "2.0", + "id": 2, + "params": {"_meta": {"pad": "x" * 5000}}, + "method": "tools/list", + } + ).encode() + await request("tools/list", oversized_body) + + assert dispatch.await_count == 1 + + @pytest.mark.asyncio async def test_streamable_mcp_tools_call_is_not_charged_by_setup_limiter(monkeypatch: pytest.MonkeyPatch) -> None: auth: Final = UserAPIKeyAuth(api_key="mcp-tools-call-rpm-key", rpm_limit=1) diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py index e644f6781f2..81ad107b6b0 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py @@ -2388,10 +2388,13 @@ async def test_mcp_routing_chunked_initialize_to_stateful(): @pytest.mark.asyncio -async def test_mcp_routing_full_body_fallback_replays_oversized_chunked_body(): +async def test_mcp_routing_caps_body_peek_for_oversized_chunked_body(): """ - A no-session-id POST with a truncated routing peek should read the full - body for charge resolution, then replay every chunk to the downstream handler. + A no-session-id POST with a very large chunked body should not force + the proxy to buffer the entire body just to decide routing — the peek + should stop once ``_MCP_ROUTING_PEEK_MAX_BYTES`` worth of body has been + consumed, and the remaining chunks should stream through the original + receive into the downstream handler. """ try: from litellm.proxy._experimental.mcp_server import server as mcp_server @@ -2468,8 +2471,12 @@ async def test_mcp_routing_full_body_fallback_replays_oversized_chunked_body(): ): await handle_streamable_http_mcp(scope, receive, send) - assert receive_count_at_dispatch["value"] == len(messages), ( - "charge resolution should read every chunk after the routing peek is truncated" + # The routing peek must stop pulling from the wire once the cap is reached. + # Without the cap fix, every chunk would have been pulled before dispatch, + # so this assertion guards against unbounded pre-dispatch buffering. + assert receive_count_at_dispatch["value"] == 1, ( + "routing should stop reading after the peek cap is filled, " + f"but consumed {receive_count_at_dispatch['value']} chunks before dispatching" ) # All chunks must still reach the downstream handler via replay+stream. total_streamed = sum(len(b) for b in stateless_received_chunks) @@ -2514,7 +2521,7 @@ async def test_mcp_routing_peek_survives_multibyte_char_split_at_cap( received: Final[asyncio.Future[bytes]] = asyncio.get_running_loop().create_future() async def handle_request(incoming: Scope, downstream_receive: Receive, outgoing: Send) -> None: - assert receive.await_count == len(messages) + assert receive.await_count == (2 if chunked else 1) received.set_result(await _drain_body(downstream_receive)) async def dispatch(context): @@ -2526,16 +2533,12 @@ async def test_mcp_routing_peek_survives_multibyte_char_split_at_cap( stateless_handle: Final = AsyncMock(side_effect=handle_request) stateful_handle: Final = AsyncMock() - max_request_body_size: Final = mcp_module.session_manager_stateless.max_request_body_size scope: Final[Scope] = {"type": "http", "method": "POST", "path": "/mcp", "headers": []} with ( _client_allowlist_patches({}, None), patch( "litellm.proxy._experimental.mcp_server.server.session_manager_stateless", - SimpleNamespace( - handle_request=stateless_handle, - max_request_body_size=max_request_body_size, - ), + SimpleNamespace(handle_request=stateless_handle), ), patch( "litellm.proxy._experimental.mcp_server.server.session_manager_stateful", From 36b18ceecff19e61025fb777fd75b53e1de16e01 Mon Sep 17 00:00:00 2001 From: yassin Date: Wed, 21 Oct 2026 11:28:59 +0000 Subject: [PATCH 4/5] refactor(mcp): narrow the full-body charge gate to direct rate-limit fields Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/_experimental/mcp_server/server.py | 85 +++---------------- 1 file changed, 14 insertions(+), 71 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 77c8d260511..f8645fc4884 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -136,78 +136,29 @@ _MCP_PROTOCOL_VERSION_HEADER: Final = b"mcp-protocol-version" _DECODED_JSON: Final = TypeAdapter(object) -def _mcp_setup_request_limiter(user_api_key_auth: UserAPIKeyAuth | None) -> object | None: - if user_api_key_auth is None: - return None - - from litellm.proxy.hooks.parallel_request_limiter_v3 import ( - PROXY_MaxParallelRequestsHandler_v3, - RateLimitedModel, +def _has_request_rate_limit(user_api_key_auth: UserAPIKeyAuth) -> bool: + return ( + user_api_key_auth.rpm_limit is not None + or user_api_key_auth.max_parallel_requests is not None + or user_api_key_auth.user_rpm_limit is not None + or user_api_key_auth.team_rpm_limit is not None + or user_api_key_auth.team_member_rpm_limit is not None + or user_api_key_auth.organization_rpm_limit is not None + or user_api_key_auth.end_user_rpm_limit is not None ) - from litellm.proxy.proxy_server import llm_router, proxy_logging_obj - from litellm.router_utils.common_utils import resolve_model_group_alias - - limiter: Final = proxy_logging_obj.get_proxy_hook("parallel_request_limiter") - if not isinstance(limiter, PROXY_MaxParallelRequestsHandler_v3): - return None - - if user_api_key_auth.api_key and ( - user_api_key_auth.rpm_limit is not None or user_api_key_auth.max_parallel_requests is not None - ): - return limiter - if user_api_key_auth.user_id and user_api_key_auth.user_rpm_limit is not None: - return limiter - if user_api_key_auth.team_id and user_api_key_auth.team_rpm_limit is not None: - return limiter - if user_api_key_auth.user_id and user_api_key_auth.team_member_rpm_limit is not None: - return limiter - if user_api_key_auth.org_id and user_api_key_auth.organization_rpm_limit is not None: - return limiter - if user_api_key_auth.end_user_id and user_api_key_auth.end_user_rpm_limit is not None: - return limiter - - from litellm.proxy.auth.auth_utils import get_key_model_rpm_limit, get_model_rate_limit_from_metadata - - model_name: Final = "mcp-tool-call" - model_group: Final = ( - resolve_model_group_alias(llm_router.model_group_alias, model_name) if llm_router is not None else None - ) - model: Final = RateLimitedModel(requested=model_name, group=model_group or model_name) - key_model_rpm_limit: Final = model.limit_in(get_key_model_rpm_limit(user_api_key_auth, model_name=model.group)) - if key_model_rpm_limit is not None: - return limiter - - team_model_rpm_limit: Final = model.limit_in( - get_model_rate_limit_from_metadata(user_api_key_auth, "team_metadata", "model_rpm_limit") - ) - if team_model_rpm_limit is not None: - return limiter - - organization_model_rpm_limit: Final = model.limit_in( - get_model_rate_limit_from_metadata(user_api_key_auth, "organization_metadata", "model_rpm_limit") - ) - if organization_model_rpm_limit is not None: - return limiter - - project_model_rpm_limit: Final = model.limit_in( - get_model_rate_limit_from_metadata(user_api_key_auth, "project_metadata", "model_rpm_limit") - ) - if project_model_rpm_limit is not None: - return limiter - - return None def _mcp_setup_request_capacity( method: str | None, user_api_key_auth: UserAPIKeyAuth | None, - limiter: object | None, ) -> contextlib.AbstractAsyncContextManager[None]: - if method not in _MCP_SETUP_REQUEST_METHODS or user_api_key_auth is None or limiter is None: + if method not in _MCP_SETUP_REQUEST_METHODS or user_api_key_auth is None: return contextlib.nullcontext() from litellm.proxy.hooks.parallel_request_limiter_v3 import PROXY_MaxParallelRequestsHandler_v3 + from litellm.proxy.proxy_server import proxy_logging_obj + limiter: Final = proxy_logging_obj.get_proxy_hook("parallel_request_limiter") if not isinstance(limiter, PROXY_MaxParallelRequestsHandler_v3): return contextlib.nullcontext() return limiter.request_capacity(user_api_key_auth, "mcp-tool-call", request_data={}) @@ -2781,23 +2732,15 @@ if MCP_AVAILABLE: if use_stateful and session_id and scope.get("method") == "DELETE": _remove_stateful_session_tracking(session_id) - setup_request_limiter: Final = ( - _mcp_setup_request_limiter(user_api_key_auth) - if mcp_method in _MCP_SETUP_REQUEST_METHODS - or (mcp_method is None and peek is not None and len(body) >= _MCP_ROUTING_PEEK_MAX_BYTES) - else None - ) charged_method: Final = ( _jsonrpc_request_method(await peek.full_body()) if mcp_method is None and peek is not None and len(body) >= _MCP_ROUTING_PEEK_MAX_BYTES - and setup_request_limiter is not None + and _has_request_rate_limit(user_api_key_auth) else mcp_method ) - request_capacity: Final = _mcp_setup_request_capacity( - charged_method, user_api_key_auth, setup_request_limiter - ) + request_capacity: Final = _mcp_setup_request_capacity(charged_method, user_api_key_auth) try: async with ( session_lock if session_lock is not None else contextlib.nullcontext(), From f8bf45aae0ad214941fda6a86a43ebda9353cd0a Mon Sep 17 00:00:00 2001 From: yassin Date: Wed, 21 Oct 2026 12:15:40 +0000 Subject: [PATCH 5/5] fix(mcp): gate the full-body charge on the limiter's effective descriptors Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/_experimental/mcp_server/server.py | 24 ++-- .../hooks/parallel_request_limiter_v3.py | 21 +++- .../mcp_server/test_mcp_server.py | 109 +++++++++++++++++- .../hooks/test_parallel_request_limiter_v3.py | 15 +++ 4 files changed, 151 insertions(+), 18 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index f8645fc4884..c0b90570ee9 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -136,18 +136,6 @@ _MCP_PROTOCOL_VERSION_HEADER: Final = b"mcp-protocol-version" _DECODED_JSON: Final = TypeAdapter(object) -def _has_request_rate_limit(user_api_key_auth: UserAPIKeyAuth) -> bool: - return ( - user_api_key_auth.rpm_limit is not None - or user_api_key_auth.max_parallel_requests is not None - or user_api_key_auth.user_rpm_limit is not None - or user_api_key_auth.team_rpm_limit is not None - or user_api_key_auth.team_member_rpm_limit is not None - or user_api_key_auth.organization_rpm_limit is not None - or user_api_key_auth.end_user_rpm_limit is not None - ) - - def _mcp_setup_request_capacity( method: str | None, user_api_key_auth: UserAPIKeyAuth | None, @@ -164,6 +152,16 @@ def _mcp_setup_request_capacity( return limiter.request_capacity(user_api_key_auth, "mcp-tool-call", request_data={}) +async def _mcp_setup_request_limit_applies(user_api_key_auth: UserAPIKeyAuth) -> bool: + from litellm.proxy.hooks.parallel_request_limiter_v3 import PROXY_MaxParallelRequestsHandler_v3 + from litellm.proxy.proxy_server import proxy_logging_obj + + limiter: Final = proxy_logging_obj.get_proxy_hook("parallel_request_limiter") + if not isinstance(limiter, PROXY_MaxParallelRequestsHandler_v3): + return False + return await limiter.has_request_capacity_limit(user_api_key_auth, "mcp-tool-call") + + def reject_disallowed_mcp_origin(request: StarletteRequest) -> None: from litellm.proxy.proxy_server import origins # noqa: PLC0415 # proxy imports this module during startup @@ -2737,7 +2735,7 @@ if MCP_AVAILABLE: if mcp_method is None and peek is not None and len(body) >= _MCP_ROUTING_PEEK_MAX_BYTES - and _has_request_rate_limit(user_api_key_auth) + and await _mcp_setup_request_limit_applies(user_api_key_auth) else mcp_method ) request_capacity: Final = _mcp_setup_request_capacity(charged_method, user_api_key_auth) diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index a2bb691b401..f7430456285 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -4141,6 +4141,25 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): *await self._create_tag_rate_limit_descriptors(data), ] + @staticmethod + def _request_capacity_data( + model: str, + request_data: Mapping[str, object] | None = None, + ) -> Mapping[str, object]: + return MappingProxyType({**(request_data or MappingProxyType({})), "model": model}) + + async def has_request_capacity_limit(self, user_api_key_dict: UserAPIKeyAuth, model: str) -> bool: + data: Final = self._request_capacity_data(model) + descriptors: Final = await self._build_request_rate_limit_descriptors(user_api_key_dict, data, None) + return any( + descriptor["rate_limit"] is not None + and ( + descriptor["rate_limit"].get("requests_per_unit") is not None + or descriptor["rate_limit"].get("max_parallel_requests") is not None + ) + for descriptor in descriptors + ) + async def _release_request_capacity_when_admitted( self, admission: asyncio.Task[RateLimitResponse], @@ -4160,7 +4179,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): request_data: Mapping[str, object] | None = None, ) -> AsyncGenerator[None, None]: """Charge one non-generation provider request to RPM and hold its concurrency slot.""" - data: Final = MappingProxyType({**(request_data or MappingProxyType({})), "model": model}) + data: Final = self._request_capacity_data(model, request_data) descriptors: Final = await self._build_request_rate_limit_descriptors(user_api_key_dict, data, None) acquisition: Final = ParallelSlotAcquisition( slot_id=uuid.uuid4().hex, diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py index d33dc412b4c..c0d8edf1a50 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py @@ -11,7 +11,7 @@ from contextlib import asynccontextmanager, nullcontext from starlette.types import Receive, Scope, Send - +from litellm.types.agents import AgentResponse from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( MCPServerManager, MCPServer, @@ -3265,7 +3265,8 @@ async def test_mcp_routing_batch_body_is_not_stashed_for_auth(): async def _streamable_mcp_request_runner( monkeypatch: pytest.MonkeyPatch, auth: UserAPIKeyAuth, -) -> AsyncIterator[tuple[Callable[[str], Awaitable[None]], AsyncMock]]: + on_receive_count_at_dispatch: Callable[[int], None] | None = None, +) -> AsyncIterator[tuple[Callable[..., Awaitable[None]], AsyncMock]]: from litellm.caching.dual_cache import DualCache from litellm.proxy import proxy_server from litellm.proxy._experimental.mcp_server import server @@ -3299,7 +3300,11 @@ async def _streamable_mcp_request_runner( monkeypatch.setattr(server, "_gateway_initialize_instructions_request_scope", no_initialize_scope) - async def request(method: str, request_body: bytes | None = None) -> None: + async def request( + method: str, + request_body: bytes | None = None, + request_chunks: tuple[bytes, ...] | None = None, + ) -> None: body: Final = ( request_body if request_body is not None @@ -3320,13 +3325,41 @@ async def _streamable_mcp_request_runner( "server": ("localhost", 4000), "scheme": "http", } - receive: Final[Receive] = AsyncMock(return_value={"type": "http.request", "body": body, "more_body": False}) + request_messages: Final = ( + tuple( + { + "type": "http.request", + "body": chunk, + "more_body": index < len(request_chunks) - 1, + } + for index, chunk in enumerate(request_chunks) + ) + if request_chunks is not None + else ({"type": "http.request", "body": body, "more_body": False},) + ) + receive: Final[Receive] = AsyncMock(side_effect=request_messages) + if on_receive_count_at_dispatch is not None: + def observe_receive_count(*_args: object, **_kwargs: object) -> None: + on_receive_count_at_dispatch(receive.await_count) + + dispatch.side_effect = observe_receive_count send: Final[Send] = AsyncMock() await server.handle_streamable_http_mcp(scope, receive, send) yield request, dispatch +def _padded_tools_list_body() -> bytes: + return json.dumps( + { + "jsonrpc": "2.0", + "id": 2, + "params": {"_meta": {"pad": "x" * 5000}}, + "method": "tools/list", + } + ).encode() + + @pytest.mark.asyncio async def test_streamable_mcp_setup_requests_obey_virtual_key_rpm_limit(monkeypatch: pytest.MonkeyPatch) -> None: from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError @@ -3368,6 +3401,54 @@ async def test_streamable_mcp_oversized_setup_request_obeys_virtual_key_rpm_limi assert dispatch.await_count == 1 +@pytest.mark.asyncio +async def test_streamable_mcp_oversized_setup_request_obeys_model_metadata_rpm_limit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + + auth: Final = UserAPIKeyAuth( + api_key="mcp-oversized-model-rpm-key", + metadata={"model_rpm_limit": {"mcp-tool-call": 1}}, + ) + async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch): + oversized_body: Final = _padded_tools_list_body() + await request("tools/list", oversized_body) + with pytest.raises(ProxyRateLimitError) as rate_limit_error: + await request("tools/list", oversized_body) + + assert rate_limit_error.value.status_code == 429 + assert rate_limit_error.value.headers.get("retry-after") is not None + assert dispatch.await_count == 1 + + +@pytest.mark.asyncio +async def test_streamable_mcp_oversized_setup_request_obeys_agent_rpm_limit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + + agent_id: Final = "mcp-setup-agent-rpm" + agent_policy: Final = AgentResponse( + agent_id=agent_id, + agent_name="mcp-setup-agent", + agent_card_params={"name": "mcp-setup-agent"}, + rpm_limit=1, + ) + auth: Final = UserAPIKeyAuth(api_key="mcp-oversized-agent-rpm-key", agent_id=agent_id).model_copy( + update={"managed_agent_policy": agent_policy} + ) + async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch): + oversized_body: Final = _padded_tools_list_body() + await request("tools/list", oversized_body) + with pytest.raises(ProxyRateLimitError) as rate_limit_error: + await request("tools/list", oversized_body) + + assert rate_limit_error.value.status_code == 429 + assert rate_limit_error.value.headers.get("retry-after") is not None + assert dispatch.await_count == 1 + + @pytest.mark.asyncio async def test_streamable_mcp_oversized_setup_request_dispatches_without_virtual_key_limits( monkeypatch: pytest.MonkeyPatch, @@ -3387,6 +3468,26 @@ async def test_streamable_mcp_oversized_setup_request_dispatches_without_virtual assert dispatch.await_count == 1 +@pytest.mark.asyncio +async def test_streamable_mcp_oversized_unlimited_setup_request_keeps_routing_peek_bounded( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy._experimental.mcp_server import server as mcp_server + + auth: Final = UserAPIKeyAuth(api_key="mcp-oversized-unlimited-peek-key") + + def assert_receive_count(receive_count: int) -> None: + assert receive_count == 1 + + async with _streamable_mcp_request_runner(monkeypatch, auth, assert_receive_count) as (request, dispatch): + assert await mcp_server._mcp_setup_request_limit_applies(auth) is False + oversized_body: Final = _padded_tools_list_body() + peek_cap: Final = mcp_server._MCP_ROUTING_PEEK_MAX_BYTES + await request("tools/list", oversized_body, (oversized_body[:peek_cap], oversized_body[peek_cap:])) + + assert dispatch.await_count == 1 + + @pytest.mark.asyncio async def test_streamable_mcp_tools_call_is_not_charged_by_setup_limiter(monkeypatch: pytest.MonkeyPatch) -> None: auth: Final = UserAPIKeyAuth(api_key="mcp-tools-call-rpm-key", rpm_limit=1) diff --git a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py index 1ec505362dd..4de06e42e43 100644 --- a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py @@ -7436,6 +7436,21 @@ async def test_request_capacity_enforces_shared_rpm_scopes( assert counter_scope in str(exc.value.detail) +@pytest.mark.asyncio +async def test_has_request_capacity_limit_matches_effective_request_descriptors(): + handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache())) + no_limits_auth = UserAPIKeyAuth(api_key=hash_token("sk-count-no-request-limit")) + key_rpm_auth = UserAPIKeyAuth(api_key=hash_token("sk-count-key-request-limit"), rpm_limit=1) + model_rpm_auth = UserAPIKeyAuth( + api_key=hash_token("sk-count-model-request-limit"), + metadata={"model_rpm_limit": {"mcp-tool-call": 1}}, + ) + + assert await handler.has_request_capacity_limit(no_limits_auth, "mcp-tool-call") is False + assert await handler.has_request_capacity_limit(key_rpm_auth, "mcp-tool-call") is True + assert await handler.has_request_capacity_limit(model_rpm_auth, "mcp-tool-call") is True + + @pytest.mark.asyncio async def test_request_capacity_keeps_dynamic_rpm_policy(monkeypatch): import litellm.proxy.proxy_server as proxy_server