This commit is contained in:
devin-ai-integration[bot] 2026-10-24 19:17:37 +00:00 • committed by GitHub
commit 5342fdea34
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 427 additions and 16 deletions

View file

@ -115,6 +115,7 @@ if TYPE_CHECKING:
_redact_mcp_resource_url: Final = redact_mcp_resource_url
_MCP_SETUP_REQUEST_METHODS: Final = frozenset({"initialize", "tools/list"})
_STATEFUL_SESSION_IDLE_TIMEOUT_SECONDS: Final = 30 * 60
# Upper bound on concurrent stateful sessions a single caller may hold. Each
# `initialize` creates a session that survives until the idle timeout, so
@ -135,6 +136,32 @@ _MCP_PROTOCOL_VERSION_HEADER: Final = b"mcp-protocol-version"
_DECODED_JSON: Final = TypeAdapter(object)
def _mcp_setup_request_capacity(
method: str | None,
user_api_key_auth: UserAPIKeyAuth | None,
) -> contextlib.AbstractAsyncContextManager[None]:
if method not in _MCP_SETUP_REQUEST_METHODS or user_api_key_auth is None:
return contextlib.nullcontext()
from litellm.proxy.hooks.parallel_request_limiter_v3 import PROXY_MaxParallelRequestsHandler_v3
from litellm.proxy.proxy_server import proxy_logging_obj
limiter: Final = proxy_logging_obj.get_proxy_hook("parallel_request_limiter")
if not isinstance(limiter, PROXY_MaxParallelRequestsHandler_v3):
return contextlib.nullcontext()
return limiter.request_capacity(user_api_key_auth, "mcp-tool-call", request_data={})
async def _mcp_setup_request_limit_applies(user_api_key_auth: UserAPIKeyAuth) -> bool:
from litellm.proxy.hooks.parallel_request_limiter_v3 import PROXY_MaxParallelRequestsHandler_v3
from litellm.proxy.proxy_server import proxy_logging_obj
limiter: Final = proxy_logging_obj.get_proxy_hook("parallel_request_limiter")
if not isinstance(limiter, PROXY_MaxParallelRequestsHandler_v3):
return False
return await limiter.has_request_capacity_limit(user_api_key_auth, "mcp-tool-call")
def reject_disallowed_mcp_origin(request: StarletteRequest) -> None:
from litellm.proxy.proxy_server import origins # noqa: PLC0415 # proxy imports this module during startup
@ -1403,18 +1430,15 @@ if MCP_AVAILABLE:
return b"{}"
return json.dumps({"method": method}).encode()
def _is_initialize_request(body: bytes) -> bool:
"""
Check if the request body is a JSON-RPC initialize method.
Returns True if method is "initialize", False otherwise or on parse error.
"""
def _mcp_request_method(body: bytes) -> str | None:
if not body:
return False
return None
try:
data: Final = as_str_mapping(_DECODED_JSON.validate_python(json.loads(body)))
return data is not None and data.get("method") == "initialize"
method: Final = data.get("method") if data is not None else None
return method if isinstance(method, str) else None
except (json.JSONDecodeError, UnicodeDecodeError, TypeError):
return False
return None
def _jsonrpc_request_method(body: bytes) -> str | None:
try:
@ -2433,7 +2457,8 @@ if MCP_AVAILABLE:
) = await extract_mcp_auth_context(scope, path)
admit_mcp_request(scope, user_api_key_auth, mcp_servers)
body: Final = await peek.body() if peek is not None else b""
is_initialize: Final = _is_initialize_request(body)
mcp_method: Final = _mcp_request_method(body)
is_initialize: Final = mcp_method == "initialize"
scoped_server_endpoint: Final = len(_get_mcp_servers_in_path(path) or []) == 1
request_headers: Final = StarletteRequest(scope).headers
defer_upstream_probes: Final = request_headers.get(
@ -2715,11 +2740,20 @@ if MCP_AVAILABLE:
if use_stateful and session_id and scope.get("method") == "DELETE":
_remove_stateful_session_tracking(session_id)
charged_method: Final = (
_jsonrpc_request_method(await peek.full_body())
if mcp_method is None
and peek is not None
and len(body) >= _MCP_ROUTING_PEEK_MAX_BYTES
and await _mcp_setup_request_limit_applies(user_api_key_auth)
else mcp_method
)
request_capacity: Final = _mcp_setup_request_capacity(charged_method, user_api_key_auth)
try:
if session_lock is not None:
async with session_lock:
await _dispatch()
else:
async with (
session_lock if session_lock is not None else contextlib.nullcontext(),
request_capacity,
):
await _dispatch()
finally:
for active_request_session_id in active_request_session_ids:

View file

@ -4141,6 +4141,25 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
*await self._create_tag_rate_limit_descriptors(data),
]
@staticmethod
def _request_capacity_data(
model: str,
request_data: Mapping[str, object] | None = None,
) -> Mapping[str, object]:
return MappingProxyType({**(request_data or MappingProxyType({})), "model": model})
async def has_request_capacity_limit(self, user_api_key_dict: UserAPIKeyAuth, model: str) -> bool:
data: Final = self._request_capacity_data(model)
descriptors: Final = await self._build_request_rate_limit_descriptors(user_api_key_dict, data, None)
return any(
descriptor["rate_limit"] is not None
and (
descriptor["rate_limit"].get("requests_per_unit") is not None
or descriptor["rate_limit"].get("max_parallel_requests") is not None
)
for descriptor in descriptors
)
async def _release_request_capacity_when_admitted(
self,
admission: asyncio.Task[RateLimitResponse],
@ -4160,7 +4179,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
request_data: Mapping[str, object] | None = None,
) -> AsyncGenerator[None, None]:
"""Charge one non-generation provider request to RPM and hold its concurrency slot."""
data: Final = MappingProxyType({**(request_data or MappingProxyType({})), "model": model})
data: Final = self._request_capacity_data(model, request_data)
descriptors: Final = await self._build_request_rate_limit_descriptors(user_api_key_dict, data, None)
acquisition: Final = ParallelSlotAcquisition(
slot_id=uuid.uuid4().hex,

View file

@ -269,3 +269,83 @@ def test_mcp_key_team_and_server_rpm_limits_share_redis(tmp_path: Path, monkeypa
server_rejected: Final = _call_tool(second_replica, key_three, server_id, "rpm-add")
_assert_rate_limit(server_rejected, "mcp_server")
assert tool_calls(peer.drain()) == ()
def test_streamable_mcp_setup_requests_use_virtual_key_rpm_limit(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
results_directory: Final = tmp_path / "results"
results_directory.mkdir()
monkeypatch.setenv("INTEGRATION_RESULTS_DIR", str(results_directory))
assert os.environ.get("DATABASE_URL"), "This integration case requires disposable-database access"
with (
owned_redis(tmp_path) as redis,
scratch_database() as database_url,
mcp_peer() as peer,
httpx.Client() as client,
):
monkeypatch.setenv("DATABASE_URL", database_url)
seed: Final = Gateway(client, "sk-mcp-streamable-setup-rate-limit", peer.url)
config: Final = _config_file(
tmp_path,
seed.key,
redis,
upstream=peer.url,
store_model_in_db=True,
)
environment: Final = {
"DATABASE_URL": database_url,
"LITELLM_SALT_KEY": "mcp-streamable-setup-rate-limit",
"LITELLM_RATE_LIMIT_WINDOW_SIZE": "30",
}
options: Final = {
"config": config,
"remove_environment": ("DATABASE_URL_READ_REPLICA", "LITELLM_LICENSE", "LITELLM_LICENSE_PATH"),
}
with owned_proxy(seed, tmp_path / "proxy", environment, **options) as gateway, gateway.scenario() as scenario:
server_id: Final = register_mcp(scenario, peer, "rpm", allowed_tools=["add"])
key: Final = scenario.key(rpm_limit=2, object_permission={"mcp_servers": [server_id]})
def post_mcp(payload: dict[str, object], session_id: str | None = None) -> httpx.Response:
headers: Final = {
"Authorization": f"Bearer {key}",
"Accept": "application/json, text/event-stream",
**({"mcp-session-id": session_id} if session_id is not None else {}),
}
return gateway.client.post("/mcp/", headers=headers, json=payload)
initialize: Final = post_mcp(
{
"jsonrpc": "2.0",
"id": 1,
"method": "initialize",
"params": {
"protocolVersion": "2025-03-26",
"capabilities": {},
"clientInfo": {"name": "litellm-test", "version": "1"},
},
}
)
assert initialize.status_code == 200, initialize.text
session_id: Final = initialize.headers["mcp-session-id"]
initialized: Final = post_mcp(
{"jsonrpc": "2.0", "method": "notifications/initialized"},
session_id,
)
assert initialized.status_code == 202, initialized.text
first_list: Final = post_mcp(
{"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}},
session_id,
)
assert first_list.status_code == 200, first_list.text
second_list: Final = post_mcp(
{"jsonrpc": "2.0", "id": 3, "method": "tools/list", "params": {}},
session_id,
)
assert second_list.status_code == 429, second_list.text
assert second_list.headers.get("retry-after") is not None

View file

@ -2,6 +2,7 @@
import asyncio
import json
import os
from collections.abc import AsyncIterator, Awaitable, Callable
from typing import Final
import pytest
@ -10,13 +11,14 @@ from contextlib import asynccontextmanager, nullcontext
from starlette.types import Receive, Scope, Send
from litellm.types.agents import AgentResponse
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
MCPServerManager,
MCPServer,
MCPTransport,
)
from litellm.proxy._types import LiteLLM_ObjectPermissionTable
from litellm.proxy._types import UserAPIKeyAuth
from mcp.types import Tool as MCPTool, CallToolResult, ListToolsResult
from mcp.types import TextContent
@ -3259,6 +3261,267 @@ async def test_mcp_routing_batch_body_is_not_stashed_for_auth():
assert await _read_request_body(_admission_request(scope)) == {}
@asynccontextmanager
async def _streamable_mcp_request_runner(
monkeypatch: pytest.MonkeyPatch,
auth: UserAPIKeyAuth,
on_receive_count_at_dispatch: Callable[[int], None] | None = None,
) -> AsyncIterator[tuple[Callable[..., Awaitable[None]], AsyncMock]]:
from litellm.caching.dual_cache import DualCache
from litellm.proxy import proxy_server
from litellm.proxy._experimental.mcp_server import server
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.hooks.parallel_request_limiter_v3 import PROXY_MaxParallelRequestsHandler_v3
from litellm.proxy.utils import InternalUsageCache, ProxyLogging
proxy_logging: Final = ProxyLogging(user_api_key_cache=UserApiKeyCache())
limiter: Final = PROXY_MaxParallelRequestsHandler_v3(
internal_usage_cache=InternalUsageCache(dual_cache=DualCache())
)
proxy_logging.proxy_hook_mapping["parallel_request_limiter"] = limiter
monkeypatch.setattr(proxy_server, "proxy_logging_obj", proxy_logging)
dispatch: Final = AsyncMock()
session_manager: Final = MagicMock()
session_manager.max_request_body_size = server.session_manager_stateless.max_request_body_size
session_manager.handle_request = dispatch
monkeypatch.setattr(server, "_SESSION_MANAGERS_INITIALIZED", True)
monkeypatch.setattr(server, "session_manager_stateless", session_manager)
monkeypatch.setattr(server, "session_manager_stateful", session_manager)
monkeypatch.setattr(
server,
"extract_mcp_auth_context",
AsyncMock(return_value=(auth, None, ["echo_server"], {}, {}, {})),
)
@asynccontextmanager
async def no_initialize_scope(*_args: object, **_kwargs: object) -> AsyncIterator[None]:
yield
monkeypatch.setattr(server, "_gateway_initialize_instructions_request_scope", no_initialize_scope)
async def request(
method: str,
request_body: bytes | None = None,
request_chunks: tuple[bytes, ...] | None = None,
) -> None:
body: Final = (
request_body
if request_body is not None
else json.dumps(
{
"jsonrpc": "2.0",
"method": method,
**({"id": 1} if method != "notifications/initialized" else {}),
}
).encode()
)
scope: Final[Scope] = {
"type": "http",
"method": "POST",
"path": "/mcp/echo_server",
"headers": [(b"content-type", b"application/json")],
"query_string": b"",
"server": ("localhost", 4000),
"scheme": "http",
}
request_messages: Final = (
tuple(
{
"type": "http.request",
"body": chunk,
"more_body": index < len(request_chunks) - 1,
}
for index, chunk in enumerate(request_chunks)
)
if request_chunks is not None
else ({"type": "http.request", "body": body, "more_body": False},)
)
receive: Final[Receive] = AsyncMock(side_effect=request_messages)
if on_receive_count_at_dispatch is not None:
def observe_receive_count(*_args: object, **_kwargs: object) -> None:
on_receive_count_at_dispatch(receive.await_count)
dispatch.side_effect = observe_receive_count
send: Final[Send] = AsyncMock()
await server.handle_streamable_http_mcp(scope, receive, send)
yield request, dispatch
def _padded_tools_list_body() -> bytes:
return json.dumps(
{
"jsonrpc": "2.0",
"id": 2,
"params": {"_meta": {"pad": "x" * 5000}},
"method": "tools/list",
}
).encode()
@pytest.mark.asyncio
async def test_streamable_mcp_setup_requests_obey_virtual_key_rpm_limit(monkeypatch: pytest.MonkeyPatch) -> None:
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
auth: Final = UserAPIKeyAuth(api_key="mcp-setup-rpm-key", rpm_limit=2)
async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch):
await request("initialize")
await request("tools/list")
with pytest.raises(ProxyRateLimitError) as rate_limit_error:
await request("tools/list")
assert rate_limit_error.value.status_code == 429
assert rate_limit_error.value.headers.get("retry-after") is not None
assert dispatch.await_count == 2
@pytest.mark.asyncio
async def test_streamable_mcp_oversized_setup_request_obeys_virtual_key_rpm_limit(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
auth: Final = UserAPIKeyAuth(api_key="mcp-oversized-setup-rpm-key", rpm_limit=1)
async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch):
await request("tools/list")
oversized_body: Final = json.dumps(
{
"jsonrpc": "2.0",
"id": 2,
"params": {"_meta": {"pad": "x" * 5000}},
"method": "tools/list",
}
).encode()
with pytest.raises(ProxyRateLimitError) as rate_limit_error:
await request("tools/list", oversized_body)
assert rate_limit_error.value.status_code == 429
assert rate_limit_error.value.headers.get("retry-after") is not None
assert dispatch.await_count == 1
@pytest.mark.asyncio
async def test_streamable_mcp_oversized_setup_request_obeys_model_metadata_rpm_limit(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
auth: Final = UserAPIKeyAuth(
api_key="mcp-oversized-model-rpm-key",
metadata={"model_rpm_limit": {"mcp-tool-call": 1}},
)
async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch):
oversized_body: Final = _padded_tools_list_body()
await request("tools/list", oversized_body)
with pytest.raises(ProxyRateLimitError) as rate_limit_error:
await request("tools/list", oversized_body)
assert rate_limit_error.value.status_code == 429
assert rate_limit_error.value.headers.get("retry-after") is not None
assert dispatch.await_count == 1
@pytest.mark.asyncio
async def test_streamable_mcp_oversized_setup_request_obeys_agent_rpm_limit(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
agent_id: Final = "mcp-setup-agent-rpm"
agent_policy: Final = AgentResponse(
agent_id=agent_id,
agent_name="mcp-setup-agent",
agent_card_params={"name": "mcp-setup-agent"},
rpm_limit=1,
)
auth: Final = UserAPIKeyAuth(api_key="mcp-oversized-agent-rpm-key", agent_id=agent_id).model_copy(
update={"managed_agent_policy": agent_policy}
)
async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch):
oversized_body: Final = _padded_tools_list_body()
await request("tools/list", oversized_body)
with pytest.raises(ProxyRateLimitError) as rate_limit_error:
await request("tools/list", oversized_body)
assert rate_limit_error.value.status_code == 429
assert rate_limit_error.value.headers.get("retry-after") is not None
assert dispatch.await_count == 1
@pytest.mark.asyncio
async def test_streamable_mcp_oversized_setup_request_dispatches_without_virtual_key_limits(
monkeypatch: pytest.MonkeyPatch,
) -> None:
auth: Final = UserAPIKeyAuth(api_key="mcp-oversized-unlimited-key")
async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch):
oversized_body: Final = json.dumps(
{
"jsonrpc": "2.0",
"id": 2,
"params": {"_meta": {"pad": "x" * 5000}},
"method": "tools/list",
}
).encode()
await request("tools/list", oversized_body)
assert dispatch.await_count == 1
@pytest.mark.asyncio
async def test_streamable_mcp_oversized_unlimited_setup_request_keeps_routing_peek_bounded(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from litellm.proxy._experimental.mcp_server import server as mcp_server
auth: Final = UserAPIKeyAuth(api_key="mcp-oversized-unlimited-peek-key")
def assert_receive_count(receive_count: int) -> None:
assert receive_count == 1
async with _streamable_mcp_request_runner(monkeypatch, auth, assert_receive_count) as (request, dispatch):
assert await mcp_server._mcp_setup_request_limit_applies(auth) is False
oversized_body: Final = _padded_tools_list_body()
peek_cap: Final = mcp_server._MCP_ROUTING_PEEK_MAX_BYTES
await request("tools/list", oversized_body, (oversized_body[:peek_cap], oversized_body[peek_cap:]))
assert dispatch.await_count == 1
@pytest.mark.asyncio
async def test_streamable_mcp_tools_call_is_not_charged_by_setup_limiter(monkeypatch: pytest.MonkeyPatch) -> None:
auth: Final = UserAPIKeyAuth(api_key="mcp-tools-call-rpm-key", rpm_limit=1)
async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch):
await request("tools/call")
await request("initialize")
assert dispatch.await_count == 2
@pytest.mark.asyncio
async def test_streamable_mcp_initialized_notification_does_not_consume_rpm(
monkeypatch: pytest.MonkeyPatch,
) -> None:
auth: Final = UserAPIKeyAuth(api_key="mcp-initialized-rpm-key", rpm_limit=1)
async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch):
await request("notifications/initialized")
await request("initialize")
assert dispatch.await_count == 2
@pytest.mark.asyncio
async def test_streamable_mcp_setup_request_releases_parallel_request_slot(
monkeypatch: pytest.MonkeyPatch,
) -> None:
auth: Final = UserAPIKeyAuth(api_key="mcp-setup-parallel-key", max_parallel_requests=1)
async with _streamable_mcp_request_runner(monkeypatch, auth) as (request, dispatch):
await request("initialize")
await request("tools/list")
assert dispatch.await_count == 2
@pytest.mark.asyncio
async def test_mcp_routing_truncated_json_body_is_not_stashed_for_auth():
try:

View file

@ -3006,7 +3006,7 @@ def test_oversized_initialize_peek_neither_routes_stateful_nor_attributes_client
peeked = full_body[: mcp_server._MCP_ROUTING_PEEK_MAX_BYTES]
assert mcp_server._extract_initialize_client_info(full_body) is not None
assert mcp_server._is_initialize_request(peeked) is False
assert mcp_server._mcp_request_method(peeked) is None
assert mcp_server._extract_initialize_client_info(peeked) is None

View file

@ -7436,6 +7436,21 @@ async def test_request_capacity_enforces_shared_rpm_scopes(
assert counter_scope in str(exc.value.detail)
@pytest.mark.asyncio
async def test_has_request_capacity_limit_matches_effective_request_descriptors():
handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache()))
no_limits_auth = UserAPIKeyAuth(api_key=hash_token("sk-count-no-request-limit"))
key_rpm_auth = UserAPIKeyAuth(api_key=hash_token("sk-count-key-request-limit"), rpm_limit=1)
model_rpm_auth = UserAPIKeyAuth(
api_key=hash_token("sk-count-model-request-limit"),
metadata={"model_rpm_limit": {"mcp-tool-call": 1}},
)
assert await handler.has_request_capacity_limit(no_limits_auth, "mcp-tool-call") is False
assert await handler.has_request_capacity_limit(key_rpm_auth, "mcp-tool-call") is True
assert await handler.has_request_capacity_limit(model_rpm_auth, "mcp-tool-call") is True
@pytest.mark.asyncio
async def test_request_capacity_keeps_dynamic_rpm_policy(monkeypatch):
import litellm.proxy.proxy_server as proxy_server