From 6b98e485f6667ef967d40203bcc469582f4ff312 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:15:00 -0700 Subject: [PATCH] fix(proxy): free the max_parallel_requests slot when a replayed /v1/messages MCP stream completes A streamed /v1/messages request that auto-executes MCP tools runs its model rounds without streaming and replays the final answer as a stream. The first round claims the request's only success log, and its limiter callback keeps the slot because the tool call reused it afterwards. The stream path released the slot only on a client disconnect, so after message_stop the key stayed at its max_parallel_requests limit and its next request answered 429. Stream cleanup now releases the stashed slot when the stream completed and its logging object already claimed the success log, which only happens when the stream replays a response that was logged before it started. A real stream still releases through its own success callback. --- litellm/litellm_core_utils/litellm_logging.py | 4 + litellm/proxy/common_request_processing.py | 10 +- .../hooks/test_parallel_request_limiter_v3.py | 91 +++++++++++++++++++ 3 files changed, 104 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index a59672b5cb2..b5714989cc8 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2583,6 +2583,10 @@ class Logging(LiteLLMLoggingBaseClass): self._async_success_scheduled = True return True + @property + def async_success_claimed(self) -> bool: + return self._async_success_scheduled + def mark_logging_complete( self, event_type: Literal["async_success", "sync_success", "async_failure", "sync_failure"], diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 6fda5780f63..6557e5088a3 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -4353,6 +4353,15 @@ class ProxyBaseLLMRequestProcessing: and user_api_key_dict is not None ): await proxy_logging_obj.arelease_max_parallel_requests_on_disconnect(user_api_key_dict) + logging_obj: Final = request_data.get("litellm_logging_obj") + if ( + stream_completed + and isinstance(logging_obj, LiteLLMLoggingObj) + and logging_obj.async_success_claimed + and proxy_logging_obj is not None + and user_api_key_dict is not None + ): + await proxy_logging_obj.arelease_max_parallel_requests_on_disconnect(user_api_key_dict) if hasattr(response, "aclose"): try: @@ -4362,7 +4371,6 @@ class ProxyBaseLLMRequestProcessing: "async_streaming_data_generator: error closing response stream: %s", e, ) - logging_obj: Final = request_data.get("litellm_logging_obj") if ( not stream_completed and isinstance(logging_obj, LiteLLMLoggingObj) diff --git a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py index 1ec505362dd..67cf5d97e6f 100644 --- a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py @@ -4332,6 +4332,97 @@ async def test_first_round_logged_after_tool_call_reuse_releases_only_at_request assert await in_flight() == 0 +@pytest.mark.asyncio +async def test_completed_mcp_replay_stream_frees_slot_its_logged_first_round_kept() -> None: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import ( + FakeAnthropicMessagesStreamIterator, + ) + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + + reused_at: Final = datetime(2026, 1, 5, 12, 0, 0) + api_key: Final = hash_token("sk-mcp-replayed-stream") + local_cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache), + time_provider=lambda: reused_at, + ) + proxy_logging_obj: Final = ProxyLogging(user_api_key_cache=DualCache()) + proxy_logging_obj.proxy_hook_mapping["parallel_request_limiter"] = handler + user_api_key_dict: Final = UserAPIKeyAuth(api_key=api_key, max_parallel_requests=1) + parallel_key: Final = f"{{api_key:{api_key}}}:max_parallel_requests" + owner_call_id: Final = "mcp-replayed-stream-owner" + messages: Final = [{"role": "user", "content": "add 2 and 3"}] + + async def in_flight() -> int: + return handler._gauge_in_flight_from_cache_value( + await local_cache.async_get_cache(key=parallel_key) + ) + + async def pre_call(data: dict[str, object], call_type: str) -> None: + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data=data, + call_type=call_type, + ) + + await pre_call( + {"model": "claude-haiku-5-5", "messages": messages, "stream": True, "litellm_call_id": owner_call_id}, + CallTypes.anthropic_messages.value, + ) + await pre_call( + {"model": "MCP: add", "messages": [{"role": "user", "content": "Tool: add"}], "mcp_server_name": "math"}, + CallTypes.call_mcp_tool.value, + ) + logging_obj: Final = LiteLLMLoggingObj( + model="claude-haiku-5-5", + messages=messages, + stream=True, + call_type=CallTypes.anthropic_messages.value, + start_time=None, + litellm_call_id=owner_call_id, + function_id="mcp-replayed-stream", + ) + assert logging_obj.claim_async_success_log() + await handler.async_log_success_event( + kwargs={"litellm_call_id": owner_call_id, "stream": False}, + response_obj=None, + start_time=None, + end_time=reused_at - timedelta(seconds=1), + ) + assert await in_flight() == 1 + + replay: Final = FakeAnthropicMessagesStreamIterator( + response={ + "id": "msg_replayed", + "type": "message", + "role": "assistant", + "model": "claude-haiku-5-5", + "content": [{"type": "text", "text": "2 + 3 = 5"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 20, "output_tokens": 8}, + } + ) + with _override_litellm_callbacks([]): + streamed: Final = [ + chunk + async for chunk in ProxyBaseLLMRequestProcessing.async_sse_data_generator( + response=replay, + user_api_key_dict=user_api_key_dict, + request_data={ + "model": "claude-haiku-5-5", + "litellm_call_id": owner_call_id, + "litellm_logging_obj": logging_obj, + }, + proxy_logging_obj=proxy_logging_obj, + ) + ] + + assert b"event: message_stop" in streamed[-1] + assert await in_flight() == 0 + + @pytest.mark.asyncio async def test_mcp_tool_call_reusing_request_slot_keeps_request_max_parallel_headers() -> None: api_key: Final = hash_token("sk-mcp-reuse-headers")