fix(mcp): port the replay guard onto main's router core and MCP paths

Main moved the router retry and fallback loop into router_python_core.py and renamed the MCP client, server manager and Anthropic handler modules. Re-apply the same guard there: tag an error raised after an MCP tool call was sent, and let the router surface a tagged error instead of retrying or falling back

The Anthropic loop now also awaits with_safeguard_results after the tools ran, and the streamed chat follow-up is prepared after them, so both run under the guard too. Main no longer lists the MCP tools a second time after the Responses follow-up, so that guard and its test are gone

Signed-off-by: Atharva-Kanherkar <atharvakanherkar25@gmail.com>
This commit is contained in:
Atharva-Kanherkar 2026-11-03 15:12:21 +09:00
parent 295678e9ee
commit a66cb5ea25
12 changed files with 764 additions and 19 deletions

View file

@ -13,6 +13,7 @@ from typing import Any, Final, NamedTuple
from litellm._logging import verbose_logger
from litellm.responses.mcp.request_context import MCPRequestContext
from litellm.router_utils.mcp_tool_execution import MCPToolReplayGuard
from litellm.types.llms.anthropic import (
AnthropicMessagesTool,
AnthropicMessagesToolResultParam,
@ -149,6 +150,7 @@ async def anthropic_messages_with_mcp(
messages=list(working_messages), stream=False, **loop_call_args
)
replay_guard: Final = MCPToolReplayGuard()
for _ in range(MAX_MCP_TOOL_USE_ITERATIONS):
if _get_stop_reason(response) != "tool_use":
break
@ -170,6 +172,7 @@ async def anthropic_messages_with_mcp(
litellm_trace_id=context.litellm_trace_id,
request_tags=list(context.request_tags) if context.request_tags else None,
guardrail_context=context.guardrail_context,
on_tool_dispatched=replay_guard.record_tool_dispatch,
)
# Every tool call was skipped, so there is nothing to feed back; a
@ -182,9 +185,10 @@ async def anthropic_messages_with_mcp(
{"role": "assistant", "content": list(_get_response_content(response))},
_build_tool_result_message(tool_results),
)
response = await _AnthropicMessagesCall(fn=litellm.anthropic_messages).fn(
messages=list(working_messages), stream=False, **loop_call_args
)
with replay_guard:
response = await _AnthropicMessagesCall(fn=litellm.anthropic_messages).fn(
messages=list(working_messages), stream=False, **loop_call_args
)
else:
verbose_logger.warning(
"MCP tool loop hit its %s iteration cap for model %s; returning the last response",
@ -192,11 +196,12 @@ async def anthropic_messages_with_mcp(
model,
)
client_response: Final = (
await with_safeguard_results(response, request_safeguards_evaluator(kwargs, working_messages))
if classified_here
else response
)
with replay_guard:
client_response: Final = (
await with_safeguard_results(response, request_safeguards_evaluator(kwargs, working_messages))
if classified_here
else response
)
if stream:
from litellm.llms.anthropic.messages.native.fake_stream_iterator import (
FakeAnthropicMessagesStreamIterator,

View file

@ -1146,6 +1146,7 @@ class MCPClient:
raise_on_error: bool = False,
allow_input_required: bool = False,
persistent_session: "PersistentMCPSession | None" = None,
on_dispatch: Callable[[], None] | None = None,
) -> MCPCallToolResult | InputRequiredResult:
"""
Call an MCP Tool.
@ -1162,6 +1163,8 @@ class MCPClient:
allow_input_required: When True, a 2026-07-28 upstream may answer with an interim
``InputRequiredResult`` and it is returned as is. The SDK rejects it otherwise, so
callers only opt in when the downstream side can carry it.
on_dispatch: Called right before the tool call is sent on a ready session, so a caller
can tell a call the server may have run from one that never left.
"""
verbose_logger.info("MCP client calling tool '%s'", call_tool_request_params.name)
@ -1195,6 +1198,8 @@ class MCPClient:
async def request(
responses: InputResponses | None, state: str | None
) -> MCPCallToolResult | InputRequiredResult:
if on_dispatch is not None:
on_dispatch()
return await session.call_tool(
name=call_tool_request_params.name,
arguments=call_tool_request_params.arguments,

View file

@ -6540,6 +6540,7 @@ class MCPServerManager:
client_ip: str | None = None,
allow_input_required: bool = False,
persistent_session: PersistentMCPSession | None = None,
on_dispatch: Callable[[], None] | None = None,
) -> CallToolResult | InputRequiredResult:
"""Call a token_exchange (OBO) tool; on an upstream 401/403 re-mint the token once and retry.
@ -6556,6 +6557,7 @@ class MCPServerManager:
raise_on_error=True,
allow_input_required=allow_input_required,
persistent_session=persistent_session,
on_dispatch=on_dispatch,
)
except Exception as exc:
auth_failure: Final = _extract_upstream_auth_failure(exc)
@ -6583,6 +6585,7 @@ class MCPServerManager:
host_progress_callback=host_progress_callback,
allow_input_required=allow_input_required,
persistent_session=retry_persistent_session,
on_dispatch=on_dispatch,
)
async def _call_regular_mcp_tool(
@ -6603,6 +6606,7 @@ class MCPServerManager:
allow_input_required: bool = False,
input_responses: InputResponses | None = None,
request_state: str | None = None,
on_dispatch: Callable[[], None] | None = None,
) -> CallToolResult | InputRequiredResult:
"""
Call a regular MCP tool using the MCP client.
@ -6759,6 +6763,7 @@ class MCPServerManager:
client_ip=client_ip,
allow_input_required=allow_input_required,
persistent_session=persistent_session,
on_dispatch=on_dispatch,
)
else:
# Scoped to the two client-forwarded token modes this stack introduced; legacy
@ -6776,6 +6781,7 @@ class MCPServerManager:
host_progress_callback=host_progress_callback,
allow_input_required=allow_input_required,
persistent_session=persistent_session,
on_dispatch=on_dispatch,
)
# The client-forwarded modes carry the caller's own upstream token, so an upstream
# 401 (expired/invalid token) is the caller's to resolve: relay it as
@ -6793,6 +6799,7 @@ class MCPServerManager:
raise_on_error=True,
allow_input_required=allow_input_required,
persistent_session=persistent_session,
on_dispatch=on_dispatch,
)
except Exception as e:
required_scope: Final = upstream_insufficient_scope(e) if managed_per_user else None
@ -7072,6 +7079,7 @@ class MCPServerManager:
wire_compat: WireCompat = WireCompat.LEGACY,
input_responses: InputResponses | None = None,
request_state: str | None = None,
on_dispatch: Callable[[], None] | None = None,
*,
guardrail_auth: UserAPIKeyAuth | None = None,
catalog_auth_header: str | None | EllipsisType = ...,
@ -7094,6 +7102,8 @@ class MCPServerManager:
litellm_logging_obj: Optional request logger the guardrail hooks record
their evaluations onto, so MCP guardrail activity reaches the
Guardrails Monitor. See ``pre_call_tool_check``
on_dispatch: Optional callback run right before the call is sent to the server,
after its headers and client are ready, even if the server never returns a result
Returns:
@ -7214,6 +7224,8 @@ class MCPServerManager:
resolved_token: Final = request_resolved_auth_headers.set(resolved_auth_headers)
try:
async with self._limit_outbound_concurrency(mcp_server):
if on_dispatch is not None:
on_dispatch()
return await self._call_openapi_tool_handler(mcp_server, name, arguments, wire_compat)
finally:
request_auth_header.reset(auth_token)
@ -7239,6 +7251,7 @@ class MCPServerManager:
allow_input_required=wire_compat is WireCompat.MODERN,
input_responses=input_responses,
request_state=request_state,
on_dispatch=on_dispatch,
)
return await self._gather_openapi_tool_tasks(tasks, proxy_logging_obj)

View file

@ -68,6 +68,7 @@ from litellm.responses.mcp.round_billing import (
)
from litellm.responses.mcp.tool_precheck import may_name_litellm_mcp_gateway
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.router_utils.mcp_tool_execution import MCPToolReplayGuard
from litellm.types.llms.openai import (
PromptObject,
Reasoning,
@ -472,6 +473,7 @@ async def aresponses_api_with_mcp(
)
tool_trace_id: Final = _request_trace_id(request_kwargs)
replay_guard: Final = MCPToolReplayGuard()
tool_results: Final = await LiteLLM_Proxy_MCP_Handler.execute_tool_calls(
tool_server_map=tool_server_map,
served_tools=original_mcp_tools,
@ -488,6 +490,7 @@ async def aresponses_api_with_mcp(
MappingProxyType({**kwargs, "metadata": metadata, "model": model})
),
request_trace_id=_langfuse_request_trace_id(request_kwargs, tool_trace_id),
on_tool_dispatched=replay_guard.record_tool_dispatch,
)
if tool_results:
@ -514,8 +517,11 @@ async def aresponses_api_with_mcp(
tool_calls=tool_calls, tool_results=tool_results
)
with internal_sub_call(
logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObj) else None
with (
internal_sub_call(
logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObj) else None
),
replay_guard,
):
final_response = await LiteLLM_Proxy_MCP_Handler.make_follow_up_call(
follow_up_input=follow_up_input,

View file

@ -22,6 +22,7 @@ from litellm.responses.mcp.round_billing import (
first_round_billed_on_failure,
summed_cost_breakdown,
)
from litellm.router_utils.mcp_tool_execution import MCPToolReplayGuard
from litellm.types.utils import Message, ModelResponse, Usage
from litellm.utils import CustomStreamWrapper
@ -261,6 +262,7 @@ async def acompletion_with_mcp(
self.follow_up_stream = None
self.follow_up_iterator = None
self.follow_up_exhausted = False
self.replay_guard = MCPToolReplayGuard()
self._round_billing: ChatStreamRoundBilling | None = engaged_round_billing(initial_stream.logging_obj)
self._unsent_usage_chunk: ModelResponseStream | None = None
@ -354,7 +356,8 @@ async def acompletion_with_mcp(
async def _follow_up_or_release_first_round(self, usage_chunk: ModelResponseStream | None) -> None:
if self.tool_results and self.complete_response:
await self._prepare_follow_up_call()
with self.replay_guard:
await self._prepare_follow_up_call()
if self.follow_up_stream is not None:
return
self._unsent_usage_chunk = usage_chunk
@ -438,7 +441,8 @@ async def acompletion_with_mcp(
verbose_logger.debug("Follow-up stream iterator created")
try:
chunk = await self.follow_up_iterator.__anext__()
with self.replay_guard:
chunk = await self.follow_up_iterator.__anext__()
from litellm._logging import verbose_logger
verbose_logger.debug("Follow-up chunk yielded: %s", chunk)
@ -509,6 +513,7 @@ async def acompletion_with_mcp(
litellm_trace_id=self.litellm_trace_id,
request_tags=self.request_tags,
guardrail_context=context.guardrail_context,
on_tool_dispatched=self.replay_guard.record_tool_dispatch,
)
async def _prepare_follow_up_call(self):
@ -679,6 +684,7 @@ async def acompletion_with_mcp(
)
return initial_response
replay_guard: Final = MCPToolReplayGuard()
with first_round_billed_on_failure(logging_obj, initial_response):
tool_results: Final = await LiteLLM_Proxy_MCP_Handler.execute_tool_calls(
tool_server_map=tool_server_map,
@ -693,6 +699,7 @@ async def acompletion_with_mcp(
litellm_trace_id=context.litellm_trace_id,
request_tags=request_tags,
guardrail_context=context.guardrail_context,
on_tool_dispatched=replay_guard.record_tool_dispatch,
)
if not tool_results:
@ -713,7 +720,10 @@ async def acompletion_with_mcp(
follow_up_call_args["messages"] = follow_up_messages
follow_up_call_args["stream"] = stream
with internal_sub_call(logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObj) else None):
with (
internal_sub_call(logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObj) else None),
replay_guard,
):
final_response: Final = await litellm_acompletion(**follow_up_call_args)
if not isinstance(final_response, ModelResponse):

View file

@ -740,8 +740,14 @@ class LiteLLM_Proxy_MCP_Handler:
guardrail_context: Mapping[str, object] | None = None,
served_tools: Sequence[MCPTool] | None = None,
request_trace_id: str | None = None,
on_tool_dispatched: Callable[[], None] | None = None,
) -> list[MCPToolResult]:
"""Execute tool calls and return results."""
"""Execute tool calls and return results.
``on_tool_dispatched`` runs for every call that is sent to its MCP server, even one that then
times out. Calls rejected or failing before that, for example on a pre-call guardrail or a
credential the caller has not set, do not trigger it.
"""
from fastapi import HTTPException
from litellm._uuid import uuid
@ -908,6 +914,7 @@ class LiteLLM_Proxy_MCP_Handler:
proxy_logging_obj=proxy_logging_obj,
litellm_logging_obj=litellm_logging_obj,
guardrail_context=guardrail_context,
on_dispatch=on_tool_dispatched,
listed_tool=(
next((tool for tool in served_tools if tool.name == tool_name), None)
if served_tools is not None

View file

@ -366,6 +366,7 @@ from litellm.router_utils.kubernetes_pod_discovery import (
resolve_pods_after,
resolve_pods_after_bound,
)
from litellm.router_utils.mcp_tool_execution import mcp_tools_executed
from litellm.router_utils.pre_call_checks.continuation_prefill_check import (
MID_STREAM_CONTINUATION_MARKER,
RESPONSES_CONTINUATION_KWARG,
@ -9135,6 +9136,7 @@ class PythonCore(DeprecatedRouterMembers):
or original_model_group is None
or is_guardrail_intervention(e)
or unsupported_params_without_fallback
or mcp_tools_executed(e)
):
raise e
@ -9559,7 +9561,7 @@ class PythonCore(DeprecatedRouterMembers):
response = add_retry_headers_to_response(response=response, attempted_retries=0, max_retries=None)
return response
except Exception as e:
if is_guardrail_intervention(e):
if is_guardrail_intervention(e) or mcp_tools_executed(e):
raise
current_attempt = None
original_exception = e
@ -9670,6 +9672,8 @@ class PythonCore(DeprecatedRouterMembers):
# Always track the latest error so we raise the most
# recent exception instead of the first one.
original_exception = e
if mcp_tools_executed(e):
raise
## LOGGING
kwargs = self._core_log_retry(kwargs=kwargs, e=e)

View file

@ -0,0 +1,62 @@
from collections.abc import Iterator
from types import TracebackType
from typing import Final
_MCP_TOOLS_EXECUTED_ATTR: Final = "_litellm_mcp_tools_executed"
_MAX_EXCEPTION_CHAIN_DEPTH: Final = 20
def mark_mcp_tools_executed(exception: BaseException) -> None:
"""Tag an error raised after an MCP gateway loop already executed tool calls.
Retrying or falling back re-runs the whole loop, which executes those tool calls
again, so the router surfaces a tagged error instead of replaying the request. The
exception is tagged rather than wrapped so the client sees the same error.
"""
setattr(exception, _MCP_TOOLS_EXECUTED_ATTR, True)
def _exception_chain(exception: BaseException) -> Iterator[BaseException]:
current = exception # rebind-ok: advances one link per iteration of the bounded walk
for _ in range(_MAX_EXCEPTION_CHAIN_DEPTH):
yield current
following = current.__cause__ or current.__context__
if following is None:
return
current = following
def mcp_tools_executed(exception: BaseException | None) -> bool:
"""Whether ``exception`` or an exception in its chain was tagged by ``mark_mcp_tools_executed``.
The chain matters because a caller can re-raise a tagged error as a new exception, as
``exception_type`` does for anything that is not already a LiteLLM exception.
"""
return exception is not None and any(
getattr(link, _MCP_TOOLS_EXECUTED_ATTR, False) is True for link in _exception_chain(exception)
)
class MCPToolReplayGuard:
"""Tags errors raised inside ``with guard:`` once the request has sent an MCP tool call to a server."""
__slots__ = ("_tool_dispatched",)
def __init__(self) -> None:
self._tool_dispatched: bool = False
def record_tool_dispatch(self) -> None:
self._tool_dispatched = True
def __enter__(self) -> None:
return None
def __exit__(
self,
exc_type: type[BaseException] | None,
exc: BaseException | None,
traceback: TracebackType | None,
) -> None:
if self._tool_dispatched and isinstance(exc, Exception):
mark_mcp_tools_executed(exc)

View file

@ -1035,6 +1035,36 @@ async def test_call_tool_does_not_log_arguments():
assert "gpt-5-mini" not in logged
@pytest.mark.asyncio
@pytest.mark.parametrize("session_opens", [True, False])
async def test_call_tool_reports_dispatch_only_when_the_call_is_sent(session_opens):
"""
on_dispatch marks the moment a tool call can reach the server, so it fires right before the
session sends the call and never when the session could not be opened.
"""
from mcp.types import CallToolRequestParams
events: list[str] = []
async def send(**_):
events.append("sent")
return MagicMock()
async def run_with_session(operation, quiet_on_error=False):
if not session_opens:
raise ConnectionError("cannot reach the server")
return await operation(MagicMock(call_tool=send))
client = MCPClient(server_url="http://test-server")
client.run_with_session = run_with_session
await client.call_tool(
CallToolRequestParams(name="search_tool", arguments={}), on_dispatch=lambda: events.append("dispatched")
)
assert events == (["dispatched", "sent"] if session_opens else [])
@pytest.mark.asyncio
async def test_get_prompt_does_not_log_arguments():
from mcp.types import GetPromptRequestParams

View file

@ -67,8 +67,9 @@ class _AdmissionProbeClient(MCPClient):
raise_on_error: bool = False,
allow_input_required: bool = False,
persistent_session: PersistentMCPSession | None = None,
on_dispatch: Callable[[], None] | None = None,
) -> CallToolResult:
del call_tool_request_params, host_progress_callback, raise_on_error, allow_input_required
del call_tool_request_params, host_progress_callback, raise_on_error, allow_input_required, on_dispatch
if self._submitted is not None:
self._submitted.set()
if persistent_session is None:
@ -121,6 +122,7 @@ def _patch_client_with_tracker(manager: MCPServerManager, tracker: _ConcurrencyT
host_progress_callback=None,
allow_input_required=False,
persistent_session: PersistentMCPSession | None = None,
on_dispatch=None,
):
tracker.enter(server.server_id)
try:

View file

@ -7644,6 +7644,7 @@ class TestMCPServerManager:
host_progress_callback=None,
allow_input_required=False,
persistent_session: PersistentMCPSession | None = None,
on_dispatch=None,
):
# Return a mock CallToolResult
result = MagicMock(spec=CallToolResult)
@ -7682,6 +7683,30 @@ class TestMCPServerManager:
# Verify the MCP client call was awaited exactly once
assert mock_client.call_tool.await_count == 1
@pytest.mark.asyncio
async def test_call_tool_hands_on_dispatch_to_the_mcp_client(self):
"""The MCP client fires on_dispatch when it sends the call, so call_tool must hand it down."""
from mcp.types import CallToolResult
auth = self._unrestricted_auth()
manager, proxy_logging_obj = self._manager_ready_for_call_tool(
[MCPTool(name="test_tool", inputSchema={"type": "object"})],
caller=ListedToolsCaller(user_api_key_auth=auth),
)
manager.create_mcp_client = AsyncMock(return_value=_RetryFakeClient(result=CallToolResult(content=[])))
dispatched = MagicMock()
await manager.call_tool(
server_name="test-server",
name="test_tool",
arguments={},
user_api_key_auth=auth,
proxy_logging_obj=proxy_logging_obj,
on_dispatch=dispatched,
)
dispatched.assert_called_once_with()
@staticmethod
def _manager_ready_for_call_tool(
listed_tools: list[MCPTool], caller: ListedToolsCaller | None = None
@ -12179,9 +12204,12 @@ class _RetryFakeClient:
raise_on_error=False,
allow_input_required=False,
persistent_session: PersistentMCPSession | None = None,
on_dispatch=None,
):
self.attempts += 1
self.persistent_sessions.append(persistent_session)
if on_dispatch is not None:
on_dispatch()
if self._raises is not None:
if raise_on_error:
raise self._raises
@ -12319,6 +12347,56 @@ class TestOBOCallToolRetry:
manager._cred_provider.invalidate_credentials.assert_awaited_once()
assert first.attempts == 1 and retry.attempts == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("path", ["plain", "relay", "obo_retry"])
async def test_call_regular_reports_every_sent_attempt_through_on_dispatch(self, path):
"""on_dispatch tells the caller a tool may have run, so every attempt that is sent must reach
it: once on the plain and relay paths, twice when the OBO path retries after an upstream 401."""
manager = self._manager()
success = CallToolResult(content=[], isError=False)
clients = [_RetryFakeClient(raises=_UpstreamAuthError(401)), _RetryFakeClient(result=success)]
manager.create_mcp_client = AsyncMock(side_effect=clients if path == "obo_retry" else clients[1:])
servers = {
"plain": MCPServer(
server_id="plain-srv", name="plain", url="https://upstream.example/mcp", transport=MCPTransport.sse
),
"relay": MCPServer(
server_id="relay-srv",
name="relay",
url="https://upstream.example/mcp",
transport=MCPTransport.sse,
auth_type=MCPAuth.true_passthrough,
),
"obo_retry": MCPServer(
server_id="id-jag-srv",
name="id-jag",
url="https://upstream.example/mcp",
transport=MCPTransport.sse,
auth_type=MCPAuth.oauth2_id_jag,
client_id="gateway-client",
client_secret="gateway-secret",
token_exchange_endpoint="https://org-idp.example/oauth2/token",
id_jag_resource_token_endpoint="https://resource-as.example/oauth2/token",
),
}
dispatched = MagicMock()
result = await manager._call_regular_mcp_tool(
mcp_server=servers[path],
original_tool_name="tool",
arguments={},
tasks=[],
mcp_auth_header=None,
mcp_server_auth_headers=None,
oauth2_headers={"Authorization": "Bearer caller-id-token"} if path == "obo_retry" else None,
raw_headers={"authorization": "Bearer caller-token"} if path == "relay" else None,
proxy_logging_obj=None,
on_dispatch=dispatched,
)
assert result is success
assert dispatched.call_count == (2 if path == "obo_retry" else 1)
@pytest.mark.asyncio
async def test_non_auth_error_does_not_retry(self):
manager = self._manager()
@ -12464,6 +12542,7 @@ class TestOBOConcurrencyLimit:
raise_on_error=False,
allow_input_required=False,
persistent_session: PersistentMCPSession | None = None,
on_dispatch=None,
):
inflight["current"] += 1
inflight["peak"] = max(inflight["peak"], inflight["current"])
@ -12548,6 +12627,7 @@ class TestOBOConcurrencyLimit:
raise_on_error: bool = False,
allow_input_required: bool = False,
persistent_session: PersistentMCPSession | None = None,
on_dispatch: Callable[[], None] | None = None,
) -> CallToolResult:
sessions_seen.append(persistent_session)
return CallToolResult(content=[], isError=False)

View file

@ -12,14 +12,19 @@ Regression tests for https://github.com/BerriAI/litellm/issues/21343
import asyncio
import datetime
from collections.abc import Awaitable, Callable
from typing import Final
import json
from collections.abc import AsyncIterator, Awaitable, Callable
from typing import TYPE_CHECKING, Final, NamedTuple
from unittest.mock import AsyncMock, patch
import httpx
import pytest
import respx
import litellm
from litellm import Router
from litellm.exceptions import GuardrailRaisedException
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.llm_response_utils.response_metadata import (
_union_duration_ms,
@ -27,8 +32,14 @@ from litellm.litellm_core_utils.llm_response_utils.response_metadata import (
)
from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
from litellm.litellm_core_utils.rules import Rules
from litellm.router_utils.mcp_tool_execution import mark_mcp_tools_executed
from litellm.types.router import RetryPolicy
from litellm.utils import function_setup
if TYPE_CHECKING:
from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager
from litellm.proxy._types import UserAPIKeyAuth
def _make_rate_limit_error(message="Rate limited"):
"""Create a RateLimitError for testing."""
@ -66,7 +77,7 @@ def _make_not_found_error(message="Model not found"):
)
def _create_router(num_retries=2):
def _create_router(num_retries=2, retry_policy=None):
"""Create a Router with two deployments for testing."""
return Router(
model_list=[
@ -86,6 +97,7 @@ def _create_router(num_retries=2):
},
],
num_retries=num_retries,
retry_policy=retry_policy,
)
@ -357,3 +369,512 @@ async def test_retry_attempts_accumulate_timing_in_shared_request_metadata():
assert timing_metrics["litellm_overhead_time_ms"] == pytest.approx(
round(total_response_time_ms - union_duration_ms, 4)
)
def _make_follow_up_error(tools_executed: bool) -> litellm.InternalServerError:
error: Final = litellm.InternalServerError(message="follow-up failed", llm_provider="openai", model="gpt-4")
if tools_executed:
mark_mcp_tools_executed(error)
return error
@pytest.mark.asyncio
@pytest.mark.parametrize("retry_policy", [None, RetryPolicy(InternalServerErrorRetries=2)])
@pytest.mark.parametrize("tools_executed", [True, False])
async def test_500_after_mcp_tool_execution_is_not_retried(retry_policy, tools_executed):
"""
Regression test for #43153: a retry re-runs the whole MCP gateway loop, so a
failure raised after the loop executed tool calls must surface instead of
executing those tools again. A retry policy that retries 500s must not override
that, and a 500 raised before any tool ran must still be retried.
"""
router: Final = _create_router(num_retries=2, retry_policy=retry_policy)
error: Final = _make_follow_up_error(tools_executed)
attempt: Final = AsyncMock(side_effect=error)
with pytest.raises(litellm.InternalServerError) as exc_info:
await router.async_function_with_retries(**{**_base_kwargs(), "original_function": attempt, "num_retries": 2})
assert exc_info.value is error
assert attempt.await_count == (1 if tools_executed else 3)
@pytest.mark.asyncio
async def test_500_after_mcp_tool_execution_stops_an_ongoing_retry_loop():
"""
The first attempt fails before any tool ran and is retried; the retry executes
the tools and then fails, so the loop must stop there instead of running them
a second time.
"""
router: Final = _create_router(num_retries=3)
after_tools: Final = _make_follow_up_error(tools_executed=True)
attempt: Final = AsyncMock(side_effect=[_make_follow_up_error(tools_executed=False), after_tools])
with pytest.raises(litellm.InternalServerError) as exc_info:
await router.async_function_with_retries(**{**_base_kwargs(), "original_function": attempt, "num_retries": 3})
assert exc_info.value is after_tools
assert attempt.await_count == 2
@pytest.mark.asyncio
@pytest.mark.parametrize("tools_executed", [True, False])
async def test_500_after_mcp_tool_execution_does_not_fall_back(tools_executed):
"""
A fallback re-runs the whole MCP gateway loop on the fallback group, so it must
not start once the primary attempt executed tool calls.
"""
router: Final = Router(
model_list=[
{"model_name": "test-model", "litellm_params": {"model": "openai/gpt-4", "api_key": "fake-key-1"}},
{"model_name": "backup-model", "litellm_params": {"model": "openai/gpt-4", "api_key": "fake-key-2"}},
],
num_retries=0,
fallbacks=[{"test-model": ["backup-model"]}],
)
error: Final = _make_follow_up_error(tools_executed)
fallback_response: Final = litellm.ModelResponse()
async def primary_fails(**kwargs):
if kwargs["model"] == "test-model":
raise error
return fallback_response
attempt: Final = AsyncMock(side_effect=primary_fails)
call: Final = router.async_function_with_fallbacks(
**{**_base_kwargs(), "original_function": attempt, "num_retries": 0}
)
if tools_executed:
with pytest.raises(litellm.InternalServerError) as exc_info:
await call
assert exc_info.value is error
else:
assert await call is fallback_response
attempted_models: Final = [awaited.kwargs["model"] for awaited in attempt.await_args_list]
assert attempted_models == (["test-model"] if tools_executed else ["test-model", "backup-model"])
_UPSTREAM: Final = "https://upstream.example/v1"
_TOOL_NAME: Final = "ledger-commit_write"
def _ledger_tools() -> list[dict[str, str]]:
return [{"type": "mcp", "server_url": "litellm_proxy/ledger", "require_approval": "never"}]
def _ledger_caller() -> "UserAPIKeyAuth":
from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth
return UserAPIKeyAuth(
object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="test", mcp_servers=["ledger"])
)
class _LedgerGateway(NamedTuple):
tool: AsyncMock
list_tools: AsyncMock
manager: "MCPServerManager"
@pytest.fixture
def ledger_gateway(monkeypatch: pytest.MonkeyPatch) -> _LedgerGateway:
"""Serve one auto-approved MCP tool through the real MCP manager and tool registry."""
from mcp.types import Tool
from litellm.caching.caching import DualCache
from litellm.proxy import proxy_server
from litellm.proxy.mcp_server import mcp_server_manager, server, tool_registry
from litellm.proxy.mcp_server.faults.list_outcomes import AggregateToolListing
from litellm.proxy.utils import ProxyLogging
from litellm.types.mcp_server.mcp_server_manager import MCPServer
manager: Final = mcp_server_manager.MCPServerManager()
manager.registry = {
"ledger": MCPServer(
server_id="ledger",
name="ledger",
server_name="ledger",
transport="http",
url="https://ledger.example/mcp",
spec_path="ledger.json",
auth_type="none",
)
}
manager.tool_name_to_mcp_server_name_mapping = {_TOOL_NAME: "ledger"}
tool: Final = AsyncMock(return_value={"written": True})
registry: Final = tool_registry.MCPToolRegistry()
registry.register_tool(_TOOL_NAME, "Write one ledger entry", {"type": "object"}, tool)
list_tools: Final = AsyncMock(
return_value=AggregateToolListing(tools=[Tool(name=_TOOL_NAME, inputSchema={"type": "object"})], outcomes={})
)
monkeypatch.setattr(tool_registry, "global_mcp_tool_registry", registry)
monkeypatch.setattr(mcp_server_manager, "global_mcp_server_manager", manager)
monkeypatch.setattr(proxy_server, "proxy_logging_obj", ProxyLogging(user_api_key_cache=DualCache()))
monkeypatch.setattr(server, "_get_tools_from_mcp_servers", list_tools)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
return _LedgerGateway(tool=tool, list_tools=list_tools, manager=manager)
def _is_follow_up(request: httpx.Request) -> bool:
body: Final = json.loads(request.content)
messages: Final = body.get("messages") or []
items: Final = body["input"] if isinstance(body.get("input"), list) else []
return any(message.get("role") == "tool" for message in messages) or any(
item.get("type") == "function_call_output" for item in items
)
def _chat_completion(message: dict[str, object], finish_reason: str) -> httpx.Response:
return httpx.Response(
200,
json={
"id": "chatcmpl-1",
"object": "chat.completion",
"created": 0,
"model": "gpt-4.1-mini",
"choices": [{"index": 0, "message": message, "finish_reason": finish_reason}],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
},
)
def _responses_api_response(output_item: dict[str, object]) -> httpx.Response:
return httpx.Response(
200,
json={
"id": "resp_1",
"object": "response",
"created_at": 0,
"status": "completed",
"model": "gpt-4.1-mini",
"output": [output_item],
"parallel_tool_calls": True,
"tool_choice": "auto",
"tools": [],
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
},
)
def _tool_call(responses_api: bool) -> httpx.Response:
arguments: Final = json.dumps({"entry": "alpha"})
if responses_api:
return _responses_api_response(
{"type": "function_call", "id": "fc_1", "call_id": "call_1", "name": _TOOL_NAME, "arguments": arguments}
)
return _chat_completion(
{
"role": "assistant",
"content": None,
"tool_calls": [
{"id": "call_1", "type": "function", "function": {"name": _TOOL_NAME, "arguments": arguments}}
],
},
"tool_calls",
)
def _final_answer(responses_api: bool) -> httpx.Response:
if responses_api:
return _responses_api_response(
{
"type": "message",
"id": "msg_1",
"role": "assistant",
"status": "completed",
"content": [{"type": "output_text", "text": "Recorded alpha", "annotations": []}],
}
)
return _chat_completion({"role": "assistant", "content": "Recorded alpha"}, "stop")
class _FakeModel:
"""Answers both OpenAI routes: a tool call, then a final answer once the tool result arrives."""
def __init__(self, failing_initial_calls: int = 0, failing_follow_ups: int = 0) -> None:
self.calls: list[str] = []
self._failures: Final = {"initial": failing_initial_calls, "follow_up": failing_follow_ups}
def __call__(self, request: httpx.Request) -> httpx.Response:
kind: Final = "follow_up" if _is_follow_up(request) else "initial"
self.calls.append(kind)
if self.calls.count(kind) <= self._failures[kind]:
return httpx.Response(500, json={"error": {"message": "upstream failed", "type": "server_error"}})
responses_api: Final = request.url.path.endswith("/responses")
return _tool_call(responses_api) if kind == "initial" else _final_answer(responses_api)
class _StreamBrokenBeforeFirstChunk(httpx.AsyncByteStream):
async def __aiter__(self) -> AsyncIterator[bytes]:
raise httpx.ReadError("connection dropped before the first chunk")
yield b""
def _chat_tool_call_stream() -> httpx.Response:
tool_call: Final = {
"index": 0,
"id": "call_1",
"type": "function",
"function": {"name": _TOOL_NAME, "arguments": json.dumps({"entry": "alpha"})},
}
chunks: Final = (
{"role": "assistant", "content": None, "tool_calls": [tool_call]},
{},
)
body: Final = "".join(
"data: "
+ json.dumps(
{
"id": "chatcmpl-1",
"object": "chat.completion.chunk",
"created": 0,
"model": "gpt-4.1-mini",
"choices": [{"index": 0, "delta": delta, "finish_reason": None if delta else "tool_calls"}],
}
)
+ "\n\n"
for delta in chunks
)
return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=f"{body}data: [DONE]\n\n")
def _serve(model: _FakeModel) -> None:
respx.post(f"{_UPSTREAM}/chat/completions").mock(side_effect=model)
respx.post(f"{_UPSTREAM}/responses").mock(side_effect=model)
def _mcp_router(num_retries: int, fallbacks: list[dict[str, list[str]]] | None = None) -> litellm.Router:
deployment: Final = {"model": "openai/gpt-4.1-mini", "api_key": "sk-test", "api_base": _UPSTREAM}
return litellm.Router(
model_list=[
{"model_name": "repro-model", "litellm_params": deployment},
{"model_name": "repro-model", "litellm_params": deployment},
{"model_name": "backup-model", "litellm_params": deployment},
],
num_retries=num_retries,
fallbacks=fallbacks,
)
def _chat(router: litellm.Router) -> Awaitable[object]:
return router.acompletion(
model="repro-model",
messages=[{"role": "user", "content": "Record entry alpha"}],
tools=_ledger_tools(),
metadata={"user_api_key_auth": _ledger_caller()},
)
def _responses(router: litellm.Router) -> Awaitable[object]:
return router.aresponses(
model="repro-model",
input="Record entry alpha",
tools=_ledger_tools(),
litellm_metadata={"user_api_key_auth": _ledger_caller()},
)
def _messages(router: litellm.Router) -> Awaitable[object]:
return router.aanthropic_messages(
model="repro-model",
max_tokens=64,
messages=[{"role": "user", "content": "Record entry alpha"}],
tools=_ledger_tools(),
litellm_metadata={"user_api_key_auth": _ledger_caller()},
)
_ENDPOINTS: Final = pytest.mark.parametrize(
"send", [_chat, _responses, _messages], ids=["chat_completions", "responses", "messages"]
)
@pytest.mark.asyncio
@_ENDPOINTS
@pytest.mark.parametrize(
"router_kwargs",
[{"num_retries": 2}, {"num_retries": 0, "fallbacks": [{"repro-model": ["backup-model"]}]}],
ids=["retries", "fallbacks"],
)
@respx.mock
async def test_router_does_not_replay_an_executed_mcp_tool(
ledger_gateway: _LedgerGateway,
send: Callable[[litellm.Router], Awaitable[object]],
router_kwargs: dict[str, object],
):
"""
Regression test for #43153: an auto-approved MCP tool must run once per request.
Given: The model asks for the tool, the tool runs, and every follow-up model call returns a 500
When: The router has retries or a fallback group configured
Then: The client gets the 500 after one tool execution and one follow-up call
A retry or fallback re-runs the whole MCP loop, so before the fix every replay executed
the tool again (three writes with num_retries=2, two with one fallback).
"""
model: Final = _FakeModel(failing_follow_ups=3)
_serve(model)
with pytest.raises(litellm.InternalServerError):
await send(_mcp_router(**router_kwargs))
assert ledger_gateway.tool.await_count == 1
assert model.calls == ["initial", "follow_up"]
@pytest.mark.asyncio
@_ENDPOINTS
@respx.mock
async def test_router_retries_when_a_guardrail_blocked_every_mcp_tool(
ledger_gateway: _LedgerGateway,
send: Callable[[litellm.Router], Awaitable[object]],
monkeypatch: pytest.MonkeyPatch,
):
"""
Given: A pre-call guardrail blocks the tool call, so no tool runs
When: The follow-up model call fails once with a 500
Then: The router still retries, because replaying the request cannot repeat a side effect
"""
class BlockEveryToolCall(CustomGuardrail):
async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type):
raise GuardrailRaisedException(message="tool calls are blocked", blocked_content=True)
monkeypatch.setattr(
litellm, "callbacks", [BlockEveryToolCall(guardrail_name="block", event_hook="pre_mcp_call", default_on=True)]
)
model: Final = _FakeModel(failing_follow_ups=1)
_serve(model)
await send(_mcp_router(num_retries=2))
assert ledger_gateway.tool.await_count == 0
assert model.calls == ["initial", "follow_up", "initial", "follow_up"]
@pytest.mark.asyncio
@_ENDPOINTS
@respx.mock
async def test_router_does_not_replay_a_tool_call_that_failed_after_it_was_sent(
ledger_gateway: _LedgerGateway,
send: Callable[[litellm.Router], Awaitable[object]],
monkeypatch: pytest.MonkeyPatch,
):
"""
Given: The tool runs, but its call fails afterwards, here because a during-call guardrail
rejects the result, which leaves the router in the same spot as a timeout after the write
When: Every follow-up model call returns a 500
Then: The router does not replay the request, because the tool may already have written
"""
tool_ran: Final = asyncio.Event()
def write(**_: object) -> dict[str, bool]:
tool_ran.set()
return {"written": True}
class RejectToolResult(CustomGuardrail):
async def async_moderation_hook(self, data, user_api_key_dict, call_type):
await tool_ran.wait()
raise GuardrailRaisedException(message="tool result rejected", blocked_content=True)
ledger_gateway.tool.side_effect = write
monkeypatch.setattr(
litellm, "callbacks", [RejectToolResult(guardrail_name="reject", event_hook="during_mcp_call", default_on=True)]
)
model: Final = _FakeModel(failing_follow_ups=3)
_serve(model)
with pytest.raises(litellm.InternalServerError):
await send(_mcp_router(num_retries=2))
assert ledger_gateway.tool.await_count == 1
assert model.calls == ["initial", "follow_up"]
@pytest.mark.asyncio
@respx.mock
async def test_router_retries_when_an_mcp_tool_call_never_reached_its_server(
ledger_gateway: _LedgerGateway,
monkeypatch: pytest.MonkeyPatch,
):
"""
Given: The MCP server needs a per-user header the caller has not set, so the tool call fails
before anything is sent to the server
When: The follow-up model call fails once with a 500
Then: The router still retries, because replaying the request cannot repeat a side effect
"""
registry: Final = ledger_gateway.manager.registry
monkeypatch.setitem(
registry,
"ledger",
registry["ledger"].model_copy(
update={
"spec_path": None,
"static_headers": {"Authorization": "Bearer ${LEDGER_TOKEN}"},
"env_vars": [{"name": "LEDGER_TOKEN", "scope": "user"}],
}
),
)
model: Final = _FakeModel(failing_follow_ups=1)
_serve(model)
await _chat(_mcp_router(num_retries=2))
assert model.calls == ["initial", "follow_up", "initial", "follow_up"]
@pytest.mark.asyncio
@respx.mock
async def test_router_does_not_fall_back_when_a_streamed_follow_up_breaks_after_a_tool_ran(
ledger_gateway: _LedgerGateway,
):
"""
Given: A streaming Chat Completions request whose tool ran
When: The follow-up stream breaks before its first chunk and a fallback group is configured
Then: The client gets the error instead of a fallback that runs the tool again
"""
calls: Final[list[str]] = []
def stream_model(request: httpx.Request) -> httpx.Response:
kind: Final = "follow_up" if _is_follow_up(request) else "initial"
calls.append(kind)
if kind == "follow_up":
return httpx.Response(
200, headers={"content-type": "text/event-stream"}, stream=_StreamBrokenBeforeFirstChunk()
)
return _chat_tool_call_stream()
respx.post(f"{_UPSTREAM}/chat/completions").mock(side_effect=stream_model)
stream: Final = await _mcp_router(num_retries=0, fallbacks=[{"repro-model": ["backup-model"]}]).acompletion(
model="repro-model",
messages=[{"role": "user", "content": "Record entry alpha"}],
tools=_ledger_tools(),
metadata={"user_api_key_auth": _ledger_caller()},
stream=True,
)
with pytest.raises(litellm.APIConnectionError):
async for _ in stream:
pass
assert ledger_gateway.tool.await_count == 1
assert calls == ["initial", "follow_up"]
@pytest.mark.asyncio
@respx.mock
async def test_router_still_retries_a_failure_before_any_mcp_tool_ran(ledger_gateway: _LedgerGateway):
"""
Given: The initial model call fails once with a 500, then asks for the tool
When: The router retries
Then: The retry succeeds and the tool runs once
"""
model: Final = _FakeModel(failing_initial_calls=1)
_serve(model)
await _chat(_mcp_router(num_retries=2))
assert ledger_gateway.tool.await_count == 1
assert model.calls == ["initial", "initial", "follow_up"]