From 8898a20da299231b6405a731f9b57ffe906e00d1 Mon Sep 17 00:00:00 2001 From: Atharva-Kanherkar <142440039+Atharva-Kanherkar@users.noreply.github.com> Date: Fri, 25 Sep 2026 16:05:10 +0530 Subject: [PATCH 01/15] fix(router): stop retries and fallbacks after MCP tools executed A retry or fallback re-runs the whole MCP gateway loop, so a failure raised after the loop already executed tool calls ran those tools again on every replay. Add a tag for such errors and have the router surface a tagged error from the retry loop and the fallback path instead of replaying the request The error is tagged rather than wrapped, like the advisor orchestration tag, so its type, status, and message reach the client unchanged. Untagged errors keep their current retry and fallback behavior Signed-off-by: Atharva-Kanherkar <142440039+Atharva-Kanherkar@users.noreply.github.com> --- litellm/router.py | 7 +- litellm/router_utils/mcp_tool_execution.py | 18 ++++ .../test_router_retry_non_retryable_errors.py | 97 ++++++++++++++++++- 3 files changed, 120 insertions(+), 2 deletions(-) create mode 100644 litellm/router_utils/mcp_tool_execution.py diff --git a/litellm/router.py b/litellm/router.py index 023b99cd64e..8e2944d6fe6 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -223,6 +223,7 @@ from litellm.router_utils.handle_error import ( send_llm_exception_alert, ) from litellm.router_utils.health_state_cache import DeploymentHealthCache +from litellm.router_utils.mcp_tool_execution import mcp_tools_executed from litellm.router_utils.pre_call_checks.deployment_affinity_check import ( DeploymentAffinityCheck, warn_on_unknown_model_group_affinity_flags, @@ -7196,7 +7197,7 @@ class Router: hop_depth: Final = kwargs.get("fallback_depth") nested_fallback_hop: Final = isinstance(hop_depth, int) and hop_depth > 0 - if disable_fallbacks is True or original_model_group is None: + if disable_fallbacks is True or original_model_group is None or mcp_tools_executed(e): raise e input_kwargs: Final = { @@ -7610,6 +7611,8 @@ class Router: response = add_retry_headers_to_response(response=response, attempted_retries=0, max_retries=None) return response except Exception as e: + if mcp_tools_executed(e): + raise current_attempt = None original_exception = e deployment_num_retries: Final = getattr(e, "num_retries", None) @@ -7709,6 +7712,8 @@ class Router: # Always track the latest error so we raise the most # recent exception instead of the first one. original_exception = e + if mcp_tools_executed(e): + raise ## LOGGING kwargs = self.log_retry(kwargs=kwargs, e=e) diff --git a/litellm/router_utils/mcp_tool_execution.py b/litellm/router_utils/mcp_tool_execution.py new file mode 100644 index 00000000000..81c0d6383ed --- /dev/null +++ b/litellm/router_utils/mcp_tool_execution.py @@ -0,0 +1,18 @@ +from typing import Final + +_MCP_TOOLS_EXECUTED_ATTR: Final = "_litellm_mcp_tools_executed" + + +def mark_mcp_tools_executed(exception: BaseException) -> None: + """Tag an error raised after an MCP gateway loop already executed tool calls. + + Retrying or falling back re-runs the whole loop, which executes those tool calls + again, so the router surfaces a tagged error instead of replaying the request. The + exception is tagged rather than wrapped so the client sees the same error. + """ + setattr(exception, _MCP_TOOLS_EXECUTED_ATTR, True) + + +def mcp_tools_executed(exception: BaseException | None) -> bool: + """Whether ``exception`` was tagged by ``mark_mcp_tools_executed``.""" + return getattr(exception, _MCP_TOOLS_EXECUTED_ATTR, False) is True diff --git a/tests/test_litellm/test_router_retry_non_retryable_errors.py b/tests/test_litellm/test_router_retry_non_retryable_errors.py index 98a7db7a079..9e29ae2d51b 100644 --- a/tests/test_litellm/test_router_retry_non_retryable_errors.py +++ b/tests/test_litellm/test_router_retry_non_retryable_errors.py @@ -27,6 +27,8 @@ from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( ) from litellm.litellm_core_utils.logging_utils import track_llm_api_timing from litellm.litellm_core_utils.rules import Rules +from litellm.router_utils.mcp_tool_execution import mark_mcp_tools_executed +from litellm.types.router import RetryPolicy from litellm.utils import function_setup @@ -66,7 +68,7 @@ def _make_not_found_error(message="Model not found"): ) -def _create_router(num_retries=2): +def _create_router(num_retries=2, retry_policy=None): """Create a Router with two deployments for testing.""" return Router( model_list=[ @@ -86,6 +88,7 @@ def _create_router(num_retries=2): }, ], num_retries=num_retries, + retry_policy=retry_policy, ) @@ -357,3 +360,95 @@ async def test_retry_attempts_accumulate_timing_in_shared_request_metadata(): assert timing_metrics["litellm_overhead_time_ms"] == pytest.approx( round(total_response_time_ms - union_duration_ms, 4) ) + + +def _make_follow_up_error(tools_executed: bool) -> litellm.InternalServerError: + error: Final = litellm.InternalServerError(message="follow-up failed", llm_provider="openai", model="gpt-4") + if tools_executed: + mark_mcp_tools_executed(error) + return error + + +@pytest.mark.asyncio +@pytest.mark.parametrize("retry_policy", [None, RetryPolicy(InternalServerErrorRetries=2)]) +@pytest.mark.parametrize("tools_executed", [True, False]) +async def test_500_after_mcp_tool_execution_is_not_retried(retry_policy, tools_executed): + """ + Regression test for #43153: a retry re-runs the whole MCP gateway loop, so a + failure raised after the loop executed tool calls must surface instead of + executing those tools again. A retry policy that retries 500s must not override + that, and a 500 raised before any tool ran must still be retried. + """ + router: Final = _create_router(num_retries=2, retry_policy=retry_policy) + error: Final = _make_follow_up_error(tools_executed) + make_call: Final = AsyncMock(side_effect=error) + + with ( + patch.object(router, "make_call", make_call), + patch.object(router, "_async_get_healthy_deployments", return_value=(["d1", "d2"], ["d1", "d2"])), + patch.object(router, "_time_to_sleep_before_retry", return_value=0), + pytest.raises(litellm.InternalServerError) as exc_info, + ): + await router.async_function_with_retries(num_retries=2, **_base_kwargs()) + + assert exc_info.value is error + assert make_call.await_count == (1 if tools_executed else 3) + + +@pytest.mark.asyncio +async def test_500_after_mcp_tool_execution_stops_an_ongoing_retry_loop(): + """ + The first attempt fails before any tool ran and is retried; the retry executes + the tools and then fails, so the loop must stop there instead of running them + a second time. + """ + router: Final = _create_router(num_retries=3) + after_tools: Final = _make_follow_up_error(tools_executed=True) + make_call: Final = AsyncMock(side_effect=[_make_follow_up_error(tools_executed=False), after_tools]) + + with ( + patch.object(router, "make_call", make_call), + patch.object(router, "_async_get_healthy_deployments", return_value=(["d1", "d2"], ["d1", "d2"])), + patch.object(router, "_time_to_sleep_before_retry", return_value=0), + pytest.raises(litellm.InternalServerError) as exc_info, + ): + await router.async_function_with_retries(num_retries=3, **_base_kwargs()) + + assert exc_info.value is after_tools + assert make_call.await_count == 2 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("tools_executed", [True, False]) +async def test_500_after_mcp_tool_execution_does_not_fall_back(tools_executed): + """ + A fallback re-runs the whole MCP gateway loop on the fallback group, so it must + not start once the primary attempt executed tool calls. + """ + router: Final = Router( + model_list=[ + {"model_name": "test-model", "litellm_params": {"model": "openai/gpt-4", "api_key": "fake-key-1"}}, + {"model_name": "backup-model", "litellm_params": {"model": "openai/gpt-4", "api_key": "fake-key-2"}}, + ], + num_retries=0, + fallbacks=[{"test-model": ["backup-model"]}], + ) + error: Final = _make_follow_up_error(tools_executed) + + async def primary_fails(original_function, *args, **kwargs): + if kwargs["model"] == "test-model": + raise error + return "fallback response" + + make_call: Final = AsyncMock(side_effect=primary_fails) + + with patch.object(router, "make_call", make_call): + if tools_executed: + with pytest.raises(litellm.InternalServerError) as exc_info: + await router.async_function_with_fallbacks(num_retries=0, **_base_kwargs()) + assert exc_info.value is error + else: + assert await router.async_function_with_fallbacks(num_retries=0, **_base_kwargs()) == "fallback response" + + attempted_models: Final = [call.kwargs["model"] for call in make_call.await_args_list] + assert attempted_models == (["test-model"] if tools_executed else ["test-model", "backup-model"]) From f89d03118f73185c8ba26cdabaf485f238914ef8 Mon Sep 17 00:00:00 2001 From: Atharva-Kanherkar <142440039+Atharva-Kanherkar@users.noreply.github.com> Date: Fri, 25 Sep 2026 16:09:54 +0530 Subject: [PATCH 02/15] fix(mcp): tag a chat follow-up failure after MCP tools executed When the non-streaming Chat Completions MCP loop has executed tool calls and the follow-up model call fails, tag the error so the router surfaces it instead of replaying the loop and executing the tools again The regression test drives a real Router through the MCP gateway with a counting tool and a 500 from the follow-up call. It asserts one tool execution and one follow-up call with retries and with a fallback group, and a control asserts that a 500 before any tool ran is still retried Signed-off-by: Atharva-Kanherkar <142440039+Atharva-Kanherkar@users.noreply.github.com> --- .../responses/mcp/chat_completions_handler.py | 7 +- .../mcp/test_chat_completions_handler.py | 174 ++++++++++++++++++ 2 files changed, 180 insertions(+), 1 deletion(-) diff --git a/litellm/responses/mcp/chat_completions_handler.py b/litellm/responses/mcp/chat_completions_handler.py index b17e0befba6..f16913bc221 100644 --- a/litellm/responses/mcp/chat_completions_handler.py +++ b/litellm/responses/mcp/chat_completions_handler.py @@ -10,6 +10,7 @@ from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, ) from litellm.responses.mcp.request_context import MCPRequestContext +from litellm.router_utils.mcp_tool_execution import mark_mcp_tools_executed from litellm.types.utils import Message, ModelResponse from litellm.utils import CustomStreamWrapper @@ -640,7 +641,11 @@ async def acompletion_with_mcp( follow_up_call_args["messages"] = follow_up_messages follow_up_call_args["stream"] = stream - response = await litellm_acompletion(**follow_up_call_args) + try: + response = await litellm_acompletion(**follow_up_call_args) + except Exception as follow_up_exception: + mark_mcp_tools_executed(follow_up_exception) + raise if isinstance(response, (ModelResponse, CustomStreamWrapper)): _add_mcp_metadata_to_response( response=response, diff --git a/tests/test_litellm/responses/mcp/test_chat_completions_handler.py b/tests/test_litellm/responses/mcp/test_chat_completions_handler.py index 54e98cc2b6c..9affcb26158 100644 --- a/tests/test_litellm/responses/mcp/test_chat_completions_handler.py +++ b/tests/test_litellm/responses/mcp/test_chat_completions_handler.py @@ -1479,3 +1479,177 @@ async def test_request_selected_mcp_guardrail_blocks_before_upstream(monkeypatch assert upstream.await_count == (0 if selected else 1) tool_message = model_call.await_args.kwargs["messages"][-1] assert ("request-selected MCP block" in tool_message["content"]) is selected + + +_REPLAY_UPSTREAM = "https://upstream.example/v1" + + +def _counting_ledger_mcp_tool(monkeypatch) -> AsyncMock: + """Serve one auto-approved MCP tool through the real manager and registry, returning the tool itself.""" + from mcp.types import Tool + + from litellm.caching.caching import DualCache + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server import mcp_server_manager, server, tool_registry + from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing + from litellm.proxy.utils import ProxyLogging + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + manager = mcp_server_manager.MCPServerManager() + manager.registry = { + "ledger": MCPServer( + server_id="ledger", + name="ledger", + server_name="ledger", + transport="http", + url="https://ledger.example/mcp", + spec_path="ledger.json", + auth_type="none", + ) + } + manager.tool_name_to_mcp_server_name_mapping = {"ledger-commit_write": "ledger"} + tool = AsyncMock(return_value={"written": True}) + registry = tool_registry.MCPToolRegistry() + registry.register_tool("ledger-commit_write", "Write one ledger entry", {"type": "object"}, tool) + monkeypatch.setattr(tool_registry, "global_mcp_tool_registry", registry) + monkeypatch.setattr(mcp_server_manager, "global_mcp_server_manager", manager) + monkeypatch.setattr(proxy_server, "proxy_logging_obj", ProxyLogging(user_api_key_cache=DualCache())) + monkeypatch.setattr( + server, + "_get_tools_from_mcp_servers", + AsyncMock( + return_value=AggregateToolListing( + tools=[Tool(name="ledger-commit_write", inputSchema={"type": "object"})], outcomes={} + ) + ), + ) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + return tool + + +def _is_follow_up(request) -> bool: + return any(message.get("role") == "tool" for message in json.loads(request.content)["messages"]) + + +def _upstream_call_kind(call) -> str: + return "follow_up" if _is_follow_up(call.request) else "initial" + + +def _chat_completion_response(message: dict, finish_reason: str) -> Response: + return Response( + 200, + json={ + "id": "chatcmpl-replay", + "object": "chat.completion", + "created": 0, + "model": "gpt-4.1-mini", + "choices": [{"index": 0, "message": message, "finish_reason": finish_reason}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + + +def _ledger_tool_call_response() -> Response: + return _chat_completion_response( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "ledger-commit_write", "arguments": '{"entry": "alpha"}'}, + } + ], + }, + "tool_calls", + ) + + +def _server_error_response() -> Response: + return Response(500, json={"error": {"message": "upstream failed", "type": "server_error"}}) + + +async def _router_mcp_completion(router: litellm.Router) -> ModelResponse: + from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth + + return await router.acompletion( + model="repro-model", + messages=[{"role": "user", "content": "Record entry alpha"}], + tools=[{"type": "mcp", "server_url": "litellm_proxy/ledger", "require_approval": "never"}], + metadata={ + "user_api_key_auth": UserAPIKeyAuth( + object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="test", mcp_servers=["ledger"]) + ) + }, + ) + + +def _replay_router(num_retries: int, fallbacks: list | None = None) -> litellm.Router: + deployment = {"model": "openai/gpt-4.1-mini", "api_key": "sk-test", "api_base": _REPLAY_UPSTREAM} + return litellm.Router( + # two replicas, so the router retries right away instead of backing off + model_list=[ + {"model_name": "repro-model", "litellm_params": deployment}, + {"model_name": "repro-model", "litellm_params": deployment}, + {"model_name": "backup-model", "litellm_params": deployment}, + ], + num_retries=num_retries, + fallbacks=fallbacks, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "router_kwargs", + [{"num_retries": 2}, {"num_retries": 0, "fallbacks": [{"repro-model": ["backup-model"]}]}], + ids=["retries", "fallbacks"], +) +@respx.mock +async def test_router_does_not_replay_an_executed_mcp_tool_after_the_follow_up_fails(monkeypatch, router_kwargs): + """ + Regression test for #43153: an auto-approved MCP tool must run once per request. + + Given: The model asks for the tool, the tool runs, and the follow-up model call returns a 500 + When: The router has retries or a fallback group configured + Then: The client gets that 500 after one tool execution and one follow-up call + + A retry or fallback re-runs the whole MCP loop, so before the fix every replay executed + the tool again (three writes with num_retries=2, two with one fallback) and the client + still received the same 500. + """ + tool = _counting_ledger_mcp_tool(monkeypatch) + upstream = respx.post(f"{_REPLAY_UPSTREAM}/chat/completions").mock( + side_effect=lambda request: _server_error_response() if _is_follow_up(request) else _ledger_tool_call_response() + ) + + with pytest.raises(litellm.InternalServerError) as exc_info: + await _router_mcp_completion(_replay_router(**router_kwargs)) + + assert exc_info.value.status_code == 500 + assert tool.await_count == 1 + assert [_upstream_call_kind(call) for call in upstream.calls] == ["initial", "follow_up"] + + +@pytest.mark.asyncio +@respx.mock +async def test_router_still_retries_a_failure_before_any_mcp_tool_ran(monkeypatch): + """ + Given: The initial model call fails once with a 500, then asks for the tool + When: The router retries + Then: The retry succeeds, the tool runs once, and the follow-up answer reaches the client + """ + tool = _counting_ledger_mcp_tool(monkeypatch) + upstream = respx.post(f"{_REPLAY_UPSTREAM}/chat/completions").mock( + side_effect=[ + _server_error_response(), + _ledger_tool_call_response(), + _chat_completion_response({"role": "assistant", "content": "Recorded alpha"}, "stop"), + ] + ) + + response = await _router_mcp_completion(_replay_router(num_retries=2)) + + assert response.choices[0].message.content == "Recorded alpha" + assert tool.await_count == 1 + assert [_upstream_call_kind(call) for call in upstream.calls] == ["initial", "initial", "follow_up"] From dcaeff71306aebc0f8e997890722c44ed74e7678 Mon Sep 17 00:00:00 2001 From: Atharva-Kanherkar <142440039+Atharva-Kanherkar@users.noreply.github.com> Date: Fri, 25 Sep 2026 16:12:18 +0530 Subject: [PATCH 03/15] fix(mcp): tag a Responses follow-up failure after MCP tools executed The non-streaming Responses MCP loop has the same replay problem as Chat Completions: once it has executed tool calls, a failed follow-up call made the router re-run the loop. Tag that failure so the router surfaces it, and leave a failure of the initial call untagged so it is still retried Signed-off-by: Atharva-Kanherkar <142440039+Atharva-Kanherkar@users.noreply.github.com> --- litellm/responses/main.py | 19 ++++++---- .../mcp/test_litellm_proxy_mcp_handler.py | 38 +++++++++++++++++++ 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 93c72bc2d3b..74d9ea96681 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -35,6 +35,7 @@ from litellm.responses.litellm_completion_transformation.handler import ( ) from litellm.responses.mcp.request_context import MCPRequestContext from litellm.responses.utils import ResponsesAPIRequestUtils +from litellm.router_utils.mcp_tool_execution import mark_mcp_tools_executed from litellm.types.llms.openai import ( PromptObject, Reasoning, @@ -375,13 +376,17 @@ async def aresponses_api_with_mcp( tool_calls=tool_calls, tool_results=tool_results ) - final_response = await LiteLLM_Proxy_MCP_Handler._make_follow_up_call( - follow_up_input=follow_up_input, - model=model, - all_tools=all_tools, - response_id=previous_response_id if persistence_disabled else response.id, - **follow_up_call_params, - ) + try: + final_response = await LiteLLM_Proxy_MCP_Handler._make_follow_up_call( + follow_up_input=follow_up_input, + model=model, + all_tools=all_tools, + response_id=previous_response_id if persistence_disabled else response.id, + **follow_up_call_params, + ) + except Exception as follow_up_exception: + mark_mcp_tools_executed(follow_up_exception) + raise # If streaming and we have tool execution events, wrap the response if ( diff --git a/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py b/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py index 57cebf489a2..ac5cd630bba 100644 --- a/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py +++ b/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py @@ -11,12 +11,14 @@ from fastapi import HTTPException from mcp.types import CallToolResult, TextContent from openai.types.responses.tool_param import Mcp +from litellm.exceptions import InternalServerError from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing from litellm.responses import main as responses_main from litellm.responses.mcp import litellm_proxy_mcp_handler as mcp_handler_module from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, ) +from litellm.router_utils.mcp_tool_execution import mcp_tools_executed from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.responses.main import OutputFunctionToolCall from litellm.types.utils import ModelResponse @@ -1295,3 +1297,39 @@ async def test_responses_discovery_logs_sanitized_caller_headers(monkeypatch: py logged: Final = setup.call_args.kwargs["metadata"]["headers"] assert logged == {"x-app-id": "app-a", "x-nuid": "user-a", "x-user-id": "identity-a"} assert headers["x-mcp-deepwiki-authorization"] == "upstream-sentinel" + + +@pytest.mark.parametrize("failing_call", ["initial", "follow_up"]) +@pytest.mark.asyncio +async def test_aresponses_api_with_mcp_tags_a_failure_after_tools_executed( + monkeypatch: pytest.MonkeyPatch, failing_call: str +): + """ + Regression test for #43153: a router retry or fallback re-runs the whole MCP loop, so + a follow-up failure raised after the loop executed tools must carry the tag that stops + the replay. A failure before any tool ran stays untagged and is still retried. + """ + error: Final = InternalServerError(message="upstream failed", llm_provider="openai", model="gpt-5") + model_calls: Final = AsyncMock( + side_effect=[error] if failing_call == "initial" else [_response_with_reasoning_and_tool_call(), error] + ) + execute: Final = AsyncMock(return_value=[{"tool_call_id": "call-1", "name": "foo", "result": "done"}]) + monkeypatch.setattr(responses_main, "aresponses", model_calls) + monkeypatch.setattr(mcp_handler_module, "aresponses", model_calls) + monkeypatch.setattr( + LiteLLM_Proxy_MCP_Handler, + "_process_mcp_tools_without_openai_transform", + AsyncMock(return_value=([], {"foo": "litellm_proxy"})), + ) + monkeypatch.setattr(LiteLLM_Proxy_MCP_Handler, "_execute_tool_calls", execute) + + with pytest.raises(InternalServerError) as exc_info: + await responses_main.aresponses_api_with_mcp( + input="hi", + model="gpt-5", + tools=[{"type": "mcp", "server_url": "litellm_proxy", "require_approval": "never"}], + ) + + assert exc_info.value is error + assert mcp_tools_executed(error) is (failing_call == "follow_up") + assert execute.await_count == (1 if failing_call == "follow_up" else 0) From 7ccee389e9ff6451f24311d7fdbda83892d132fa Mon Sep 17 00:00:00 2001 From: Atharva-Kanherkar <142440039+Atharva-Kanherkar@users.noreply.github.com> Date: Fri, 25 Sep 2026 16:13:22 +0530 Subject: [PATCH 04/15] fix(mcp): tag a /v1/messages follow-up failure after MCP tools executed The /v1/messages MCP loop runs inside one router attempt for both streaming and non-streaming requests, so a failed follow-up call after tool execution made the router re-run the loop. Tag that failure so the router surfaces it, and leave a failure of the initial call untagged so it is still retried Signed-off-by: Atharva-Kanherkar <142440039+Atharva-Kanherkar@users.noreply.github.com> --- .../messages/mcp_handler.py | 11 +++-- .../messages/test_mcp_handler.py | 48 +++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/mcp_handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/mcp_handler.py index a0585dfb369..c1313bdba5f 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/mcp_handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/mcp_handler.py @@ -13,6 +13,7 @@ from typing import Any, Final, NamedTuple from litellm._logging import verbose_logger from litellm.responses.mcp.request_context import MCPRequestContext +from litellm.router_utils.mcp_tool_execution import mark_mcp_tools_executed from litellm.types.llms.anthropic import ( AnthropicMessagesTool, AnthropicMessagesToolResultParam, @@ -169,9 +170,13 @@ async def anthropic_messages_with_mcp( {"role": "assistant", "content": list(_get_response_content(response))}, _build_tool_result_message(tool_results), ) - response = await _AnthropicMessagesCall(fn=litellm.anthropic_messages).fn( - messages=list(working_messages), stream=False, **base_call_args - ) + try: + response = await _AnthropicMessagesCall(fn=litellm.anthropic_messages).fn( + messages=list(working_messages), stream=False, **base_call_args + ) + except Exception as follow_up_exception: + mark_mcp_tools_executed(follow_up_exception) + raise else: verbose_logger.warning( "MCP tool loop hit its %s iteration cap for model %s; returning the last response", diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_mcp_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_mcp_handler.py index 93adde12c4b..9bc7eb3c03f 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_mcp_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_mcp_handler.py @@ -3,6 +3,7 @@ from unittest.mock import AsyncMock, patch import pytest +from litellm.exceptions import InternalServerError from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) @@ -13,6 +14,7 @@ from litellm.llms.anthropic.experimental_pass_through.messages.mcp_handler impor _build_tool_result_message, _extract_tool_use_blocks, ) +from litellm.router_utils.mcp_tool_execution import mcp_tools_executed MCP_REFERENCE = { "type": "mcp", @@ -274,3 +276,49 @@ async def test_anthropic_messages_with_mcp_stops_when_every_tool_call_is_skipped "With no tool results there is nothing to send back, so the loop must not call the model again" ) assert result == tool_use_response + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failing_call", ["initial", "follow_up"]) +async def test_anthropic_messages_with_mcp_tags_a_failure_after_tools_executed(failing_call): + """ + Regression test for #43153: a router retry or fallback re-runs the whole MCP loop, so + a follow-up failure raised after the loop executed tools must carry the tag that stops + the replay. A failure before any tool ran stays untagged and is still retried. + """ + from litellm.llms.anthropic.experimental_pass_through.messages import mcp_handler + from litellm.responses.mcp.request_context import MCPRequestContext + + error = InternalServerError(message="upstream failed", llm_provider="anthropic", model="claude-sonnet-4-5") + tool_use_response = { + "stop_reason": "tool_use", + "content": [{"type": "tool_use", "id": "toolu_1", "name": "t", "input": {}}], + } + execute = AsyncMock(return_value=[{"tool_call_id": "toolu_1", "result": "ok", "name": "t"}]) + model_calls = AsyncMock(side_effect=[error] if failing_call == "initial" else [tool_use_response, error]) + + with ( + patch.object(MCPRequestContext, "resolve", return_value=MCPRequestContext(user_api_key_auth="auth")), + patch.object( + import_module("litellm.responses.mcp.litellm_proxy_mcp_handler").LiteLLM_Proxy_MCP_Handler, + "_process_mcp_tools_without_openai_transform", + new=AsyncMock(return_value=([], {})), + ), + patch.object( + import_module("litellm.responses.mcp.litellm_proxy_mcp_handler").LiteLLM_Proxy_MCP_Handler, + "_execute_tool_calls", + new=execute, + ), + patch("litellm.anthropic_messages", new=model_calls), + pytest.raises(InternalServerError) as exc_info, + ): + await mcp_handler.anthropic_messages_with_mcp( + max_tokens=100, + messages=[{"role": "user", "content": "hi"}], + model="claude-sonnet-4-5", + tools=[MCP_REFERENCE], + ) + + assert exc_info.value is error + assert mcp_tools_executed(error) is (failing_call == "follow_up") + assert execute.await_count == (1 if failing_call == "follow_up" else 0) From 33a4363a1b95339d2aa40e911883e6589ad838c6 Mon Sep 17 00:00:00 2001 From: Atharva-Kanherkar <142440039+Atharva-Kanherkar@users.noreply.github.com> Date: Fri, 25 Sep 2026 17:02:56 +0530 Subject: [PATCH 05/15] fix(mcp): only block router replays after a tool actually executed The tag was set whenever tool results existed, including results for calls that a pre-call guardrail rejected before any MCP server ran them, so those requests lost their retries and fallbacks. The executor now reports each call an MCP server returned for, and a small guard tags failures only after that The guard also covers the Responses tool re-listing that runs after the follow-up call, and the router finds the tag through an exception's cause and context, because exception_type re-raises errors that are not LiteLLM exceptions as new objects The regression tests move to a test file for the new module. They drive a real Router through the MCP gateway for Chat Completions, Responses and /v1/messages with respx at the HTTP boundary, so no test patches a private class method Signed-off-by: Atharva-Kanherkar <142440039+Atharva-Kanherkar@users.noreply.github.com> --- .../messages/mcp_handler.py | 9 +- litellm/responses/main.py | 32 +- .../responses/mcp/chat_completions_handler.py | 9 +- .../mcp/litellm_proxy_mcp_handler.py | 9 +- litellm/router_utils/mcp_tool_execution.py | 37 ++- .../messages/test_mcp_handler.py | 48 --- .../mcp/test_chat_completions_handler.py | 174 ---------- .../mcp/test_litellm_proxy_mcp_handler.py | 38 --- .../router_utils/test_mcp_tool_execution.py | 312 ++++++++++++++++++ 9 files changed, 379 insertions(+), 289 deletions(-) create mode 100644 tests/test_litellm/router_utils/test_mcp_tool_execution.py diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/mcp_handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/mcp_handler.py index c1313bdba5f..b84c41a28f0 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/mcp_handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/mcp_handler.py @@ -13,7 +13,7 @@ from typing import Any, Final, NamedTuple from litellm._logging import verbose_logger from litellm.responses.mcp.request_context import MCPRequestContext -from litellm.router_utils.mcp_tool_execution import mark_mcp_tools_executed +from litellm.router_utils.mcp_tool_execution import MCPToolReplayGuard from litellm.types.llms.anthropic import ( AnthropicMessagesTool, AnthropicMessagesToolResultParam, @@ -138,6 +138,7 @@ async def anthropic_messages_with_mcp( messages=list(working_messages), stream=False, **base_call_args ) + replay_guard: Final = MCPToolReplayGuard() for _ in range(MAX_MCP_TOOL_USE_ITERATIONS): if _get_stop_reason(response) != "tool_use": break @@ -158,6 +159,7 @@ async def anthropic_messages_with_mcp( litellm_trace_id=context.litellm_trace_id, request_tags=list(context.request_tags) if context.request_tags else None, guardrail_context=context.guardrail_context, + on_tool_executed=replay_guard.record_tool_execution, ) # Every tool call was skipped, so there is nothing to feed back; a @@ -170,13 +172,10 @@ async def anthropic_messages_with_mcp( {"role": "assistant", "content": list(_get_response_content(response))}, _build_tool_result_message(tool_results), ) - try: + with replay_guard: response = await _AnthropicMessagesCall(fn=litellm.anthropic_messages).fn( messages=list(working_messages), stream=False, **base_call_args ) - except Exception as follow_up_exception: - mark_mcp_tools_executed(follow_up_exception) - raise else: verbose_logger.warning( "MCP tool loop hit its %s iteration cap for model %s; returning the last response", diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 74d9ea96681..c621713c0fc 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -35,7 +35,7 @@ from litellm.responses.litellm_completion_transformation.handler import ( ) from litellm.responses.mcp.request_context import MCPRequestContext from litellm.responses.utils import ResponsesAPIRequestUtils -from litellm.router_utils.mcp_tool_execution import mark_mcp_tools_executed +from litellm.router_utils.mcp_tool_execution import MCPToolReplayGuard from litellm.types.llms.openai import ( PromptObject, Reasoning, @@ -338,6 +338,7 @@ async def aresponses_api_with_mcp( tools=tools, ) + replay_guard: Final = MCPToolReplayGuard() tool_results: Final = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( tool_server_map=tool_server_map, tool_calls=tool_calls, @@ -352,6 +353,7 @@ async def aresponses_api_with_mcp( guardrail_context=MCPRequestContext.resolve_guardrail_context( MappingProxyType({**kwargs, "metadata": metadata, "model": model}) ), + on_tool_executed=replay_guard.record_tool_execution, ) if tool_results: @@ -376,7 +378,7 @@ async def aresponses_api_with_mcp( tool_calls=tool_calls, tool_results=tool_results ) - try: + with replay_guard: final_response = await LiteLLM_Proxy_MCP_Handler._make_follow_up_call( follow_up_input=follow_up_input, model=model, @@ -384,9 +386,6 @@ async def aresponses_api_with_mcp( response_id=previous_response_id if persistence_disabled else response.id, **follow_up_call_params, ) - except Exception as follow_up_exception: - mark_mcp_tools_executed(follow_up_exception) - raise # If streaming and we have tool execution events, wrap the response if ( @@ -408,17 +407,18 @@ async def aresponses_api_with_mcp( # Add custom output elements to the final response (for non-streaming) elif isinstance(final_response, ResponsesAPIResponse): # Fetch MCP tools again for output elements (without OpenAI transformation) - ( - mcp_tools_for_output, - _, - ) = await LiteLLM_Proxy_MCP_Handler._process_mcp_tools_without_openai_transform( - user_api_key_auth=user_api_key_auth, - mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy, - mcp_auth_header=mcp_auth_header, - mcp_server_auth_headers=mcp_server_auth_headers, - request_tags=LiteLLM_Proxy_MCP_Handler._get_parent_request_tags(kwargs), - raw_headers=discovery_raw_headers, - ) + with replay_guard: + ( + mcp_tools_for_output, + _, + ) = await LiteLLM_Proxy_MCP_Handler._process_mcp_tools_without_openai_transform( + user_api_key_auth=user_api_key_auth, + mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + request_tags=LiteLLM_Proxy_MCP_Handler._get_parent_request_tags(kwargs), + raw_headers=discovery_raw_headers, + ) final_response = LiteLLM_Proxy_MCP_Handler._add_mcp_output_elements_to_response( response=final_response, mcp_tools_fetched=mcp_tools_for_output, diff --git a/litellm/responses/mcp/chat_completions_handler.py b/litellm/responses/mcp/chat_completions_handler.py index f16913bc221..3391ed0d9a2 100644 --- a/litellm/responses/mcp/chat_completions_handler.py +++ b/litellm/responses/mcp/chat_completions_handler.py @@ -10,7 +10,7 @@ from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, ) from litellm.responses.mcp.request_context import MCPRequestContext -from litellm.router_utils.mcp_tool_execution import mark_mcp_tools_executed +from litellm.router_utils.mcp_tool_execution import MCPToolReplayGuard from litellm.types.utils import Message, ModelResponse from litellm.utils import CustomStreamWrapper @@ -607,6 +607,7 @@ async def acompletion_with_mcp( return initial_response # Execute tool calls + replay_guard: Final = MCPToolReplayGuard() tool_results: Final = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( tool_server_map=tool_server_map, tool_calls=tool_calls, @@ -619,6 +620,7 @@ async def acompletion_with_mcp( litellm_trace_id=context.litellm_trace_id, request_tags=request_tags, guardrail_context=context.guardrail_context, + on_tool_executed=replay_guard.record_tool_execution, ) if not tool_results: @@ -641,11 +643,8 @@ async def acompletion_with_mcp( follow_up_call_args["messages"] = follow_up_messages follow_up_call_args["stream"] = stream - try: + with replay_guard: response = await litellm_acompletion(**follow_up_call_args) - except Exception as follow_up_exception: - mark_mcp_tools_executed(follow_up_exception) - raise if isinstance(response, (ModelResponse, CustomStreamWrapper)): _add_mcp_metadata_to_response( response=response, diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index 71f61079154..10ed52d6bca 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -696,8 +696,13 @@ class LiteLLM_Proxy_MCP_Handler: litellm_trace_id: str | None = None, request_tags: list[str] | None = None, guardrail_context: Mapping[str, object] | None = None, + on_tool_executed: Callable[[], None] | None = None, ) -> list[MCPToolResult]: - """Execute tool calls and return results.""" + """Execute tool calls and return results. + + ``on_tool_executed`` runs each time an MCP server returns for a call, so the caller + can tell executed calls apart from ones that were rejected before reaching a server. + """ from fastapi import HTTPException from litellm._uuid import uuid @@ -861,6 +866,8 @@ class LiteLLM_Proxy_MCP_Handler: litellm_logging_obj=litellm_logging_obj, guardrail_context=guardrail_context, ) + if on_tool_executed is not None: + on_tool_executed() if proxy_logging_obj: result = await proxy_logging_obj.post_mcp_call_hook( diff --git a/litellm/router_utils/mcp_tool_execution.py b/litellm/router_utils/mcp_tool_execution.py index 81c0d6383ed..9e8cccd65f4 100644 --- a/litellm/router_utils/mcp_tool_execution.py +++ b/litellm/router_utils/mcp_tool_execution.py @@ -1,3 +1,4 @@ +from types import TracebackType from typing import Final _MCP_TOOLS_EXECUTED_ATTR: Final = "_litellm_mcp_tools_executed" @@ -14,5 +15,37 @@ def mark_mcp_tools_executed(exception: BaseException) -> None: def mcp_tools_executed(exception: BaseException | None) -> bool: - """Whether ``exception`` was tagged by ``mark_mcp_tools_executed``.""" - return getattr(exception, _MCP_TOOLS_EXECUTED_ATTR, False) is True + """Whether ``exception`` or an exception in its chain was tagged by ``mark_mcp_tools_executed``. + + The chain matters because a caller can re-raise a tagged error as a new exception, as + ``exception_type`` does for anything that is not already a LiteLLM exception. + """ + return exception is not None and ( + getattr(exception, _MCP_TOOLS_EXECUTED_ATTR, False) is True + or mcp_tools_executed(exception.__cause__) + or mcp_tools_executed(exception.__context__) + ) + + +class MCPToolReplayGuard: + """Tags errors raised inside ``with guard:`` once the request has executed an MCP tool call.""" + + __slots__ = ("_tool_executed",) + + def __init__(self) -> None: + self._tool_executed: bool = False + + def record_tool_execution(self) -> None: + self._tool_executed = True + + def __enter__(self) -> None: + return None + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + traceback: TracebackType | None, + ) -> None: + if self._tool_executed and isinstance(exc, Exception): + mark_mcp_tools_executed(exc) diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_mcp_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_mcp_handler.py index 9bc7eb3c03f..93adde12c4b 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_mcp_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_mcp_handler.py @@ -3,7 +3,6 @@ from unittest.mock import AsyncMock, patch import pytest -from litellm.exceptions import InternalServerError from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) @@ -14,7 +13,6 @@ from litellm.llms.anthropic.experimental_pass_through.messages.mcp_handler impor _build_tool_result_message, _extract_tool_use_blocks, ) -from litellm.router_utils.mcp_tool_execution import mcp_tools_executed MCP_REFERENCE = { "type": "mcp", @@ -276,49 +274,3 @@ async def test_anthropic_messages_with_mcp_stops_when_every_tool_call_is_skipped "With no tool results there is nothing to send back, so the loop must not call the model again" ) assert result == tool_use_response - - -@pytest.mark.asyncio -@pytest.mark.parametrize("failing_call", ["initial", "follow_up"]) -async def test_anthropic_messages_with_mcp_tags_a_failure_after_tools_executed(failing_call): - """ - Regression test for #43153: a router retry or fallback re-runs the whole MCP loop, so - a follow-up failure raised after the loop executed tools must carry the tag that stops - the replay. A failure before any tool ran stays untagged and is still retried. - """ - from litellm.llms.anthropic.experimental_pass_through.messages import mcp_handler - from litellm.responses.mcp.request_context import MCPRequestContext - - error = InternalServerError(message="upstream failed", llm_provider="anthropic", model="claude-sonnet-4-5") - tool_use_response = { - "stop_reason": "tool_use", - "content": [{"type": "tool_use", "id": "toolu_1", "name": "t", "input": {}}], - } - execute = AsyncMock(return_value=[{"tool_call_id": "toolu_1", "result": "ok", "name": "t"}]) - model_calls = AsyncMock(side_effect=[error] if failing_call == "initial" else [tool_use_response, error]) - - with ( - patch.object(MCPRequestContext, "resolve", return_value=MCPRequestContext(user_api_key_auth="auth")), - patch.object( - import_module("litellm.responses.mcp.litellm_proxy_mcp_handler").LiteLLM_Proxy_MCP_Handler, - "_process_mcp_tools_without_openai_transform", - new=AsyncMock(return_value=([], {})), - ), - patch.object( - import_module("litellm.responses.mcp.litellm_proxy_mcp_handler").LiteLLM_Proxy_MCP_Handler, - "_execute_tool_calls", - new=execute, - ), - patch("litellm.anthropic_messages", new=model_calls), - pytest.raises(InternalServerError) as exc_info, - ): - await mcp_handler.anthropic_messages_with_mcp( - max_tokens=100, - messages=[{"role": "user", "content": "hi"}], - model="claude-sonnet-4-5", - tools=[MCP_REFERENCE], - ) - - assert exc_info.value is error - assert mcp_tools_executed(error) is (failing_call == "follow_up") - assert execute.await_count == (1 if failing_call == "follow_up" else 0) diff --git a/tests/test_litellm/responses/mcp/test_chat_completions_handler.py b/tests/test_litellm/responses/mcp/test_chat_completions_handler.py index 9affcb26158..54e98cc2b6c 100644 --- a/tests/test_litellm/responses/mcp/test_chat_completions_handler.py +++ b/tests/test_litellm/responses/mcp/test_chat_completions_handler.py @@ -1479,177 +1479,3 @@ async def test_request_selected_mcp_guardrail_blocks_before_upstream(monkeypatch assert upstream.await_count == (0 if selected else 1) tool_message = model_call.await_args.kwargs["messages"][-1] assert ("request-selected MCP block" in tool_message["content"]) is selected - - -_REPLAY_UPSTREAM = "https://upstream.example/v1" - - -def _counting_ledger_mcp_tool(monkeypatch) -> AsyncMock: - """Serve one auto-approved MCP tool through the real manager and registry, returning the tool itself.""" - from mcp.types import Tool - - from litellm.caching.caching import DualCache - from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server import mcp_server_manager, server, tool_registry - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing - from litellm.proxy.utils import ProxyLogging - from litellm.types.mcp_server.mcp_server_manager import MCPServer - - manager = mcp_server_manager.MCPServerManager() - manager.registry = { - "ledger": MCPServer( - server_id="ledger", - name="ledger", - server_name="ledger", - transport="http", - url="https://ledger.example/mcp", - spec_path="ledger.json", - auth_type="none", - ) - } - manager.tool_name_to_mcp_server_name_mapping = {"ledger-commit_write": "ledger"} - tool = AsyncMock(return_value={"written": True}) - registry = tool_registry.MCPToolRegistry() - registry.register_tool("ledger-commit_write", "Write one ledger entry", {"type": "object"}, tool) - monkeypatch.setattr(tool_registry, "global_mcp_tool_registry", registry) - monkeypatch.setattr(mcp_server_manager, "global_mcp_server_manager", manager) - monkeypatch.setattr(proxy_server, "proxy_logging_obj", ProxyLogging(user_api_key_cache=DualCache())) - monkeypatch.setattr( - server, - "_get_tools_from_mcp_servers", - AsyncMock( - return_value=AggregateToolListing( - tools=[Tool(name="ledger-commit_write", inputSchema={"type": "object"})], outcomes={} - ) - ), - ) - monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - return tool - - -def _is_follow_up(request) -> bool: - return any(message.get("role") == "tool" for message in json.loads(request.content)["messages"]) - - -def _upstream_call_kind(call) -> str: - return "follow_up" if _is_follow_up(call.request) else "initial" - - -def _chat_completion_response(message: dict, finish_reason: str) -> Response: - return Response( - 200, - json={ - "id": "chatcmpl-replay", - "object": "chat.completion", - "created": 0, - "model": "gpt-4.1-mini", - "choices": [{"index": 0, "message": message, "finish_reason": finish_reason}], - "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, - }, - ) - - -def _ledger_tool_call_response() -> Response: - return _chat_completion_response( - { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": {"name": "ledger-commit_write", "arguments": '{"entry": "alpha"}'}, - } - ], - }, - "tool_calls", - ) - - -def _server_error_response() -> Response: - return Response(500, json={"error": {"message": "upstream failed", "type": "server_error"}}) - - -async def _router_mcp_completion(router: litellm.Router) -> ModelResponse: - from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth - - return await router.acompletion( - model="repro-model", - messages=[{"role": "user", "content": "Record entry alpha"}], - tools=[{"type": "mcp", "server_url": "litellm_proxy/ledger", "require_approval": "never"}], - metadata={ - "user_api_key_auth": UserAPIKeyAuth( - object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="test", mcp_servers=["ledger"]) - ) - }, - ) - - -def _replay_router(num_retries: int, fallbacks: list | None = None) -> litellm.Router: - deployment = {"model": "openai/gpt-4.1-mini", "api_key": "sk-test", "api_base": _REPLAY_UPSTREAM} - return litellm.Router( - # two replicas, so the router retries right away instead of backing off - model_list=[ - {"model_name": "repro-model", "litellm_params": deployment}, - {"model_name": "repro-model", "litellm_params": deployment}, - {"model_name": "backup-model", "litellm_params": deployment}, - ], - num_retries=num_retries, - fallbacks=fallbacks, - ) - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "router_kwargs", - [{"num_retries": 2}, {"num_retries": 0, "fallbacks": [{"repro-model": ["backup-model"]}]}], - ids=["retries", "fallbacks"], -) -@respx.mock -async def test_router_does_not_replay_an_executed_mcp_tool_after_the_follow_up_fails(monkeypatch, router_kwargs): - """ - Regression test for #43153: an auto-approved MCP tool must run once per request. - - Given: The model asks for the tool, the tool runs, and the follow-up model call returns a 500 - When: The router has retries or a fallback group configured - Then: The client gets that 500 after one tool execution and one follow-up call - - A retry or fallback re-runs the whole MCP loop, so before the fix every replay executed - the tool again (three writes with num_retries=2, two with one fallback) and the client - still received the same 500. - """ - tool = _counting_ledger_mcp_tool(monkeypatch) - upstream = respx.post(f"{_REPLAY_UPSTREAM}/chat/completions").mock( - side_effect=lambda request: _server_error_response() if _is_follow_up(request) else _ledger_tool_call_response() - ) - - with pytest.raises(litellm.InternalServerError) as exc_info: - await _router_mcp_completion(_replay_router(**router_kwargs)) - - assert exc_info.value.status_code == 500 - assert tool.await_count == 1 - assert [_upstream_call_kind(call) for call in upstream.calls] == ["initial", "follow_up"] - - -@pytest.mark.asyncio -@respx.mock -async def test_router_still_retries_a_failure_before_any_mcp_tool_ran(monkeypatch): - """ - Given: The initial model call fails once with a 500, then asks for the tool - When: The router retries - Then: The retry succeeds, the tool runs once, and the follow-up answer reaches the client - """ - tool = _counting_ledger_mcp_tool(monkeypatch) - upstream = respx.post(f"{_REPLAY_UPSTREAM}/chat/completions").mock( - side_effect=[ - _server_error_response(), - _ledger_tool_call_response(), - _chat_completion_response({"role": "assistant", "content": "Recorded alpha"}, "stop"), - ] - ) - - response = await _router_mcp_completion(_replay_router(num_retries=2)) - - assert response.choices[0].message.content == "Recorded alpha" - assert tool.await_count == 1 - assert [_upstream_call_kind(call) for call in upstream.calls] == ["initial", "initial", "follow_up"] diff --git a/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py b/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py index ac5cd630bba..57cebf489a2 100644 --- a/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py +++ b/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py @@ -11,14 +11,12 @@ from fastapi import HTTPException from mcp.types import CallToolResult, TextContent from openai.types.responses.tool_param import Mcp -from litellm.exceptions import InternalServerError from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing from litellm.responses import main as responses_main from litellm.responses.mcp import litellm_proxy_mcp_handler as mcp_handler_module from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, ) -from litellm.router_utils.mcp_tool_execution import mcp_tools_executed from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.responses.main import OutputFunctionToolCall from litellm.types.utils import ModelResponse @@ -1297,39 +1295,3 @@ async def test_responses_discovery_logs_sanitized_caller_headers(monkeypatch: py logged: Final = setup.call_args.kwargs["metadata"]["headers"] assert logged == {"x-app-id": "app-a", "x-nuid": "user-a", "x-user-id": "identity-a"} assert headers["x-mcp-deepwiki-authorization"] == "upstream-sentinel" - - -@pytest.mark.parametrize("failing_call", ["initial", "follow_up"]) -@pytest.mark.asyncio -async def test_aresponses_api_with_mcp_tags_a_failure_after_tools_executed( - monkeypatch: pytest.MonkeyPatch, failing_call: str -): - """ - Regression test for #43153: a router retry or fallback re-runs the whole MCP loop, so - a follow-up failure raised after the loop executed tools must carry the tag that stops - the replay. A failure before any tool ran stays untagged and is still retried. - """ - error: Final = InternalServerError(message="upstream failed", llm_provider="openai", model="gpt-5") - model_calls: Final = AsyncMock( - side_effect=[error] if failing_call == "initial" else [_response_with_reasoning_and_tool_call(), error] - ) - execute: Final = AsyncMock(return_value=[{"tool_call_id": "call-1", "name": "foo", "result": "done"}]) - monkeypatch.setattr(responses_main, "aresponses", model_calls) - monkeypatch.setattr(mcp_handler_module, "aresponses", model_calls) - monkeypatch.setattr( - LiteLLM_Proxy_MCP_Handler, - "_process_mcp_tools_without_openai_transform", - AsyncMock(return_value=([], {"foo": "litellm_proxy"})), - ) - monkeypatch.setattr(LiteLLM_Proxy_MCP_Handler, "_execute_tool_calls", execute) - - with pytest.raises(InternalServerError) as exc_info: - await responses_main.aresponses_api_with_mcp( - input="hi", - model="gpt-5", - tools=[{"type": "mcp", "server_url": "litellm_proxy", "require_approval": "never"}], - ) - - assert exc_info.value is error - assert mcp_tools_executed(error) is (failing_call == "follow_up") - assert execute.await_count == (1 if failing_call == "follow_up" else 0) diff --git a/tests/test_litellm/router_utils/test_mcp_tool_execution.py b/tests/test_litellm/router_utils/test_mcp_tool_execution.py new file mode 100644 index 00000000000..5a6f07a84e3 --- /dev/null +++ b/tests/test_litellm/router_utils/test_mcp_tool_execution.py @@ -0,0 +1,312 @@ +import json +from collections.abc import Awaitable, Callable +from typing import Final, NamedTuple +from unittest.mock import AsyncMock + +import httpx +import pytest +import respx +from mcp.types import Tool + +import litellm +from litellm.caching.caching import DualCache +from litellm.exceptions import GuardrailRaisedException +from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.proxy import proxy_server +from litellm.proxy._experimental.mcp_server import mcp_server_manager, server, tool_registry +from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing +from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth +from litellm.proxy.utils import ProxyLogging +from litellm.types.mcp_server.mcp_server_manager import MCPServer + +_UPSTREAM: Final = "https://upstream.example/v1" +_TOOL_NAME: Final = "ledger-commit_write" + + +def _ledger_tools() -> list[dict[str, str]]: + return [{"type": "mcp", "server_url": "litellm_proxy/ledger", "require_approval": "never"}] + + +def _ledger_caller() -> UserAPIKeyAuth: + return UserAPIKeyAuth( + object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="test", mcp_servers=["ledger"]) + ) + + +class _LedgerGateway(NamedTuple): + tool: AsyncMock + list_tools: AsyncMock + + +@pytest.fixture +def ledger_gateway(monkeypatch: pytest.MonkeyPatch) -> _LedgerGateway: + """Serve one auto-approved MCP tool through the real MCP manager and tool registry.""" + manager: Final = mcp_server_manager.MCPServerManager() + manager.registry = { + "ledger": MCPServer( + server_id="ledger", + name="ledger", + server_name="ledger", + transport="http", + url="https://ledger.example/mcp", + # a spec_path server runs its tools from the local tool registry instead of over the network + spec_path="ledger.json", + auth_type="none", + ) + } + manager.tool_name_to_mcp_server_name_mapping = {_TOOL_NAME: "ledger"} + tool: Final = AsyncMock(return_value={"written": True}) + registry: Final = tool_registry.MCPToolRegistry() + registry.register_tool(_TOOL_NAME, "Write one ledger entry", {"type": "object"}, tool) + list_tools: Final = AsyncMock( + return_value=AggregateToolListing(tools=[Tool(name=_TOOL_NAME, inputSchema={"type": "object"})], outcomes={}) + ) + monkeypatch.setattr(tool_registry, "global_mcp_tool_registry", registry) + monkeypatch.setattr(mcp_server_manager, "global_mcp_server_manager", manager) + monkeypatch.setattr(proxy_server, "proxy_logging_obj", ProxyLogging(user_api_key_cache=DualCache())) + monkeypatch.setattr(server, "_get_tools_from_mcp_servers", list_tools) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + return _LedgerGateway(tool=tool, list_tools=list_tools) + + +def _is_follow_up(request: httpx.Request) -> bool: + body: Final = json.loads(request.content) + messages: Final = body.get("messages") or [] + items: Final = body["input"] if isinstance(body.get("input"), list) else [] + return any(message.get("role") == "tool" for message in messages) or any( + item.get("type") == "function_call_output" for item in items + ) + + +def _chat_completion(message: dict[str, object], finish_reason: str) -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 0, + "model": "gpt-4.1-mini", + "choices": [{"index": 0, "message": message, "finish_reason": finish_reason}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + + +def _responses_api_response(output_item: dict[str, object]) -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": "resp_1", + "object": "response", + "created_at": 0, + "status": "completed", + "model": "gpt-4.1-mini", + "output": [output_item], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + }, + ) + + +def _tool_call(responses_api: bool) -> httpx.Response: + arguments: Final = json.dumps({"entry": "alpha"}) + if responses_api: + return _responses_api_response( + {"type": "function_call", "id": "fc_1", "call_id": "call_1", "name": _TOOL_NAME, "arguments": arguments} + ) + return _chat_completion( + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": _TOOL_NAME, "arguments": arguments}} + ], + }, + "tool_calls", + ) + + +def _final_answer(responses_api: bool) -> httpx.Response: + if responses_api: + return _responses_api_response( + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "Recorded alpha", "annotations": []}], + } + ) + return _chat_completion({"role": "assistant", "content": "Recorded alpha"}, "stop") + + +class _FakeModel: + """Answers both OpenAI routes: a tool call, then a final answer once the tool result arrives.""" + + def __init__(self, failing_initial_calls: int = 0, failing_follow_ups: int = 0) -> None: + self.calls: list[str] = [] + self._failures: Final = {"initial": failing_initial_calls, "follow_up": failing_follow_ups} + + def __call__(self, request: httpx.Request) -> httpx.Response: + kind: Final = "follow_up" if _is_follow_up(request) else "initial" + self.calls.append(kind) + if self.calls.count(kind) <= self._failures[kind]: + return httpx.Response(500, json={"error": {"message": "upstream failed", "type": "server_error"}}) + responses_api: Final = request.url.path.endswith("/responses") + return _tool_call(responses_api) if kind == "initial" else _final_answer(responses_api) + + +def _serve(model: _FakeModel) -> None: + respx.post(f"{_UPSTREAM}/chat/completions").mock(side_effect=model) + respx.post(f"{_UPSTREAM}/responses").mock(side_effect=model) + + +def _router(num_retries: int, fallbacks: list[dict[str, list[str]]] | None = None) -> litellm.Router: + deployment: Final = {"model": "openai/gpt-4.1-mini", "api_key": "sk-test", "api_base": _UPSTREAM} + return litellm.Router( + # two replicas, so the router retries right away instead of backing off + model_list=[ + {"model_name": "repro-model", "litellm_params": deployment}, + {"model_name": "repro-model", "litellm_params": deployment}, + {"model_name": "backup-model", "litellm_params": deployment}, + ], + num_retries=num_retries, + fallbacks=fallbacks, + ) + + +def _chat(router: litellm.Router) -> Awaitable[object]: + return router.acompletion( + model="repro-model", + messages=[{"role": "user", "content": "Record entry alpha"}], + tools=_ledger_tools(), + metadata={"user_api_key_auth": _ledger_caller()}, + ) + + +def _responses(router: litellm.Router) -> Awaitable[object]: + return router.aresponses( + model="repro-model", + input="Record entry alpha", + tools=_ledger_tools(), + litellm_metadata={"user_api_key_auth": _ledger_caller()}, + ) + + +def _messages(router: litellm.Router) -> Awaitable[object]: + return router.aanthropic_messages( + model="repro-model", + max_tokens=64, + messages=[{"role": "user", "content": "Record entry alpha"}], + tools=_ledger_tools(), + litellm_metadata={"user_api_key_auth": _ledger_caller()}, + ) + + +_ENDPOINTS: Final = pytest.mark.parametrize( + "send", [_chat, _responses, _messages], ids=["chat_completions", "responses", "messages"] +) + + +@pytest.mark.asyncio +@_ENDPOINTS +@pytest.mark.parametrize( + "router_kwargs", + [{"num_retries": 2}, {"num_retries": 0, "fallbacks": [{"repro-model": ["backup-model"]}]}], + ids=["retries", "fallbacks"], +) +@respx.mock +async def test_router_does_not_replay_an_executed_mcp_tool( + ledger_gateway: _LedgerGateway, + send: Callable[[litellm.Router], Awaitable[object]], + router_kwargs: dict[str, object], +): + """ + Regression test for #43153: an auto-approved MCP tool must run once per request. + + Given: The model asks for the tool, the tool runs, and every follow-up model call returns a 500 + When: The router has retries or a fallback group configured + Then: The client gets the 500 after one tool execution and one follow-up call + + A retry or fallback re-runs the whole MCP loop, so before the fix every replay executed + the tool again (three writes with num_retries=2, two with one fallback). + """ + model: Final = _FakeModel(failing_follow_ups=3) + _serve(model) + + with pytest.raises(litellm.InternalServerError): + await send(_router(**router_kwargs)) + + assert ledger_gateway.tool.await_count == 1 + assert model.calls == ["initial", "follow_up"] + + +@pytest.mark.asyncio +@_ENDPOINTS +@respx.mock +async def test_router_retries_when_a_guardrail_blocked_every_mcp_tool( + ledger_gateway: _LedgerGateway, + send: Callable[[litellm.Router], Awaitable[object]], + monkeypatch: pytest.MonkeyPatch, +): + """ + Given: A pre-call guardrail blocks the tool call, so no tool runs + When: The follow-up model call fails once with a 500 + Then: The router still retries, because replaying the request cannot repeat a side effect + """ + + class BlockEveryToolCall(CustomGuardrail): + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + raise GuardrailRaisedException(message="tool calls are blocked", blocked_content=True) + + monkeypatch.setattr( + litellm, "callbacks", [BlockEveryToolCall(guardrail_name="block", event_hook="pre_mcp_call", default_on=True)] + ) + model: Final = _FakeModel(failing_follow_ups=1) + _serve(model) + + await send(_router(num_retries=2)) + + assert ledger_gateway.tool.await_count == 0 + assert model.calls == ["initial", "follow_up", "initial", "follow_up"] + + +@pytest.mark.asyncio +@respx.mock +async def test_router_still_retries_a_failure_before_any_mcp_tool_ran(ledger_gateway: _LedgerGateway): + """ + Given: The initial model call fails once with a 500, then asks for the tool + When: The router retries + Then: The retry succeeds and the tool runs once + """ + model: Final = _FakeModel(failing_initial_calls=1) + _serve(model) + + await _chat(_router(num_retries=2)) + + assert ledger_gateway.tool.await_count == 1 + assert model.calls == ["initial", "initial", "follow_up"] + + +@pytest.mark.asyncio +@respx.mock +async def test_responses_tool_listing_failure_after_a_tool_ran_is_not_replayed(ledger_gateway: _LedgerGateway): + """ + Given: A Responses request whose tool ran and whose follow-up succeeded + When: Listing the MCP tools again for the output items fails + Then: The router surfaces that failure instead of replaying the request and the tool + """ + listing: Final = ledger_gateway.list_tools.return_value + listing_error: Final = RuntimeError("tool listing failed") + ledger_gateway.list_tools.side_effect = [listing, listing_error, listing, listing_error, listing, listing_error] + model: Final = _FakeModel() + _serve(model) + + with pytest.raises(Exception, match="tool listing failed"): + await _responses(_router(num_retries=2)) + + assert ledger_gateway.tool.await_count == 1 + assert ledger_gateway.list_tools.await_count == 2 + assert model.calls == ["initial", "follow_up"] From de4ba11e18fe131408cf216f085f2e3507cc399f Mon Sep 17 00:00:00 2001 From: Atharva-Kanherkar <142440039+Atharva-Kanherkar@users.noreply.github.com> Date: Fri, 25 Sep 2026 17:06:19 +0530 Subject: [PATCH 06/15] test(router): inject the attempt instead of patching router methods The tagged-error tests pass their failing attempt as original_function and let the real retry loop, healthy deployment lookup and backoff run, instead of patching make_call and other router methods Signed-off-by: Atharva-Kanherkar <142440039+Atharva-Kanherkar@users.noreply.github.com> --- .../test_router_retry_non_retryable_errors.py | 51 ++++++++----------- 1 file changed, 22 insertions(+), 29 deletions(-) diff --git a/tests/test_litellm/test_router_retry_non_retryable_errors.py b/tests/test_litellm/test_router_retry_non_retryable_errors.py index 9e29ae2d51b..9d702a55086 100644 --- a/tests/test_litellm/test_router_retry_non_retryable_errors.py +++ b/tests/test_litellm/test_router_retry_non_retryable_errors.py @@ -381,18 +381,13 @@ async def test_500_after_mcp_tool_execution_is_not_retried(retry_policy, tools_e """ router: Final = _create_router(num_retries=2, retry_policy=retry_policy) error: Final = _make_follow_up_error(tools_executed) - make_call: Final = AsyncMock(side_effect=error) + attempt: Final = AsyncMock(side_effect=error) - with ( - patch.object(router, "make_call", make_call), - patch.object(router, "_async_get_healthy_deployments", return_value=(["d1", "d2"], ["d1", "d2"])), - patch.object(router, "_time_to_sleep_before_retry", return_value=0), - pytest.raises(litellm.InternalServerError) as exc_info, - ): - await router.async_function_with_retries(num_retries=2, **_base_kwargs()) + with pytest.raises(litellm.InternalServerError) as exc_info: + await router.async_function_with_retries(**{**_base_kwargs(), "original_function": attempt, "num_retries": 2}) assert exc_info.value is error - assert make_call.await_count == (1 if tools_executed else 3) + assert attempt.await_count == (1 if tools_executed else 3) @pytest.mark.asyncio @@ -404,18 +399,13 @@ async def test_500_after_mcp_tool_execution_stops_an_ongoing_retry_loop(): """ router: Final = _create_router(num_retries=3) after_tools: Final = _make_follow_up_error(tools_executed=True) - make_call: Final = AsyncMock(side_effect=[_make_follow_up_error(tools_executed=False), after_tools]) + attempt: Final = AsyncMock(side_effect=[_make_follow_up_error(tools_executed=False), after_tools]) - with ( - patch.object(router, "make_call", make_call), - patch.object(router, "_async_get_healthy_deployments", return_value=(["d1", "d2"], ["d1", "d2"])), - patch.object(router, "_time_to_sleep_before_retry", return_value=0), - pytest.raises(litellm.InternalServerError) as exc_info, - ): - await router.async_function_with_retries(num_retries=3, **_base_kwargs()) + with pytest.raises(litellm.InternalServerError) as exc_info: + await router.async_function_with_retries(**{**_base_kwargs(), "original_function": attempt, "num_retries": 3}) assert exc_info.value is after_tools - assert make_call.await_count == 2 + assert attempt.await_count == 2 @pytest.mark.asyncio @@ -434,21 +424,24 @@ async def test_500_after_mcp_tool_execution_does_not_fall_back(tools_executed): fallbacks=[{"test-model": ["backup-model"]}], ) error: Final = _make_follow_up_error(tools_executed) + fallback_response: Final = litellm.ModelResponse() - async def primary_fails(original_function, *args, **kwargs): + async def primary_fails(**kwargs): if kwargs["model"] == "test-model": raise error - return "fallback response" + return fallback_response - make_call: Final = AsyncMock(side_effect=primary_fails) + attempt: Final = AsyncMock(side_effect=primary_fails) + call: Final = router.async_function_with_fallbacks( + **{**_base_kwargs(), "original_function": attempt, "num_retries": 0} + ) - with patch.object(router, "make_call", make_call): - if tools_executed: - with pytest.raises(litellm.InternalServerError) as exc_info: - await router.async_function_with_fallbacks(num_retries=0, **_base_kwargs()) - assert exc_info.value is error - else: - assert await router.async_function_with_fallbacks(num_retries=0, **_base_kwargs()) == "fallback response" + if tools_executed: + with pytest.raises(litellm.InternalServerError) as exc_info: + await call + assert exc_info.value is error + else: + assert await call is fallback_response - attempted_models: Final = [call.kwargs["model"] for call in make_call.await_args_list] + attempted_models: Final = [awaited.kwargs["model"] for awaited in attempt.await_args_list] assert attempted_models == (["test-model"] if tools_executed else ["test-model", "backup-model"]) From c483f99ae88c78ada253cb06278c6b5ad82baa34 Mon Sep 17 00:00:00 2001 From: Atharva-Kanherkar <142440039+Atharva-Kanherkar@users.noreply.github.com> Date: Fri, 25 Sep 2026 17:08:54 +0530 Subject: [PATCH 07/15] refactor(mcp): follow one link per step when looking for the replay tag Follow the cause, or else the context, the same way Python walks an exception chain, and state exactly when the executor calls on_tool_executed Signed-off-by: Atharva-Kanherkar <142440039+Atharva-Kanherkar@users.noreply.github.com> --- litellm/responses/mcp/litellm_proxy_mcp_handler.py | 4 ++-- litellm/router_utils/mcp_tool_execution.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index 10ed52d6bca..33b0aba0063 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -700,8 +700,8 @@ class LiteLLM_Proxy_MCP_Handler: ) -> list[MCPToolResult]: """Execute tool calls and return results. - ``on_tool_executed`` runs each time an MCP server returns for a call, so the caller - can tell executed calls apart from ones that were rejected before reaching a server. + ``on_tool_executed`` runs once for every call an MCP server returned a result for. Calls + that failed or were rejected, for example by a pre-call guardrail, do not trigger it. """ from fastapi import HTTPException diff --git a/litellm/router_utils/mcp_tool_execution.py b/litellm/router_utils/mcp_tool_execution.py index 9e8cccd65f4..1dacba80c1b 100644 --- a/litellm/router_utils/mcp_tool_execution.py +++ b/litellm/router_utils/mcp_tool_execution.py @@ -22,8 +22,7 @@ def mcp_tools_executed(exception: BaseException | None) -> bool: """ return exception is not None and ( getattr(exception, _MCP_TOOLS_EXECUTED_ATTR, False) is True - or mcp_tools_executed(exception.__cause__) - or mcp_tools_executed(exception.__context__) + or mcp_tools_executed(exception.__cause__ or exception.__context__) ) From e9e95d5fb3699916898b1cb33b17137e5ee2187e Mon Sep 17 00:00:00 2001 From: Atharva-Kanherkar <142440039+Atharva-Kanherkar@users.noreply.github.com> Date: Fri, 25 Sep 2026 17:37:08 +0530 Subject: [PATCH 08/15] fix(mcp): walk the exception chain without recursion The code quality job forbids recursive functions, so follow the chain with a bounded loop instead, the same way the database exception handler does. The depth cap also keeps a malformed cyclic chain from looping forever Signed-off-by: Atharva-Kanherkar <142440039+Atharva-Kanherkar@users.noreply.github.com> --- litellm/router_utils/mcp_tool_execution.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/litellm/router_utils/mcp_tool_execution.py b/litellm/router_utils/mcp_tool_execution.py index 1dacba80c1b..90ea17e49ef 100644 --- a/litellm/router_utils/mcp_tool_execution.py +++ b/litellm/router_utils/mcp_tool_execution.py @@ -1,8 +1,12 @@ +from collections.abc import Iterator from types import TracebackType from typing import Final _MCP_TOOLS_EXECUTED_ATTR: Final = "_litellm_mcp_tools_executed" +# Real exception chains are a few links deep; the cap also makes the walk cycle-safe. +_MAX_EXCEPTION_CHAIN_DEPTH: Final = 20 + def mark_mcp_tools_executed(exception: BaseException) -> None: """Tag an error raised after an MCP gateway loop already executed tool calls. @@ -14,15 +18,24 @@ def mark_mcp_tools_executed(exception: BaseException) -> None: setattr(exception, _MCP_TOOLS_EXECUTED_ATTR, True) +def _exception_chain(exception: BaseException) -> Iterator[BaseException]: + current = exception # rebind-ok: advances one link per iteration of the bounded walk + for _ in range(_MAX_EXCEPTION_CHAIN_DEPTH): + yield current + following = current.__cause__ or current.__context__ + if following is None: + return + current = following + + def mcp_tools_executed(exception: BaseException | None) -> bool: """Whether ``exception`` or an exception in its chain was tagged by ``mark_mcp_tools_executed``. The chain matters because a caller can re-raise a tagged error as a new exception, as ``exception_type`` does for anything that is not already a LiteLLM exception. """ - return exception is not None and ( - getattr(exception, _MCP_TOOLS_EXECUTED_ATTR, False) is True - or mcp_tools_executed(exception.__cause__ or exception.__context__) + return exception is not None and any( + getattr(link, _MCP_TOOLS_EXECUTED_ATTR, False) is True for link in _exception_chain(exception) ) From 2a8f588345b82f640036d4e221041023f8e80756 Mon Sep 17 00:00:00 2001 From: Atharva-Kanherkar <142440039+Atharva-Kanherkar@users.noreply.github.com> Date: Fri, 25 Sep 2026 17:37:13 +0530 Subject: [PATCH 09/15] fix(mcp): count a tool call once it is sent to the MCP server A tool can write and still fail to return, for example on a timeout, so waiting for the server's result left that write unrecorded and its failed follow-up replayable. MCPServerManager.call_tool now reports the moment a call passes its pre-call checks and is sent, and the replay guard records that instead. A call a pre-call guardrail rejects is still never sent, so its request keeps its retries Signed-off-by: Atharva-Kanherkar <142440039+Atharva-Kanherkar@users.noreply.github.com> --- .../messages/mcp_handler.py | 2 +- .../mcp_server/mcp_server_manager.py | 7 ++++ litellm/responses/main.py | 2 +- .../responses/mcp/chat_completions_handler.py | 2 +- .../mcp/litellm_proxy_mcp_handler.py | 10 ++--- litellm/router_utils/mcp_tool_execution.py | 12 +++--- .../router_utils/test_mcp_tool_execution.py | 40 +++++++++++++++++++ 7 files changed, 61 insertions(+), 14 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/mcp_handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/mcp_handler.py index b84c41a28f0..22ae7660019 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/mcp_handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/mcp_handler.py @@ -159,7 +159,7 @@ async def anthropic_messages_with_mcp( litellm_trace_id=context.litellm_trace_id, request_tags=list(context.request_tags) if context.request_tags else None, guardrail_context=context.guardrail_context, - on_tool_executed=replay_guard.record_tool_execution, + on_tool_dispatched=replay_guard.record_tool_dispatch, ) # Every tool call was skipped, so there is nothing to feed back; a diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index be4df55ff58..53f24cd28ad 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -6318,6 +6318,7 @@ class MCPServerManager: litellm_logging_obj: "LiteLLMLoggingObj | None" = None, guardrail_context: Mapping[str, object] | None = None, client_ip: str | None = None, + on_dispatch: Callable[[], None] | None = None, ) -> CallToolResult: """ Call a tool with the given name and arguments @@ -6333,6 +6334,8 @@ class MCPServerManager: litellm_logging_obj: Optional request logger the guardrail hooks record their evaluations onto, so MCP guardrail activity reaches the Guardrails Monitor. See ``pre_call_tool_check`` + on_dispatch: Optional callback run once the call passed its pre-call checks + and is sent to the server, even if the server never returns a result Returns: @@ -6433,8 +6436,12 @@ class MCPServerManager: _request_extra_headers.reset(extra_token) _request_resolved_auth_headers.reset(resolved_token) + if on_dispatch is not None: + on_dispatch() tasks.append(asyncio.create_task(_call_openapi_via_handler())) else: + if on_dispatch is not None: + on_dispatch() return await self._call_regular_mcp_tool( mcp_server=mcp_server, original_tool_name=name, diff --git a/litellm/responses/main.py b/litellm/responses/main.py index c621713c0fc..d3d41758f05 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -353,7 +353,7 @@ async def aresponses_api_with_mcp( guardrail_context=MCPRequestContext.resolve_guardrail_context( MappingProxyType({**kwargs, "metadata": metadata, "model": model}) ), - on_tool_executed=replay_guard.record_tool_execution, + on_tool_dispatched=replay_guard.record_tool_dispatch, ) if tool_results: diff --git a/litellm/responses/mcp/chat_completions_handler.py b/litellm/responses/mcp/chat_completions_handler.py index 3391ed0d9a2..2017644b265 100644 --- a/litellm/responses/mcp/chat_completions_handler.py +++ b/litellm/responses/mcp/chat_completions_handler.py @@ -620,7 +620,7 @@ async def acompletion_with_mcp( litellm_trace_id=context.litellm_trace_id, request_tags=request_tags, guardrail_context=context.guardrail_context, - on_tool_executed=replay_guard.record_tool_execution, + on_tool_dispatched=replay_guard.record_tool_dispatch, ) if not tool_results: diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index 33b0aba0063..4f372703234 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -696,12 +696,13 @@ class LiteLLM_Proxy_MCP_Handler: litellm_trace_id: str | None = None, request_tags: list[str] | None = None, guardrail_context: Mapping[str, object] | None = None, - on_tool_executed: Callable[[], None] | None = None, + on_tool_dispatched: Callable[[], None] | None = None, ) -> list[MCPToolResult]: """Execute tool calls and return results. - ``on_tool_executed`` runs once for every call an MCP server returned a result for. Calls - that failed or were rejected, for example by a pre-call guardrail, do not trigger it. + ``on_tool_dispatched`` runs for every call that passes its pre-call checks and is sent to + an MCP server, even one that then times out. Calls rejected before that, for example by a + pre-call guardrail, do not trigger it. """ from fastapi import HTTPException @@ -865,9 +866,8 @@ class LiteLLM_Proxy_MCP_Handler: proxy_logging_obj=proxy_logging_obj, litellm_logging_obj=litellm_logging_obj, guardrail_context=guardrail_context, + on_dispatch=on_tool_dispatched, ) - if on_tool_executed is not None: - on_tool_executed() if proxy_logging_obj: result = await proxy_logging_obj.post_mcp_call_hook( diff --git a/litellm/router_utils/mcp_tool_execution.py b/litellm/router_utils/mcp_tool_execution.py index 90ea17e49ef..3ed4d2284ed 100644 --- a/litellm/router_utils/mcp_tool_execution.py +++ b/litellm/router_utils/mcp_tool_execution.py @@ -40,15 +40,15 @@ def mcp_tools_executed(exception: BaseException | None) -> bool: class MCPToolReplayGuard: - """Tags errors raised inside ``with guard:`` once the request has executed an MCP tool call.""" + """Tags errors raised inside ``with guard:`` once the request has sent an MCP tool call to a server.""" - __slots__ = ("_tool_executed",) + __slots__ = ("_tool_dispatched",) def __init__(self) -> None: - self._tool_executed: bool = False + self._tool_dispatched: bool = False - def record_tool_execution(self) -> None: - self._tool_executed = True + def record_tool_dispatch(self) -> None: + self._tool_dispatched = True def __enter__(self) -> None: return None @@ -59,5 +59,5 @@ class MCPToolReplayGuard: exc: BaseException | None, traceback: TracebackType | None, ) -> None: - if self._tool_executed and isinstance(exc, Exception): + if self._tool_dispatched and isinstance(exc, Exception): mark_mcp_tools_executed(exc) diff --git a/tests/test_litellm/router_utils/test_mcp_tool_execution.py b/tests/test_litellm/router_utils/test_mcp_tool_execution.py index 5a6f07a84e3..79b5e581607 100644 --- a/tests/test_litellm/router_utils/test_mcp_tool_execution.py +++ b/tests/test_litellm/router_utils/test_mcp_tool_execution.py @@ -1,3 +1,4 @@ +import asyncio import json from collections.abc import Awaitable, Callable from typing import Final, NamedTuple @@ -273,6 +274,45 @@ async def test_router_retries_when_a_guardrail_blocked_every_mcp_tool( assert model.calls == ["initial", "follow_up", "initial", "follow_up"] +@pytest.mark.asyncio +@_ENDPOINTS +@respx.mock +async def test_router_does_not_replay_a_tool_call_that_failed_after_it_was_sent( + ledger_gateway: _LedgerGateway, + send: Callable[[litellm.Router], Awaitable[object]], + monkeypatch: pytest.MonkeyPatch, +): + """ + Given: The tool runs, but its call fails afterwards, here because a during-call guardrail + rejects the result, which leaves the router in the same spot as a timeout after the write + When: Every follow-up model call returns a 500 + Then: The router does not replay the request, because the tool may already have written + """ + tool_ran: Final = asyncio.Event() + + def write(**_: object) -> dict[str, bool]: + tool_ran.set() + return {"written": True} + + class RejectToolResult(CustomGuardrail): + async def async_moderation_hook(self, data, user_api_key_dict, call_type): + await tool_ran.wait() + raise GuardrailRaisedException(message="tool result rejected", blocked_content=True) + + ledger_gateway.tool.side_effect = write + monkeypatch.setattr( + litellm, "callbacks", [RejectToolResult(guardrail_name="reject", event_hook="during_mcp_call", default_on=True)] + ) + model: Final = _FakeModel(failing_follow_ups=3) + _serve(model) + + with pytest.raises(litellm.InternalServerError): + await send(_router(num_retries=2)) + + assert ledger_gateway.tool.await_count == 1 + assert model.calls == ["initial", "follow_up"] + + @pytest.mark.asyncio @respx.mock async def test_router_still_retries_a_failure_before_any_mcp_tool_ran(ledger_gateway: _LedgerGateway): From b1e4442b97a88c86582713a85aab5cf8961fda6d Mon Sep 17 00:00:00 2001 From: Atharva-Kanherkar <142440039+Atharva-Kanherkar@users.noreply.github.com> Date: Fri, 25 Sep 2026 17:37:19 +0530 Subject: [PATCH 10/15] fix(mcp): stop fallbacks from replaying a streamed chat tool call In a streaming Chat Completions request the tools run inside the stream, and a follow-up stream that breaks before its first chunk triggers the router's mid-stream fallback, which re-runs the whole MCP loop. Record the streamed tool calls and read the follow-up stream inside the replay guard, so that failure is surfaced instead Signed-off-by: Atharva-Kanherkar <142440039+Atharva-Kanherkar@users.noreply.github.com> --- .../responses/mcp/chat_completions_handler.py | 5 +- .../router_utils/test_mcp_tool_execution.py | 74 ++++++++++++++++++- 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/litellm/responses/mcp/chat_completions_handler.py b/litellm/responses/mcp/chat_completions_handler.py index 2017644b265..928ba9b5876 100644 --- a/litellm/responses/mcp/chat_completions_handler.py +++ b/litellm/responses/mcp/chat_completions_handler.py @@ -245,6 +245,7 @@ async def acompletion_with_mcp( self.follow_up_stream = None self.follow_up_iterator = None self.follow_up_exhausted = False + self.replay_guard = MCPToolReplayGuard() def __aiter__(self): return self @@ -383,7 +384,8 @@ async def acompletion_with_mcp( verbose_logger.debug("Follow-up stream iterator created") try: - chunk = await self.follow_up_iterator.__anext__() + with self.replay_guard: + chunk = await self.follow_up_iterator.__anext__() from litellm._logging import verbose_logger verbose_logger.debug("Follow-up chunk yielded: %s", chunk) @@ -446,6 +448,7 @@ async def acompletion_with_mcp( litellm_trace_id=self.litellm_trace_id, request_tags=self.request_tags, guardrail_context=context.guardrail_context, + on_tool_dispatched=self.replay_guard.record_tool_dispatch, ) async def _prepare_follow_up_call(self): diff --git a/tests/test_litellm/router_utils/test_mcp_tool_execution.py b/tests/test_litellm/router_utils/test_mcp_tool_execution.py index 79b5e581607..94ff6f1ab83 100644 --- a/tests/test_litellm/router_utils/test_mcp_tool_execution.py +++ b/tests/test_litellm/router_utils/test_mcp_tool_execution.py @@ -1,6 +1,6 @@ import asyncio import json -from collections.abc import Awaitable, Callable +from collections.abc import AsyncIterator, Awaitable, Callable from typing import Final, NamedTuple from unittest.mock import AsyncMock @@ -159,6 +159,40 @@ class _FakeModel: return _tool_call(responses_api) if kind == "initial" else _final_answer(responses_api) +class _StreamBrokenBeforeFirstChunk(httpx.AsyncByteStream): + async def __aiter__(self) -> AsyncIterator[bytes]: + raise httpx.ReadError("connection dropped before the first chunk") + yield b"" + + +def _chat_tool_call_stream() -> httpx.Response: + tool_call: Final = { + "index": 0, + "id": "call_1", + "type": "function", + "function": {"name": _TOOL_NAME, "arguments": json.dumps({"entry": "alpha"})}, + } + chunks: Final = ( + {"role": "assistant", "content": None, "tool_calls": [tool_call]}, + {}, + ) + body: Final = "".join( + "data: " + + json.dumps( + { + "id": "chatcmpl-1", + "object": "chat.completion.chunk", + "created": 0, + "model": "gpt-4.1-mini", + "choices": [{"index": 0, "delta": delta, "finish_reason": None if delta else "tool_calls"}], + } + ) + + "\n\n" + for delta in chunks + ) + return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=f"{body}data: [DONE]\n\n") + + def _serve(model: _FakeModel) -> None: respx.post(f"{_UPSTREAM}/chat/completions").mock(side_effect=model) respx.post(f"{_UPSTREAM}/responses").mock(side_effect=model) @@ -313,6 +347,44 @@ async def test_router_does_not_replay_a_tool_call_that_failed_after_it_was_sent( assert model.calls == ["initial", "follow_up"] +@pytest.mark.asyncio +@respx.mock +async def test_router_does_not_fall_back_when_a_streamed_follow_up_breaks_after_a_tool_ran( + ledger_gateway: _LedgerGateway, +): + """ + Given: A streaming Chat Completions request whose tool ran + When: The follow-up stream breaks before its first chunk and a fallback group is configured + Then: The client gets the error instead of a fallback that runs the tool again + """ + calls: Final[list[str]] = [] + + def stream_model(request: httpx.Request) -> httpx.Response: + kind: Final = "follow_up" if _is_follow_up(request) else "initial" + calls.append(kind) + if kind == "follow_up": + return httpx.Response( + 200, headers={"content-type": "text/event-stream"}, stream=_StreamBrokenBeforeFirstChunk() + ) + return _chat_tool_call_stream() + + respx.post(f"{_UPSTREAM}/chat/completions").mock(side_effect=stream_model) + stream: Final = await _router(num_retries=0, fallbacks=[{"repro-model": ["backup-model"]}]).acompletion( + model="repro-model", + messages=[{"role": "user", "content": "Record entry alpha"}], + tools=_ledger_tools(), + metadata={"user_api_key_auth": _ledger_caller()}, + stream=True, + ) + + with pytest.raises(litellm.APIConnectionError): + async for _ in stream: + pass + + assert ledger_gateway.tool.await_count == 1 + assert calls == ["initial", "follow_up"] + + @pytest.mark.asyncio @respx.mock async def test_router_still_retries_a_failure_before_any_mcp_tool_ran(ledger_gateway: _LedgerGateway): From d4caa30db0095834b2bbf08363d7aebb0e4dde27 Mon Sep 17 00:00:00 2001 From: Atharva-Kanherkar <142440039+Atharva-Kanherkar@users.noreply.github.com> Date: Fri, 25 Sep 2026 18:09:57 +0530 Subject: [PATCH 11/15] fix(mcp): count a tool call when the MCP session sends it A call was recorded before the MCP client resolved its headers and opened a session, so a call that failed there, for example on a per-user credential the caller had not set, still blocked retries although no tool could have run. MCPClient.call_tool now reports the dispatch right before the session sends the call, and the OpenAPI path right before its tool handler runs Signed-off-by: Atharva-Kanherkar <142440039+Atharva-Kanherkar@users.noreply.github.com> --- litellm/experimental_mcp_client/client.py | 5 + .../mcp_server/mcp_server_manager.py | 32 +++++-- .../mcp/litellm_proxy_mcp_handler.py | 6 +- .../test_mcp_client.py | 30 ++++++ .../test_mcp_max_concurrent_requests.py | 2 +- .../mcp_server/test_mcp_server_manager.py | 91 ++++++++++++++++++- .../router_utils/test_mcp_tool_execution.py | 32 +++++++ 7 files changed, 181 insertions(+), 17 deletions(-) diff --git a/litellm/experimental_mcp_client/client.py b/litellm/experimental_mcp_client/client.py index 1206f9abcbd..bf2ea196313 100644 --- a/litellm/experimental_mcp_client/client.py +++ b/litellm/experimental_mcp_client/client.py @@ -838,6 +838,7 @@ class MCPClient: call_tool_request_params: MCPCallToolRequestParams, host_progress_callback: Callable | None = None, raise_on_error: bool = False, + on_dispatch: Callable[[], None] | None = None, ) -> MCPCallToolResult: """ Call an MCP Tool. @@ -847,6 +848,8 @@ class MCPClient: ``isError=True`` result. The token-exchange (OBO) tool-call path uses this to detect an upstream 401 so it can re-mint the exchanged token and retry once; every other caller keeps the default and gets graceful ``isError`` degradation. + on_dispatch: Called once the session is ready, right before the tool call is sent, + so a caller can tell a call the server may have run from one that never left. """ verbose_logger.info("MCP client calling tool '%s'", call_tool_request_params.name) @@ -865,6 +868,8 @@ class MCPClient: async def _call_tool_operation(session: ClientSession): verbose_logger.debug("MCP client sending tool call to session") + if on_dispatch is not None: + on_dispatch() return await session.call_tool( name=call_tool_request_params.name, arguments=call_tool_request_params.arguments, diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 53f24cd28ad..902d03ec373 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -5820,6 +5820,7 @@ class MCPServerManager: user_api_key_auth: UserAPIKeyAuth | None, raw_headers: Mapping[str, str] | None = None, client_ip: str | None = None, + on_dispatch: Callable[[], None] | None = None, ) -> CallToolResult: """Call a token_exchange (OBO) tool; on an upstream 401/403 re-mint the token once and retry. @@ -5830,7 +5831,10 @@ class MCPServerManager: """ try: return await client.call_tool( - call_tool_params, host_progress_callback=host_progress_callback, raise_on_error=True + call_tool_params, + host_progress_callback=host_progress_callback, + raise_on_error=True, + on_dispatch=on_dispatch, ) except Exception as exc: if _extract_upstream_auth_failure(exc) is None: @@ -5848,7 +5852,9 @@ class MCPServerManager: raw_headers=raw_headers, client_ip=client_ip, ) - return await retry_client.call_tool(call_tool_params, host_progress_callback=host_progress_callback) + return await retry_client.call_tool( + call_tool_params, host_progress_callback=host_progress_callback, on_dispatch=on_dispatch + ) async def _call_regular_mcp_tool( self, @@ -5865,6 +5871,7 @@ class MCPServerManager: hook_extra_headers: dict[str, str] | None = None, user_api_key_auth: UserAPIKeyAuth | None = None, client_ip: str | None = None, + on_dispatch: Callable[[], None] | None = None, ) -> CallToolResult: """ Call a regular MCP tool using the MCP client. @@ -6028,6 +6035,7 @@ class MCPServerManager: client=client, call_tool_params=call_tool_params, host_progress_callback=host_progress_callback, + on_dispatch=on_dispatch, mcp_server=mcp_server, server_auth_header=server_auth_header, extra_headers=extra_headers, @@ -6049,7 +6057,9 @@ class MCPServerManager: async def _call_tool_via_client(client, params): async with self._limit_outbound_concurrency(mcp_server): if not relays_upstream_auth: - return await client.call_tool(params, host_progress_callback=host_progress_callback) + return await client.call_tool( + params, host_progress_callback=host_progress_callback, on_dispatch=on_dispatch + ) # The client-forwarded modes carry the caller's own upstream token, so an upstream # 401 (expired/invalid token) is the caller's to resolve: relay it as # MCPUpstreamAuthError so single-server REST callers turn it into a 401 + @@ -6061,7 +6071,10 @@ class MCPServerManager: # the same isError degradation the default path produces. try: return await client.call_tool( - params, host_progress_callback=host_progress_callback, raise_on_error=True + params, + host_progress_callback=host_progress_callback, + raise_on_error=True, + on_dispatch=on_dispatch, ) except Exception as e: auth_info: Final = _extract_upstream_auth_failure(e) @@ -6334,8 +6347,8 @@ class MCPServerManager: litellm_logging_obj: Optional request logger the guardrail hooks record their evaluations onto, so MCP guardrail activity reaches the Guardrails Monitor. See ``pre_call_tool_check`` - on_dispatch: Optional callback run once the call passed its pre-call checks - and is sent to the server, even if the server never returns a result + on_dispatch: Optional callback run right before the call is sent to the server, + after its headers and client are ready, even if the server never returns a result Returns: @@ -6430,18 +6443,16 @@ class MCPServerManager: resolved_token: Final = _request_resolved_auth_headers.set(resolved_auth_headers) try: async with self._limit_outbound_concurrency(mcp_server): + if on_dispatch is not None: + on_dispatch() return await self._call_openapi_tool_handler(mcp_server, name, arguments) finally: _request_auth_header.reset(auth_token) _request_extra_headers.reset(extra_token) _request_resolved_auth_headers.reset(resolved_token) - if on_dispatch is not None: - on_dispatch() tasks.append(asyncio.create_task(_call_openapi_via_handler())) else: - if on_dispatch is not None: - on_dispatch() return await self._call_regular_mcp_tool( mcp_server=mcp_server, original_tool_name=name, @@ -6456,6 +6467,7 @@ class MCPServerManager: host_progress_callback=host_progress_callback, hook_extra_headers=hook_result.get("extra_headers"), user_api_key_auth=user_api_key_auth, + on_dispatch=on_dispatch, ) return await self._gather_openapi_tool_tasks(tasks, proxy_logging_obj) diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index 4f372703234..bf6a83345ae 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -700,9 +700,9 @@ class LiteLLM_Proxy_MCP_Handler: ) -> list[MCPToolResult]: """Execute tool calls and return results. - ``on_tool_dispatched`` runs for every call that passes its pre-call checks and is sent to - an MCP server, even one that then times out. Calls rejected before that, for example by a - pre-call guardrail, do not trigger it. + ``on_tool_dispatched`` runs for every call that is sent to its MCP server, even one that then + times out. Calls rejected or failing before that, for example on a pre-call guardrail or a + credential the caller has not set, do not trigger it. """ from fastapi import HTTPException diff --git a/tests/test_litellm/experimental_mcp_client/test_mcp_client.py b/tests/test_litellm/experimental_mcp_client/test_mcp_client.py index ae30b086c6e..86ebfcf6e72 100644 --- a/tests/test_litellm/experimental_mcp_client/test_mcp_client.py +++ b/tests/test_litellm/experimental_mcp_client/test_mcp_client.py @@ -771,6 +771,36 @@ async def test_call_tool_does_not_log_arguments(): assert "gpt-5-mini" not in logged +@pytest.mark.asyncio +@pytest.mark.parametrize("session_opens", [True, False]) +async def test_call_tool_reports_dispatch_only_when_the_call_is_sent(session_opens): + """ + on_dispatch marks the moment a tool call can reach the server, so it fires right before the + session sends the call and never when the session could not be opened. + """ + from mcp.types import CallToolRequestParams + + events: list[str] = [] + + async def send(**_): + events.append("sent") + return MagicMock() + + async def run_with_session(operation, quiet_on_error=False): + if not session_opens: + raise ConnectionError("cannot reach the server") + return await operation(MagicMock(call_tool=send)) + + client = MCPClient(server_url="http://test-server") + client.run_with_session = run_with_session + + await client.call_tool( + CallToolRequestParams(name="search_tool", arguments={}), on_dispatch=lambda: events.append("dispatched") + ) + + assert events == (["dispatched", "sent"] if session_opens else []) + + @pytest.mark.asyncio async def test_get_prompt_does_not_log_arguments(): from mcp.types import GetPromptRequestParams diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_max_concurrent_requests.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_max_concurrent_requests.py index e11897b65c2..2f6f4a39b27 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_max_concurrent_requests.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_max_concurrent_requests.py @@ -46,7 +46,7 @@ def _make_server(server_id: str, max_concurrent_requests: Optional[int]) -> MCPS def _patch_client_with_tracker(manager: MCPServerManager, tracker: _ConcurrencyTracker): async def fake_create_mcp_client(server, **kwargs): class _ProbeClient: - async def call_tool(self, params, host_progress_callback=None): + async def call_tool(self, params, host_progress_callback=None, on_dispatch=None): tracker.enter(server.server_id) try: await asyncio.sleep(HOLD_SECONDS) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index f3d37a858ca..e95a8a490af 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -6730,7 +6730,7 @@ class TestMCPServerManager: # Create mock client that tracks call_tool usage mock_client = AsyncMock() - async def mock_call_tool(params, host_progress_callback=None): + async def mock_call_tool(params, host_progress_callback=None, on_dispatch=None): # Return a mock CallToolResult result = MagicMock(spec=CallToolResult) result.content = [{"type": "text", "text": "Tool executed successfully"}] @@ -6771,6 +6771,39 @@ class TestMCPServerManager: # Verify the MCP client call was awaited exactly once assert mock_client.call_tool.await_count == 1 + @pytest.mark.asyncio + async def test_call_tool_hands_on_dispatch_to_the_mcp_client(self): + """The MCP client fires on_dispatch when it sends the call, so call_tool must hand it down.""" + from unittest.mock import AsyncMock, MagicMock + + from mcp.types import CallToolResult + + manager = MCPServerManager() + manager.registry = { + "test-server": MCPServer( + server_id="test-server", name="test-server", transport=MCPTransport.http, url="http://test-server.com" + ) + } + manager.tool_name_to_mcp_server_name_mapping["test-server-test_tool"] = "test-server" + manager._create_mcp_client = AsyncMock(return_value=_RetryFakeClient(result=CallToolResult(content=[]))) + proxy_logging_obj = MagicMock() + proxy_logging_obj._create_mcp_request_object_from_kwargs = MagicMock(return_value={}) + proxy_logging_obj._convert_mcp_to_llm_format = MagicMock(return_value={}) + proxy_logging_obj.pre_call_hook = AsyncMock(return_value={}) + proxy_logging_obj.during_call_hook = AsyncMock(return_value=None) + dispatched = MagicMock() + + await manager.call_tool( + server_name="test-server", + name="test_tool", + arguments={}, + user_api_key_auth=MagicMock(object_permission=None, object_permission_id=None), + proxy_logging_obj=proxy_logging_obj, + on_dispatch=dispatched, + ) + + dispatched.assert_called_once_with() + @pytest.mark.asyncio async def test_get_allowed_mcp_servers_with_user_api_key_auth(self): """ @@ -10061,8 +10094,10 @@ class _RetryFakeClient: self._MCPClient = MCPClient self.attempts = 0 - async def call_tool(self, params, host_progress_callback=None, raise_on_error=False): + async def call_tool(self, params, host_progress_callback=None, raise_on_error=False, on_dispatch=None): self.attempts += 1 + if on_dispatch is not None: + on_dispatch() if self._raises is not None: if raise_on_error: raise self._raises @@ -10197,6 +10232,56 @@ class TestOBOCallToolRetry: manager._cred_provider.invalidate_credentials.assert_awaited_once() assert first.attempts == 1 and retry.attempts == 1 + @pytest.mark.asyncio + @pytest.mark.parametrize("path", ["plain", "relay", "obo_retry"]) + async def test_call_regular_reports_every_sent_attempt_through_on_dispatch(self, path): + """on_dispatch tells the caller a tool may have run, so every attempt that is sent must reach + it: once on the plain and relay paths, twice when the OBO path retries after an upstream 401.""" + manager = self._manager() + success = CallToolResult(content=[], isError=False) + clients = [_RetryFakeClient(raises=_UpstreamAuthError(401)), _RetryFakeClient(result=success)] + manager._create_mcp_client = AsyncMock(side_effect=clients if path == "obo_retry" else clients[1:]) + servers = { + "plain": MCPServer( + server_id="plain-srv", name="plain", url="https://upstream.example/mcp", transport=MCPTransport.sse + ), + "relay": MCPServer( + server_id="relay-srv", + name="relay", + url="https://upstream.example/mcp", + transport=MCPTransport.sse, + auth_type=MCPAuth.true_passthrough, + ), + "obo_retry": MCPServer( + server_id="id-jag-srv", + name="id-jag", + url="https://upstream.example/mcp", + transport=MCPTransport.sse, + auth_type=MCPAuth.oauth2_id_jag, + client_id="gateway-client", + client_secret="gateway-secret", + token_exchange_endpoint="https://org-idp.example/oauth2/token", + id_jag_resource_token_endpoint="https://resource-as.example/oauth2/token", + ), + } + dispatched = MagicMock() + + result = await manager._call_regular_mcp_tool( + mcp_server=servers[path], + original_tool_name="tool", + arguments={}, + tasks=[], + mcp_auth_header=None, + mcp_server_auth_headers=None, + oauth2_headers={"Authorization": "Bearer caller-id-token"} if path == "obo_retry" else None, + raw_headers={"authorization": "Bearer caller-token"} if path == "relay" else None, + proxy_logging_obj=None, + on_dispatch=dispatched, + ) + + assert result is success + assert dispatched.call_count == (2 if path == "obo_retry" else 1) + @pytest.mark.asyncio async def test_non_auth_error_does_not_retry(self): manager = self._manager() @@ -10273,7 +10358,7 @@ class TestOBOConcurrencyLimit: inflight = {"current": 0, "peak": 0} class _ConcurrencyRecordingClient: - async def call_tool(self, params, host_progress_callback=None, raise_on_error=False): + async def call_tool(self, params, host_progress_callback=None, raise_on_error=False, on_dispatch=None): inflight["current"] += 1 inflight["peak"] = max(inflight["peak"], inflight["current"]) try: diff --git a/tests/test_litellm/router_utils/test_mcp_tool_execution.py b/tests/test_litellm/router_utils/test_mcp_tool_execution.py index 94ff6f1ab83..8d254e322d7 100644 --- a/tests/test_litellm/router_utils/test_mcp_tool_execution.py +++ b/tests/test_litellm/router_utils/test_mcp_tool_execution.py @@ -347,6 +347,38 @@ async def test_router_does_not_replay_a_tool_call_that_failed_after_it_was_sent( assert model.calls == ["initial", "follow_up"] +@pytest.mark.asyncio +@respx.mock +async def test_router_retries_when_an_mcp_tool_call_never_reached_its_server( + ledger_gateway: _LedgerGateway, + monkeypatch: pytest.MonkeyPatch, +): + """ + Given: The MCP server needs a per-user header the caller has not set, so the tool call fails + before anything is sent to the server + When: The follow-up model call fails once with a 500 + Then: The router still retries, because replaying the request cannot repeat a side effect + """ + registry: Final = mcp_server_manager.global_mcp_server_manager.registry + monkeypatch.setitem( + registry, + "ledger", + registry["ledger"].model_copy( + update={ + "spec_path": None, + "static_headers": {"Authorization": "Bearer ${LEDGER_TOKEN}"}, + "env_vars": [{"name": "LEDGER_TOKEN", "scope": "user"}], + } + ), + ) + model: Final = _FakeModel(failing_follow_ups=1) + _serve(model) + + await _chat(_router(num_retries=2)) + + assert model.calls == ["initial", "follow_up", "initial", "follow_up"] + + @pytest.mark.asyncio @respx.mock async def test_router_does_not_fall_back_when_a_streamed_follow_up_breaks_after_a_tool_ran( From f78e072bce4febc75f8aba35e71d0fd046c55fc7 Mon Sep 17 00:00:00 2001 From: Atharva-Kanherkar <142440039+Atharva-Kanherkar@users.noreply.github.com> Date: Fri, 25 Sep 2026 18:59:12 +0530 Subject: [PATCH 12/15] test(router): keep the MCP replay tests in the router retry test file The repository extends the existing mapped test file for a bug fix, so the end-to-end MCP replay tests move next to the router's tagged-error tests in test_router_retry_non_retryable_errors.py instead of a new file Signed-off-by: Atharva-Kanherkar <142440039+Atharva-Kanherkar@users.noreply.github.com> --- .../router_utils/test_mcp_tool_execution.py | 456 ----------------- .../test_router_retry_non_retryable_errors.py | 461 +++++++++++++++++- 2 files changed, 459 insertions(+), 458 deletions(-) delete mode 100644 tests/test_litellm/router_utils/test_mcp_tool_execution.py diff --git a/tests/test_litellm/router_utils/test_mcp_tool_execution.py b/tests/test_litellm/router_utils/test_mcp_tool_execution.py deleted file mode 100644 index 8d254e322d7..00000000000 --- a/tests/test_litellm/router_utils/test_mcp_tool_execution.py +++ /dev/null @@ -1,456 +0,0 @@ -import asyncio -import json -from collections.abc import AsyncIterator, Awaitable, Callable -from typing import Final, NamedTuple -from unittest.mock import AsyncMock - -import httpx -import pytest -import respx -from mcp.types import Tool - -import litellm -from litellm.caching.caching import DualCache -from litellm.exceptions import GuardrailRaisedException -from litellm.integrations.custom_guardrail import CustomGuardrail -from litellm.proxy import proxy_server -from litellm.proxy._experimental.mcp_server import mcp_server_manager, server, tool_registry -from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing -from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth -from litellm.proxy.utils import ProxyLogging -from litellm.types.mcp_server.mcp_server_manager import MCPServer - -_UPSTREAM: Final = "https://upstream.example/v1" -_TOOL_NAME: Final = "ledger-commit_write" - - -def _ledger_tools() -> list[dict[str, str]]: - return [{"type": "mcp", "server_url": "litellm_proxy/ledger", "require_approval": "never"}] - - -def _ledger_caller() -> UserAPIKeyAuth: - return UserAPIKeyAuth( - object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="test", mcp_servers=["ledger"]) - ) - - -class _LedgerGateway(NamedTuple): - tool: AsyncMock - list_tools: AsyncMock - - -@pytest.fixture -def ledger_gateway(monkeypatch: pytest.MonkeyPatch) -> _LedgerGateway: - """Serve one auto-approved MCP tool through the real MCP manager and tool registry.""" - manager: Final = mcp_server_manager.MCPServerManager() - manager.registry = { - "ledger": MCPServer( - server_id="ledger", - name="ledger", - server_name="ledger", - transport="http", - url="https://ledger.example/mcp", - # a spec_path server runs its tools from the local tool registry instead of over the network - spec_path="ledger.json", - auth_type="none", - ) - } - manager.tool_name_to_mcp_server_name_mapping = {_TOOL_NAME: "ledger"} - tool: Final = AsyncMock(return_value={"written": True}) - registry: Final = tool_registry.MCPToolRegistry() - registry.register_tool(_TOOL_NAME, "Write one ledger entry", {"type": "object"}, tool) - list_tools: Final = AsyncMock( - return_value=AggregateToolListing(tools=[Tool(name=_TOOL_NAME, inputSchema={"type": "object"})], outcomes={}) - ) - monkeypatch.setattr(tool_registry, "global_mcp_tool_registry", registry) - monkeypatch.setattr(mcp_server_manager, "global_mcp_server_manager", manager) - monkeypatch.setattr(proxy_server, "proxy_logging_obj", ProxyLogging(user_api_key_cache=DualCache())) - monkeypatch.setattr(server, "_get_tools_from_mcp_servers", list_tools) - monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - return _LedgerGateway(tool=tool, list_tools=list_tools) - - -def _is_follow_up(request: httpx.Request) -> bool: - body: Final = json.loads(request.content) - messages: Final = body.get("messages") or [] - items: Final = body["input"] if isinstance(body.get("input"), list) else [] - return any(message.get("role") == "tool" for message in messages) or any( - item.get("type") == "function_call_output" for item in items - ) - - -def _chat_completion(message: dict[str, object], finish_reason: str) -> httpx.Response: - return httpx.Response( - 200, - json={ - "id": "chatcmpl-1", - "object": "chat.completion", - "created": 0, - "model": "gpt-4.1-mini", - "choices": [{"index": 0, "message": message, "finish_reason": finish_reason}], - "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, - }, - ) - - -def _responses_api_response(output_item: dict[str, object]) -> httpx.Response: - return httpx.Response( - 200, - json={ - "id": "resp_1", - "object": "response", - "created_at": 0, - "status": "completed", - "model": "gpt-4.1-mini", - "output": [output_item], - "parallel_tool_calls": True, - "tool_choice": "auto", - "tools": [], - "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, - }, - ) - - -def _tool_call(responses_api: bool) -> httpx.Response: - arguments: Final = json.dumps({"entry": "alpha"}) - if responses_api: - return _responses_api_response( - {"type": "function_call", "id": "fc_1", "call_id": "call_1", "name": _TOOL_NAME, "arguments": arguments} - ) - return _chat_completion( - { - "role": "assistant", - "content": None, - "tool_calls": [ - {"id": "call_1", "type": "function", "function": {"name": _TOOL_NAME, "arguments": arguments}} - ], - }, - "tool_calls", - ) - - -def _final_answer(responses_api: bool) -> httpx.Response: - if responses_api: - return _responses_api_response( - { - "type": "message", - "id": "msg_1", - "role": "assistant", - "status": "completed", - "content": [{"type": "output_text", "text": "Recorded alpha", "annotations": []}], - } - ) - return _chat_completion({"role": "assistant", "content": "Recorded alpha"}, "stop") - - -class _FakeModel: - """Answers both OpenAI routes: a tool call, then a final answer once the tool result arrives.""" - - def __init__(self, failing_initial_calls: int = 0, failing_follow_ups: int = 0) -> None: - self.calls: list[str] = [] - self._failures: Final = {"initial": failing_initial_calls, "follow_up": failing_follow_ups} - - def __call__(self, request: httpx.Request) -> httpx.Response: - kind: Final = "follow_up" if _is_follow_up(request) else "initial" - self.calls.append(kind) - if self.calls.count(kind) <= self._failures[kind]: - return httpx.Response(500, json={"error": {"message": "upstream failed", "type": "server_error"}}) - responses_api: Final = request.url.path.endswith("/responses") - return _tool_call(responses_api) if kind == "initial" else _final_answer(responses_api) - - -class _StreamBrokenBeforeFirstChunk(httpx.AsyncByteStream): - async def __aiter__(self) -> AsyncIterator[bytes]: - raise httpx.ReadError("connection dropped before the first chunk") - yield b"" - - -def _chat_tool_call_stream() -> httpx.Response: - tool_call: Final = { - "index": 0, - "id": "call_1", - "type": "function", - "function": {"name": _TOOL_NAME, "arguments": json.dumps({"entry": "alpha"})}, - } - chunks: Final = ( - {"role": "assistant", "content": None, "tool_calls": [tool_call]}, - {}, - ) - body: Final = "".join( - "data: " - + json.dumps( - { - "id": "chatcmpl-1", - "object": "chat.completion.chunk", - "created": 0, - "model": "gpt-4.1-mini", - "choices": [{"index": 0, "delta": delta, "finish_reason": None if delta else "tool_calls"}], - } - ) - + "\n\n" - for delta in chunks - ) - return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=f"{body}data: [DONE]\n\n") - - -def _serve(model: _FakeModel) -> None: - respx.post(f"{_UPSTREAM}/chat/completions").mock(side_effect=model) - respx.post(f"{_UPSTREAM}/responses").mock(side_effect=model) - - -def _router(num_retries: int, fallbacks: list[dict[str, list[str]]] | None = None) -> litellm.Router: - deployment: Final = {"model": "openai/gpt-4.1-mini", "api_key": "sk-test", "api_base": _UPSTREAM} - return litellm.Router( - # two replicas, so the router retries right away instead of backing off - model_list=[ - {"model_name": "repro-model", "litellm_params": deployment}, - {"model_name": "repro-model", "litellm_params": deployment}, - {"model_name": "backup-model", "litellm_params": deployment}, - ], - num_retries=num_retries, - fallbacks=fallbacks, - ) - - -def _chat(router: litellm.Router) -> Awaitable[object]: - return router.acompletion( - model="repro-model", - messages=[{"role": "user", "content": "Record entry alpha"}], - tools=_ledger_tools(), - metadata={"user_api_key_auth": _ledger_caller()}, - ) - - -def _responses(router: litellm.Router) -> Awaitable[object]: - return router.aresponses( - model="repro-model", - input="Record entry alpha", - tools=_ledger_tools(), - litellm_metadata={"user_api_key_auth": _ledger_caller()}, - ) - - -def _messages(router: litellm.Router) -> Awaitable[object]: - return router.aanthropic_messages( - model="repro-model", - max_tokens=64, - messages=[{"role": "user", "content": "Record entry alpha"}], - tools=_ledger_tools(), - litellm_metadata={"user_api_key_auth": _ledger_caller()}, - ) - - -_ENDPOINTS: Final = pytest.mark.parametrize( - "send", [_chat, _responses, _messages], ids=["chat_completions", "responses", "messages"] -) - - -@pytest.mark.asyncio -@_ENDPOINTS -@pytest.mark.parametrize( - "router_kwargs", - [{"num_retries": 2}, {"num_retries": 0, "fallbacks": [{"repro-model": ["backup-model"]}]}], - ids=["retries", "fallbacks"], -) -@respx.mock -async def test_router_does_not_replay_an_executed_mcp_tool( - ledger_gateway: _LedgerGateway, - send: Callable[[litellm.Router], Awaitable[object]], - router_kwargs: dict[str, object], -): - """ - Regression test for #43153: an auto-approved MCP tool must run once per request. - - Given: The model asks for the tool, the tool runs, and every follow-up model call returns a 500 - When: The router has retries or a fallback group configured - Then: The client gets the 500 after one tool execution and one follow-up call - - A retry or fallback re-runs the whole MCP loop, so before the fix every replay executed - the tool again (three writes with num_retries=2, two with one fallback). - """ - model: Final = _FakeModel(failing_follow_ups=3) - _serve(model) - - with pytest.raises(litellm.InternalServerError): - await send(_router(**router_kwargs)) - - assert ledger_gateway.tool.await_count == 1 - assert model.calls == ["initial", "follow_up"] - - -@pytest.mark.asyncio -@_ENDPOINTS -@respx.mock -async def test_router_retries_when_a_guardrail_blocked_every_mcp_tool( - ledger_gateway: _LedgerGateway, - send: Callable[[litellm.Router], Awaitable[object]], - monkeypatch: pytest.MonkeyPatch, -): - """ - Given: A pre-call guardrail blocks the tool call, so no tool runs - When: The follow-up model call fails once with a 500 - Then: The router still retries, because replaying the request cannot repeat a side effect - """ - - class BlockEveryToolCall(CustomGuardrail): - async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): - raise GuardrailRaisedException(message="tool calls are blocked", blocked_content=True) - - monkeypatch.setattr( - litellm, "callbacks", [BlockEveryToolCall(guardrail_name="block", event_hook="pre_mcp_call", default_on=True)] - ) - model: Final = _FakeModel(failing_follow_ups=1) - _serve(model) - - await send(_router(num_retries=2)) - - assert ledger_gateway.tool.await_count == 0 - assert model.calls == ["initial", "follow_up", "initial", "follow_up"] - - -@pytest.mark.asyncio -@_ENDPOINTS -@respx.mock -async def test_router_does_not_replay_a_tool_call_that_failed_after_it_was_sent( - ledger_gateway: _LedgerGateway, - send: Callable[[litellm.Router], Awaitable[object]], - monkeypatch: pytest.MonkeyPatch, -): - """ - Given: The tool runs, but its call fails afterwards, here because a during-call guardrail - rejects the result, which leaves the router in the same spot as a timeout after the write - When: Every follow-up model call returns a 500 - Then: The router does not replay the request, because the tool may already have written - """ - tool_ran: Final = asyncio.Event() - - def write(**_: object) -> dict[str, bool]: - tool_ran.set() - return {"written": True} - - class RejectToolResult(CustomGuardrail): - async def async_moderation_hook(self, data, user_api_key_dict, call_type): - await tool_ran.wait() - raise GuardrailRaisedException(message="tool result rejected", blocked_content=True) - - ledger_gateway.tool.side_effect = write - monkeypatch.setattr( - litellm, "callbacks", [RejectToolResult(guardrail_name="reject", event_hook="during_mcp_call", default_on=True)] - ) - model: Final = _FakeModel(failing_follow_ups=3) - _serve(model) - - with pytest.raises(litellm.InternalServerError): - await send(_router(num_retries=2)) - - assert ledger_gateway.tool.await_count == 1 - assert model.calls == ["initial", "follow_up"] - - -@pytest.mark.asyncio -@respx.mock -async def test_router_retries_when_an_mcp_tool_call_never_reached_its_server( - ledger_gateway: _LedgerGateway, - monkeypatch: pytest.MonkeyPatch, -): - """ - Given: The MCP server needs a per-user header the caller has not set, so the tool call fails - before anything is sent to the server - When: The follow-up model call fails once with a 500 - Then: The router still retries, because replaying the request cannot repeat a side effect - """ - registry: Final = mcp_server_manager.global_mcp_server_manager.registry - monkeypatch.setitem( - registry, - "ledger", - registry["ledger"].model_copy( - update={ - "spec_path": None, - "static_headers": {"Authorization": "Bearer ${LEDGER_TOKEN}"}, - "env_vars": [{"name": "LEDGER_TOKEN", "scope": "user"}], - } - ), - ) - model: Final = _FakeModel(failing_follow_ups=1) - _serve(model) - - await _chat(_router(num_retries=2)) - - assert model.calls == ["initial", "follow_up", "initial", "follow_up"] - - -@pytest.mark.asyncio -@respx.mock -async def test_router_does_not_fall_back_when_a_streamed_follow_up_breaks_after_a_tool_ran( - ledger_gateway: _LedgerGateway, -): - """ - Given: A streaming Chat Completions request whose tool ran - When: The follow-up stream breaks before its first chunk and a fallback group is configured - Then: The client gets the error instead of a fallback that runs the tool again - """ - calls: Final[list[str]] = [] - - def stream_model(request: httpx.Request) -> httpx.Response: - kind: Final = "follow_up" if _is_follow_up(request) else "initial" - calls.append(kind) - if kind == "follow_up": - return httpx.Response( - 200, headers={"content-type": "text/event-stream"}, stream=_StreamBrokenBeforeFirstChunk() - ) - return _chat_tool_call_stream() - - respx.post(f"{_UPSTREAM}/chat/completions").mock(side_effect=stream_model) - stream: Final = await _router(num_retries=0, fallbacks=[{"repro-model": ["backup-model"]}]).acompletion( - model="repro-model", - messages=[{"role": "user", "content": "Record entry alpha"}], - tools=_ledger_tools(), - metadata={"user_api_key_auth": _ledger_caller()}, - stream=True, - ) - - with pytest.raises(litellm.APIConnectionError): - async for _ in stream: - pass - - assert ledger_gateway.tool.await_count == 1 - assert calls == ["initial", "follow_up"] - - -@pytest.mark.asyncio -@respx.mock -async def test_router_still_retries_a_failure_before_any_mcp_tool_ran(ledger_gateway: _LedgerGateway): - """ - Given: The initial model call fails once with a 500, then asks for the tool - When: The router retries - Then: The retry succeeds and the tool runs once - """ - model: Final = _FakeModel(failing_initial_calls=1) - _serve(model) - - await _chat(_router(num_retries=2)) - - assert ledger_gateway.tool.await_count == 1 - assert model.calls == ["initial", "initial", "follow_up"] - - -@pytest.mark.asyncio -@respx.mock -async def test_responses_tool_listing_failure_after_a_tool_ran_is_not_replayed(ledger_gateway: _LedgerGateway): - """ - Given: A Responses request whose tool ran and whose follow-up succeeded - When: Listing the MCP tools again for the output items fails - Then: The router surfaces that failure instead of replaying the request and the tool - """ - listing: Final = ledger_gateway.list_tools.return_value - listing_error: Final = RuntimeError("tool listing failed") - ledger_gateway.list_tools.side_effect = [listing, listing_error, listing, listing_error, listing, listing_error] - model: Final = _FakeModel() - _serve(model) - - with pytest.raises(Exception, match="tool listing failed"): - await _responses(_router(num_retries=2)) - - assert ledger_gateway.tool.await_count == 1 - assert ledger_gateway.list_tools.await_count == 2 - assert model.calls == ["initial", "follow_up"] diff --git a/tests/test_litellm/test_router_retry_non_retryable_errors.py b/tests/test_litellm/test_router_retry_non_retryable_errors.py index 9d702a55086..14d3334c536 100644 --- a/tests/test_litellm/test_router_retry_non_retryable_errors.py +++ b/tests/test_litellm/test_router_retry_non_retryable_errors.py @@ -12,14 +12,19 @@ Regression tests for https://github.com/BerriAI/litellm/issues/21343 import asyncio import datetime -from collections.abc import Awaitable, Callable -from typing import Final +import json +from collections.abc import AsyncIterator, Awaitable, Callable +from typing import TYPE_CHECKING, Final, NamedTuple from unittest.mock import AsyncMock, patch +import httpx import pytest +import respx import litellm from litellm import Router +from litellm.exceptions import GuardrailRaisedException +from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( _union_duration_ms, @@ -31,6 +36,10 @@ from litellm.router_utils.mcp_tool_execution import mark_mcp_tools_executed from litellm.types.router import RetryPolicy from litellm.utils import function_setup +if TYPE_CHECKING: + from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy._types import UserAPIKeyAuth + def _make_rate_limit_error(message="Rate limited"): """Create a RateLimitError for testing.""" @@ -445,3 +454,451 @@ async def test_500_after_mcp_tool_execution_does_not_fall_back(tools_executed): attempted_models: Final = [awaited.kwargs["model"] for awaited in attempt.await_args_list] assert attempted_models == (["test-model"] if tools_executed else ["test-model", "backup-model"]) + + +_UPSTREAM: Final = "https://upstream.example/v1" +_TOOL_NAME: Final = "ledger-commit_write" + + +def _ledger_tools() -> list[dict[str, str]]: + return [{"type": "mcp", "server_url": "litellm_proxy/ledger", "require_approval": "never"}] + + +def _ledger_caller() -> "UserAPIKeyAuth": + from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth + + return UserAPIKeyAuth( + object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="test", mcp_servers=["ledger"]) + ) + + +class _LedgerGateway(NamedTuple): + tool: AsyncMock + list_tools: AsyncMock + manager: "MCPServerManager" + + +@pytest.fixture +def ledger_gateway(monkeypatch: pytest.MonkeyPatch) -> _LedgerGateway: + """Serve one auto-approved MCP tool through the real MCP manager and tool registry.""" + from mcp.types import Tool + + from litellm.caching.caching import DualCache + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server import mcp_server_manager, server, tool_registry + from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing + from litellm.proxy.utils import ProxyLogging + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + manager: Final = mcp_server_manager.MCPServerManager() + manager.registry = { + "ledger": MCPServer( + server_id="ledger", + name="ledger", + server_name="ledger", + transport="http", + url="https://ledger.example/mcp", + # a spec_path server runs its tools from the local tool registry instead of over the network + spec_path="ledger.json", + auth_type="none", + ) + } + manager.tool_name_to_mcp_server_name_mapping = {_TOOL_NAME: "ledger"} + tool: Final = AsyncMock(return_value={"written": True}) + registry: Final = tool_registry.MCPToolRegistry() + registry.register_tool(_TOOL_NAME, "Write one ledger entry", {"type": "object"}, tool) + list_tools: Final = AsyncMock( + return_value=AggregateToolListing(tools=[Tool(name=_TOOL_NAME, inputSchema={"type": "object"})], outcomes={}) + ) + monkeypatch.setattr(tool_registry, "global_mcp_tool_registry", registry) + monkeypatch.setattr(mcp_server_manager, "global_mcp_server_manager", manager) + monkeypatch.setattr(proxy_server, "proxy_logging_obj", ProxyLogging(user_api_key_cache=DualCache())) + monkeypatch.setattr(server, "_get_tools_from_mcp_servers", list_tools) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + return _LedgerGateway(tool=tool, list_tools=list_tools, manager=manager) + + +def _is_follow_up(request: httpx.Request) -> bool: + body: Final = json.loads(request.content) + messages: Final = body.get("messages") or [] + items: Final = body["input"] if isinstance(body.get("input"), list) else [] + return any(message.get("role") == "tool" for message in messages) or any( + item.get("type") == "function_call_output" for item in items + ) + + +def _chat_completion(message: dict[str, object], finish_reason: str) -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 0, + "model": "gpt-4.1-mini", + "choices": [{"index": 0, "message": message, "finish_reason": finish_reason}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + + +def _responses_api_response(output_item: dict[str, object]) -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": "resp_1", + "object": "response", + "created_at": 0, + "status": "completed", + "model": "gpt-4.1-mini", + "output": [output_item], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + }, + ) + + +def _tool_call(responses_api: bool) -> httpx.Response: + arguments: Final = json.dumps({"entry": "alpha"}) + if responses_api: + return _responses_api_response( + {"type": "function_call", "id": "fc_1", "call_id": "call_1", "name": _TOOL_NAME, "arguments": arguments} + ) + return _chat_completion( + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": _TOOL_NAME, "arguments": arguments}} + ], + }, + "tool_calls", + ) + + +def _final_answer(responses_api: bool) -> httpx.Response: + if responses_api: + return _responses_api_response( + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "Recorded alpha", "annotations": []}], + } + ) + return _chat_completion({"role": "assistant", "content": "Recorded alpha"}, "stop") + + +class _FakeModel: + """Answers both OpenAI routes: a tool call, then a final answer once the tool result arrives.""" + + def __init__(self, failing_initial_calls: int = 0, failing_follow_ups: int = 0) -> None: + self.calls: list[str] = [] + self._failures: Final = {"initial": failing_initial_calls, "follow_up": failing_follow_ups} + + def __call__(self, request: httpx.Request) -> httpx.Response: + kind: Final = "follow_up" if _is_follow_up(request) else "initial" + self.calls.append(kind) + if self.calls.count(kind) <= self._failures[kind]: + return httpx.Response(500, json={"error": {"message": "upstream failed", "type": "server_error"}}) + responses_api: Final = request.url.path.endswith("/responses") + return _tool_call(responses_api) if kind == "initial" else _final_answer(responses_api) + + +class _StreamBrokenBeforeFirstChunk(httpx.AsyncByteStream): + async def __aiter__(self) -> AsyncIterator[bytes]: + raise httpx.ReadError("connection dropped before the first chunk") + yield b"" + + +def _chat_tool_call_stream() -> httpx.Response: + tool_call: Final = { + "index": 0, + "id": "call_1", + "type": "function", + "function": {"name": _TOOL_NAME, "arguments": json.dumps({"entry": "alpha"})}, + } + chunks: Final = ( + {"role": "assistant", "content": None, "tool_calls": [tool_call]}, + {}, + ) + body: Final = "".join( + "data: " + + json.dumps( + { + "id": "chatcmpl-1", + "object": "chat.completion.chunk", + "created": 0, + "model": "gpt-4.1-mini", + "choices": [{"index": 0, "delta": delta, "finish_reason": None if delta else "tool_calls"}], + } + ) + + "\n\n" + for delta in chunks + ) + return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=f"{body}data: [DONE]\n\n") + + +def _serve(model: _FakeModel) -> None: + respx.post(f"{_UPSTREAM}/chat/completions").mock(side_effect=model) + respx.post(f"{_UPSTREAM}/responses").mock(side_effect=model) + + +def _mcp_router(num_retries: int, fallbacks: list[dict[str, list[str]]] | None = None) -> litellm.Router: + deployment: Final = {"model": "openai/gpt-4.1-mini", "api_key": "sk-test", "api_base": _UPSTREAM} + return litellm.Router( + # two replicas, so the router retries right away instead of backing off + model_list=[ + {"model_name": "repro-model", "litellm_params": deployment}, + {"model_name": "repro-model", "litellm_params": deployment}, + {"model_name": "backup-model", "litellm_params": deployment}, + ], + num_retries=num_retries, + fallbacks=fallbacks, + ) + + +def _chat(router: litellm.Router) -> Awaitable[object]: + return router.acompletion( + model="repro-model", + messages=[{"role": "user", "content": "Record entry alpha"}], + tools=_ledger_tools(), + metadata={"user_api_key_auth": _ledger_caller()}, + ) + + +def _responses(router: litellm.Router) -> Awaitable[object]: + return router.aresponses( + model="repro-model", + input="Record entry alpha", + tools=_ledger_tools(), + litellm_metadata={"user_api_key_auth": _ledger_caller()}, + ) + + +def _messages(router: litellm.Router) -> Awaitable[object]: + return router.aanthropic_messages( + model="repro-model", + max_tokens=64, + messages=[{"role": "user", "content": "Record entry alpha"}], + tools=_ledger_tools(), + litellm_metadata={"user_api_key_auth": _ledger_caller()}, + ) + + +_ENDPOINTS: Final = pytest.mark.parametrize( + "send", [_chat, _responses, _messages], ids=["chat_completions", "responses", "messages"] +) + + +@pytest.mark.asyncio +@_ENDPOINTS +@pytest.mark.parametrize( + "router_kwargs", + [{"num_retries": 2}, {"num_retries": 0, "fallbacks": [{"repro-model": ["backup-model"]}]}], + ids=["retries", "fallbacks"], +) +@respx.mock +async def test_router_does_not_replay_an_executed_mcp_tool( + ledger_gateway: _LedgerGateway, + send: Callable[[litellm.Router], Awaitable[object]], + router_kwargs: dict[str, object], +): + """ + Regression test for #43153: an auto-approved MCP tool must run once per request. + + Given: The model asks for the tool, the tool runs, and every follow-up model call returns a 500 + When: The router has retries or a fallback group configured + Then: The client gets the 500 after one tool execution and one follow-up call + + A retry or fallback re-runs the whole MCP loop, so before the fix every replay executed + the tool again (three writes with num_retries=2, two with one fallback). + """ + model: Final = _FakeModel(failing_follow_ups=3) + _serve(model) + + with pytest.raises(litellm.InternalServerError): + await send(_mcp_router(**router_kwargs)) + + assert ledger_gateway.tool.await_count == 1 + assert model.calls == ["initial", "follow_up"] + + +@pytest.mark.asyncio +@_ENDPOINTS +@respx.mock +async def test_router_retries_when_a_guardrail_blocked_every_mcp_tool( + ledger_gateway: _LedgerGateway, + send: Callable[[litellm.Router], Awaitable[object]], + monkeypatch: pytest.MonkeyPatch, +): + """ + Given: A pre-call guardrail blocks the tool call, so no tool runs + When: The follow-up model call fails once with a 500 + Then: The router still retries, because replaying the request cannot repeat a side effect + """ + + class BlockEveryToolCall(CustomGuardrail): + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + raise GuardrailRaisedException(message="tool calls are blocked", blocked_content=True) + + monkeypatch.setattr( + litellm, "callbacks", [BlockEveryToolCall(guardrail_name="block", event_hook="pre_mcp_call", default_on=True)] + ) + model: Final = _FakeModel(failing_follow_ups=1) + _serve(model) + + await send(_mcp_router(num_retries=2)) + + assert ledger_gateway.tool.await_count == 0 + assert model.calls == ["initial", "follow_up", "initial", "follow_up"] + + +@pytest.mark.asyncio +@_ENDPOINTS +@respx.mock +async def test_router_does_not_replay_a_tool_call_that_failed_after_it_was_sent( + ledger_gateway: _LedgerGateway, + send: Callable[[litellm.Router], Awaitable[object]], + monkeypatch: pytest.MonkeyPatch, +): + """ + Given: The tool runs, but its call fails afterwards, here because a during-call guardrail + rejects the result, which leaves the router in the same spot as a timeout after the write + When: Every follow-up model call returns a 500 + Then: The router does not replay the request, because the tool may already have written + """ + tool_ran: Final = asyncio.Event() + + def write(**_: object) -> dict[str, bool]: + tool_ran.set() + return {"written": True} + + class RejectToolResult(CustomGuardrail): + async def async_moderation_hook(self, data, user_api_key_dict, call_type): + await tool_ran.wait() + raise GuardrailRaisedException(message="tool result rejected", blocked_content=True) + + ledger_gateway.tool.side_effect = write + monkeypatch.setattr( + litellm, "callbacks", [RejectToolResult(guardrail_name="reject", event_hook="during_mcp_call", default_on=True)] + ) + model: Final = _FakeModel(failing_follow_ups=3) + _serve(model) + + with pytest.raises(litellm.InternalServerError): + await send(_mcp_router(num_retries=2)) + + assert ledger_gateway.tool.await_count == 1 + assert model.calls == ["initial", "follow_up"] + + +@pytest.mark.asyncio +@respx.mock +async def test_router_retries_when_an_mcp_tool_call_never_reached_its_server( + ledger_gateway: _LedgerGateway, + monkeypatch: pytest.MonkeyPatch, +): + """ + Given: The MCP server needs a per-user header the caller has not set, so the tool call fails + before anything is sent to the server + When: The follow-up model call fails once with a 500 + Then: The router still retries, because replaying the request cannot repeat a side effect + """ + registry: Final = ledger_gateway.manager.registry + monkeypatch.setitem( + registry, + "ledger", + registry["ledger"].model_copy( + update={ + "spec_path": None, + "static_headers": {"Authorization": "Bearer ${LEDGER_TOKEN}"}, + "env_vars": [{"name": "LEDGER_TOKEN", "scope": "user"}], + } + ), + ) + model: Final = _FakeModel(failing_follow_ups=1) + _serve(model) + + await _chat(_mcp_router(num_retries=2)) + + assert model.calls == ["initial", "follow_up", "initial", "follow_up"] + + +@pytest.mark.asyncio +@respx.mock +async def test_router_does_not_fall_back_when_a_streamed_follow_up_breaks_after_a_tool_ran( + ledger_gateway: _LedgerGateway, +): + """ + Given: A streaming Chat Completions request whose tool ran + When: The follow-up stream breaks before its first chunk and a fallback group is configured + Then: The client gets the error instead of a fallback that runs the tool again + """ + calls: Final[list[str]] = [] + + def stream_model(request: httpx.Request) -> httpx.Response: + kind: Final = "follow_up" if _is_follow_up(request) else "initial" + calls.append(kind) + if kind == "follow_up": + return httpx.Response( + 200, headers={"content-type": "text/event-stream"}, stream=_StreamBrokenBeforeFirstChunk() + ) + return _chat_tool_call_stream() + + respx.post(f"{_UPSTREAM}/chat/completions").mock(side_effect=stream_model) + stream: Final = await _mcp_router(num_retries=0, fallbacks=[{"repro-model": ["backup-model"]}]).acompletion( + model="repro-model", + messages=[{"role": "user", "content": "Record entry alpha"}], + tools=_ledger_tools(), + metadata={"user_api_key_auth": _ledger_caller()}, + stream=True, + ) + + with pytest.raises(litellm.APIConnectionError): + async for _ in stream: + pass + + assert ledger_gateway.tool.await_count == 1 + assert calls == ["initial", "follow_up"] + + +@pytest.mark.asyncio +@respx.mock +async def test_router_still_retries_a_failure_before_any_mcp_tool_ran(ledger_gateway: _LedgerGateway): + """ + Given: The initial model call fails once with a 500, then asks for the tool + When: The router retries + Then: The retry succeeds and the tool runs once + """ + model: Final = _FakeModel(failing_initial_calls=1) + _serve(model) + + await _chat(_mcp_router(num_retries=2)) + + assert ledger_gateway.tool.await_count == 1 + assert model.calls == ["initial", "initial", "follow_up"] + + +@pytest.mark.asyncio +@respx.mock +async def test_responses_tool_listing_failure_after_a_tool_ran_is_not_replayed(ledger_gateway: _LedgerGateway): + """ + Given: A Responses request whose tool ran and whose follow-up succeeded + When: Listing the MCP tools again for the output items fails + Then: The router surfaces that failure instead of replaying the request and the tool + """ + listing: Final = ledger_gateway.list_tools.return_value + listing_error: Final = RuntimeError("tool listing failed") + ledger_gateway.list_tools.side_effect = [listing, listing_error, listing, listing_error, listing, listing_error] + model: Final = _FakeModel() + _serve(model) + + with pytest.raises(Exception, match="tool listing failed"): + await _responses(_mcp_router(num_retries=2)) + + assert ledger_gateway.tool.await_count == 1 + assert ledger_gateway.list_tools.await_count == 2 + assert model.calls == ["initial", "follow_up"] From 99b3c06edeb0aca380e3c5cb8a57a60c37ee6eed Mon Sep 17 00:00:00 2001 From: Atharva-Kanherkar <142440039+Atharva-Kanherkar@users.noreply.github.com> Date: Fri, 25 Sep 2026 20:10:01 +0530 Subject: [PATCH 13/15] style(mcp): drop routine comments from the replay change The repository keeps comments for complex business logic, tool directives and TODOs, so the notes on the chain depth cap and the test fixture's setup go, and the names carry that meaning Signed-off-by: Atharva-Kanherkar <142440039+Atharva-Kanherkar@users.noreply.github.com> --- litellm/router_utils/mcp_tool_execution.py | 1 - tests/test_litellm/test_router_retry_non_retryable_errors.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/litellm/router_utils/mcp_tool_execution.py b/litellm/router_utils/mcp_tool_execution.py index 3ed4d2284ed..15f29fd1520 100644 --- a/litellm/router_utils/mcp_tool_execution.py +++ b/litellm/router_utils/mcp_tool_execution.py @@ -4,7 +4,6 @@ from typing import Final _MCP_TOOLS_EXECUTED_ATTR: Final = "_litellm_mcp_tools_executed" -# Real exception chains are a few links deep; the cap also makes the walk cycle-safe. _MAX_EXCEPTION_CHAIN_DEPTH: Final = 20 diff --git a/tests/test_litellm/test_router_retry_non_retryable_errors.py b/tests/test_litellm/test_router_retry_non_retryable_errors.py index 14d3334c536..ed9bc0af4bb 100644 --- a/tests/test_litellm/test_router_retry_non_retryable_errors.py +++ b/tests/test_litellm/test_router_retry_non_retryable_errors.py @@ -498,7 +498,6 @@ def ledger_gateway(monkeypatch: pytest.MonkeyPatch) -> _LedgerGateway: server_name="ledger", transport="http", url="https://ledger.example/mcp", - # a spec_path server runs its tools from the local tool registry instead of over the network spec_path="ledger.json", auth_type="none", ) @@ -649,7 +648,6 @@ def _serve(model: _FakeModel) -> None: def _mcp_router(num_retries: int, fallbacks: list[dict[str, list[str]]] | None = None) -> litellm.Router: deployment: Final = {"model": "openai/gpt-4.1-mini", "api_key": "sk-test", "api_base": _UPSTREAM} return litellm.Router( - # two replicas, so the router retries right away instead of backing off model_list=[ {"model_name": "repro-model", "litellm_params": deployment}, {"model_name": "repro-model", "litellm_params": deployment}, From a66cb5ea255426d500d546dd63c86d67e8ca4f9b Mon Sep 17 00:00:00 2001 From: Atharva-Kanherkar Date: Tue, 3 Nov 2026 15:12:21 +0900 Subject: [PATCH 14/15] fix(mcp): port the replay guard onto main's router core and MCP paths Main moved the router retry and fallback loop into router_python_core.py and renamed the MCP client, server manager and Anthropic handler modules. Re-apply the same guard there: tag an error raised after an MCP tool call was sent, and let the router surface a tagged error instead of retrying or falling back The Anthropic loop now also awaits with_safeguard_results after the tools ran, and the streamed chat follow-up is prepared after them, so both run under the guard too. Main no longer lists the MCP tools a second time after the Responses follow-up, so that guard and its test are gone Signed-off-by: Atharva-Kanherkar --- .../anthropic/messages/native/mcp_handler.py | 21 +- litellm/mcp_client/client.py | 5 + .../proxy/mcp_server/mcp_server_manager.py | 13 + litellm/responses/main.py | 10 +- .../responses/mcp/chat_completions_handler.py | 16 +- .../mcp/litellm_proxy_mcp_handler.py | 9 +- litellm/router_python_core.py | 6 +- litellm/router_utils/mcp_tool_execution.py | 62 +++ tests/unit/mcp_client/test_mcp_client.py | 30 + .../test_mcp_max_concurrent_requests.py | 4 +- .../mcp_server/test_mcp_server_manager.py | 80 +++ .../test_router_retry_non_retryable_errors.py | 527 +++++++++++++++++- 12 files changed, 764 insertions(+), 19 deletions(-) create mode 100644 litellm/router_utils/mcp_tool_execution.py diff --git a/litellm/llms/anthropic/messages/native/mcp_handler.py b/litellm/llms/anthropic/messages/native/mcp_handler.py index 8fe18d79f24..32b78766d53 100644 --- a/litellm/llms/anthropic/messages/native/mcp_handler.py +++ b/litellm/llms/anthropic/messages/native/mcp_handler.py @@ -13,6 +13,7 @@ from typing import Any, Final, NamedTuple from litellm._logging import verbose_logger from litellm.responses.mcp.request_context import MCPRequestContext +from litellm.router_utils.mcp_tool_execution import MCPToolReplayGuard from litellm.types.llms.anthropic import ( AnthropicMessagesTool, AnthropicMessagesToolResultParam, @@ -149,6 +150,7 @@ async def anthropic_messages_with_mcp( messages=list(working_messages), stream=False, **loop_call_args ) + replay_guard: Final = MCPToolReplayGuard() for _ in range(MAX_MCP_TOOL_USE_ITERATIONS): if _get_stop_reason(response) != "tool_use": break @@ -170,6 +172,7 @@ async def anthropic_messages_with_mcp( litellm_trace_id=context.litellm_trace_id, request_tags=list(context.request_tags) if context.request_tags else None, guardrail_context=context.guardrail_context, + on_tool_dispatched=replay_guard.record_tool_dispatch, ) # Every tool call was skipped, so there is nothing to feed back; a @@ -182,9 +185,10 @@ async def anthropic_messages_with_mcp( {"role": "assistant", "content": list(_get_response_content(response))}, _build_tool_result_message(tool_results), ) - response = await _AnthropicMessagesCall(fn=litellm.anthropic_messages).fn( - messages=list(working_messages), stream=False, **loop_call_args - ) + with replay_guard: + response = await _AnthropicMessagesCall(fn=litellm.anthropic_messages).fn( + messages=list(working_messages), stream=False, **loop_call_args + ) else: verbose_logger.warning( "MCP tool loop hit its %s iteration cap for model %s; returning the last response", @@ -192,11 +196,12 @@ async def anthropic_messages_with_mcp( model, ) - client_response: Final = ( - await with_safeguard_results(response, request_safeguards_evaluator(kwargs, working_messages)) - if classified_here - else response - ) + with replay_guard: + client_response: Final = ( + await with_safeguard_results(response, request_safeguards_evaluator(kwargs, working_messages)) + if classified_here + else response + ) if stream: from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, diff --git a/litellm/mcp_client/client.py b/litellm/mcp_client/client.py index 107b7155303..fd84342ea1c 100644 --- a/litellm/mcp_client/client.py +++ b/litellm/mcp_client/client.py @@ -1146,6 +1146,7 @@ class MCPClient: raise_on_error: bool = False, allow_input_required: bool = False, persistent_session: "PersistentMCPSession | None" = None, + on_dispatch: Callable[[], None] | None = None, ) -> MCPCallToolResult | InputRequiredResult: """ Call an MCP Tool. @@ -1162,6 +1163,8 @@ class MCPClient: allow_input_required: When True, a 2026-07-28 upstream may answer with an interim ``InputRequiredResult`` and it is returned as is. The SDK rejects it otherwise, so callers only opt in when the downstream side can carry it. + on_dispatch: Called right before the tool call is sent on a ready session, so a caller + can tell a call the server may have run from one that never left. """ verbose_logger.info("MCP client calling tool '%s'", call_tool_request_params.name) @@ -1195,6 +1198,8 @@ class MCPClient: async def request( responses: InputResponses | None, state: str | None ) -> MCPCallToolResult | InputRequiredResult: + if on_dispatch is not None: + on_dispatch() return await session.call_tool( name=call_tool_request_params.name, arguments=call_tool_request_params.arguments, diff --git a/litellm/proxy/mcp_server/mcp_server_manager.py b/litellm/proxy/mcp_server/mcp_server_manager.py index eacba25257e..aa537241f4b 100644 --- a/litellm/proxy/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/mcp_server/mcp_server_manager.py @@ -6540,6 +6540,7 @@ class MCPServerManager: client_ip: str | None = None, allow_input_required: bool = False, persistent_session: PersistentMCPSession | None = None, + on_dispatch: Callable[[], None] | None = None, ) -> CallToolResult | InputRequiredResult: """Call a token_exchange (OBO) tool; on an upstream 401/403 re-mint the token once and retry. @@ -6556,6 +6557,7 @@ class MCPServerManager: raise_on_error=True, allow_input_required=allow_input_required, persistent_session=persistent_session, + on_dispatch=on_dispatch, ) except Exception as exc: auth_failure: Final = _extract_upstream_auth_failure(exc) @@ -6583,6 +6585,7 @@ class MCPServerManager: host_progress_callback=host_progress_callback, allow_input_required=allow_input_required, persistent_session=retry_persistent_session, + on_dispatch=on_dispatch, ) async def _call_regular_mcp_tool( @@ -6603,6 +6606,7 @@ class MCPServerManager: allow_input_required: bool = False, input_responses: InputResponses | None = None, request_state: str | None = None, + on_dispatch: Callable[[], None] | None = None, ) -> CallToolResult | InputRequiredResult: """ Call a regular MCP tool using the MCP client. @@ -6759,6 +6763,7 @@ class MCPServerManager: client_ip=client_ip, allow_input_required=allow_input_required, persistent_session=persistent_session, + on_dispatch=on_dispatch, ) else: # Scoped to the two client-forwarded token modes this stack introduced; legacy @@ -6776,6 +6781,7 @@ class MCPServerManager: host_progress_callback=host_progress_callback, allow_input_required=allow_input_required, persistent_session=persistent_session, + on_dispatch=on_dispatch, ) # The client-forwarded modes carry the caller's own upstream token, so an upstream # 401 (expired/invalid token) is the caller's to resolve: relay it as @@ -6793,6 +6799,7 @@ class MCPServerManager: raise_on_error=True, allow_input_required=allow_input_required, persistent_session=persistent_session, + on_dispatch=on_dispatch, ) except Exception as e: required_scope: Final = upstream_insufficient_scope(e) if managed_per_user else None @@ -7072,6 +7079,7 @@ class MCPServerManager: wire_compat: WireCompat = WireCompat.LEGACY, input_responses: InputResponses | None = None, request_state: str | None = None, + on_dispatch: Callable[[], None] | None = None, *, guardrail_auth: UserAPIKeyAuth | None = None, catalog_auth_header: str | None | EllipsisType = ..., @@ -7094,6 +7102,8 @@ class MCPServerManager: litellm_logging_obj: Optional request logger the guardrail hooks record their evaluations onto, so MCP guardrail activity reaches the Guardrails Monitor. See ``pre_call_tool_check`` + on_dispatch: Optional callback run right before the call is sent to the server, + after its headers and client are ready, even if the server never returns a result Returns: @@ -7214,6 +7224,8 @@ class MCPServerManager: resolved_token: Final = request_resolved_auth_headers.set(resolved_auth_headers) try: async with self._limit_outbound_concurrency(mcp_server): + if on_dispatch is not None: + on_dispatch() return await self._call_openapi_tool_handler(mcp_server, name, arguments, wire_compat) finally: request_auth_header.reset(auth_token) @@ -7239,6 +7251,7 @@ class MCPServerManager: allow_input_required=wire_compat is WireCompat.MODERN, input_responses=input_responses, request_state=request_state, + on_dispatch=on_dispatch, ) return await self._gather_openapi_tool_tasks(tasks, proxy_logging_obj) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index ad1d35bbe17..f4382aeb9fc 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -68,6 +68,7 @@ from litellm.responses.mcp.round_billing import ( ) from litellm.responses.mcp.tool_precheck import may_name_litellm_mcp_gateway from litellm.responses.utils import ResponsesAPIRequestUtils +from litellm.router_utils.mcp_tool_execution import MCPToolReplayGuard from litellm.types.llms.openai import ( PromptObject, Reasoning, @@ -472,6 +473,7 @@ async def aresponses_api_with_mcp( ) tool_trace_id: Final = _request_trace_id(request_kwargs) + replay_guard: Final = MCPToolReplayGuard() tool_results: Final = await LiteLLM_Proxy_MCP_Handler.execute_tool_calls( tool_server_map=tool_server_map, served_tools=original_mcp_tools, @@ -488,6 +490,7 @@ async def aresponses_api_with_mcp( MappingProxyType({**kwargs, "metadata": metadata, "model": model}) ), request_trace_id=_langfuse_request_trace_id(request_kwargs, tool_trace_id), + on_tool_dispatched=replay_guard.record_tool_dispatch, ) if tool_results: @@ -514,8 +517,11 @@ async def aresponses_api_with_mcp( tool_calls=tool_calls, tool_results=tool_results ) - with internal_sub_call( - logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObj) else None + with ( + internal_sub_call( + logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObj) else None + ), + replay_guard, ): final_response = await LiteLLM_Proxy_MCP_Handler.make_follow_up_call( follow_up_input=follow_up_input, diff --git a/litellm/responses/mcp/chat_completions_handler.py b/litellm/responses/mcp/chat_completions_handler.py index f8a52227d1c..bc7b71b0e0f 100644 --- a/litellm/responses/mcp/chat_completions_handler.py +++ b/litellm/responses/mcp/chat_completions_handler.py @@ -22,6 +22,7 @@ from litellm.responses.mcp.round_billing import ( first_round_billed_on_failure, summed_cost_breakdown, ) +from litellm.router_utils.mcp_tool_execution import MCPToolReplayGuard from litellm.types.utils import Message, ModelResponse, Usage from litellm.utils import CustomStreamWrapper @@ -261,6 +262,7 @@ async def acompletion_with_mcp( self.follow_up_stream = None self.follow_up_iterator = None self.follow_up_exhausted = False + self.replay_guard = MCPToolReplayGuard() self._round_billing: ChatStreamRoundBilling | None = engaged_round_billing(initial_stream.logging_obj) self._unsent_usage_chunk: ModelResponseStream | None = None @@ -354,7 +356,8 @@ async def acompletion_with_mcp( async def _follow_up_or_release_first_round(self, usage_chunk: ModelResponseStream | None) -> None: if self.tool_results and self.complete_response: - await self._prepare_follow_up_call() + with self.replay_guard: + await self._prepare_follow_up_call() if self.follow_up_stream is not None: return self._unsent_usage_chunk = usage_chunk @@ -438,7 +441,8 @@ async def acompletion_with_mcp( verbose_logger.debug("Follow-up stream iterator created") try: - chunk = await self.follow_up_iterator.__anext__() + with self.replay_guard: + chunk = await self.follow_up_iterator.__anext__() from litellm._logging import verbose_logger verbose_logger.debug("Follow-up chunk yielded: %s", chunk) @@ -509,6 +513,7 @@ async def acompletion_with_mcp( litellm_trace_id=self.litellm_trace_id, request_tags=self.request_tags, guardrail_context=context.guardrail_context, + on_tool_dispatched=self.replay_guard.record_tool_dispatch, ) async def _prepare_follow_up_call(self): @@ -679,6 +684,7 @@ async def acompletion_with_mcp( ) return initial_response + replay_guard: Final = MCPToolReplayGuard() with first_round_billed_on_failure(logging_obj, initial_response): tool_results: Final = await LiteLLM_Proxy_MCP_Handler.execute_tool_calls( tool_server_map=tool_server_map, @@ -693,6 +699,7 @@ async def acompletion_with_mcp( litellm_trace_id=context.litellm_trace_id, request_tags=request_tags, guardrail_context=context.guardrail_context, + on_tool_dispatched=replay_guard.record_tool_dispatch, ) if not tool_results: @@ -713,7 +720,10 @@ async def acompletion_with_mcp( follow_up_call_args["messages"] = follow_up_messages follow_up_call_args["stream"] = stream - with internal_sub_call(logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObj) else None): + with ( + internal_sub_call(logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObj) else None), + replay_guard, + ): final_response: Final = await litellm_acompletion(**follow_up_call_args) if not isinstance(final_response, ModelResponse): diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index 1104aee353e..8d057f5864f 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -740,8 +740,14 @@ class LiteLLM_Proxy_MCP_Handler: guardrail_context: Mapping[str, object] | None = None, served_tools: Sequence[MCPTool] | None = None, request_trace_id: str | None = None, + on_tool_dispatched: Callable[[], None] | None = None, ) -> list[MCPToolResult]: - """Execute tool calls and return results.""" + """Execute tool calls and return results. + + ``on_tool_dispatched`` runs for every call that is sent to its MCP server, even one that then + times out. Calls rejected or failing before that, for example on a pre-call guardrail or a + credential the caller has not set, do not trigger it. + """ from fastapi import HTTPException from litellm._uuid import uuid @@ -908,6 +914,7 @@ class LiteLLM_Proxy_MCP_Handler: proxy_logging_obj=proxy_logging_obj, litellm_logging_obj=litellm_logging_obj, guardrail_context=guardrail_context, + on_dispatch=on_tool_dispatched, listed_tool=( next((tool for tool in served_tools if tool.name == tool_name), None) if served_tools is not None diff --git a/litellm/router_python_core.py b/litellm/router_python_core.py index 2235abc9897..bdd94660f01 100644 --- a/litellm/router_python_core.py +++ b/litellm/router_python_core.py @@ -366,6 +366,7 @@ from litellm.router_utils.kubernetes_pod_discovery import ( resolve_pods_after, resolve_pods_after_bound, ) +from litellm.router_utils.mcp_tool_execution import mcp_tools_executed from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( MID_STREAM_CONTINUATION_MARKER, RESPONSES_CONTINUATION_KWARG, @@ -9135,6 +9136,7 @@ class PythonCore(DeprecatedRouterMembers): or original_model_group is None or is_guardrail_intervention(e) or unsupported_params_without_fallback + or mcp_tools_executed(e) ): raise e @@ -9559,7 +9561,7 @@ class PythonCore(DeprecatedRouterMembers): response = add_retry_headers_to_response(response=response, attempted_retries=0, max_retries=None) return response except Exception as e: - if is_guardrail_intervention(e): + if is_guardrail_intervention(e) or mcp_tools_executed(e): raise current_attempt = None original_exception = e @@ -9670,6 +9672,8 @@ class PythonCore(DeprecatedRouterMembers): # Always track the latest error so we raise the most # recent exception instead of the first one. original_exception = e + if mcp_tools_executed(e): + raise ## LOGGING kwargs = self._core_log_retry(kwargs=kwargs, e=e) diff --git a/litellm/router_utils/mcp_tool_execution.py b/litellm/router_utils/mcp_tool_execution.py new file mode 100644 index 00000000000..15f29fd1520 --- /dev/null +++ b/litellm/router_utils/mcp_tool_execution.py @@ -0,0 +1,62 @@ +from collections.abc import Iterator +from types import TracebackType +from typing import Final + +_MCP_TOOLS_EXECUTED_ATTR: Final = "_litellm_mcp_tools_executed" + +_MAX_EXCEPTION_CHAIN_DEPTH: Final = 20 + + +def mark_mcp_tools_executed(exception: BaseException) -> None: + """Tag an error raised after an MCP gateway loop already executed tool calls. + + Retrying or falling back re-runs the whole loop, which executes those tool calls + again, so the router surfaces a tagged error instead of replaying the request. The + exception is tagged rather than wrapped so the client sees the same error. + """ + setattr(exception, _MCP_TOOLS_EXECUTED_ATTR, True) + + +def _exception_chain(exception: BaseException) -> Iterator[BaseException]: + current = exception # rebind-ok: advances one link per iteration of the bounded walk + for _ in range(_MAX_EXCEPTION_CHAIN_DEPTH): + yield current + following = current.__cause__ or current.__context__ + if following is None: + return + current = following + + +def mcp_tools_executed(exception: BaseException | None) -> bool: + """Whether ``exception`` or an exception in its chain was tagged by ``mark_mcp_tools_executed``. + + The chain matters because a caller can re-raise a tagged error as a new exception, as + ``exception_type`` does for anything that is not already a LiteLLM exception. + """ + return exception is not None and any( + getattr(link, _MCP_TOOLS_EXECUTED_ATTR, False) is True for link in _exception_chain(exception) + ) + + +class MCPToolReplayGuard: + """Tags errors raised inside ``with guard:`` once the request has sent an MCP tool call to a server.""" + + __slots__ = ("_tool_dispatched",) + + def __init__(self) -> None: + self._tool_dispatched: bool = False + + def record_tool_dispatch(self) -> None: + self._tool_dispatched = True + + def __enter__(self) -> None: + return None + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + traceback: TracebackType | None, + ) -> None: + if self._tool_dispatched and isinstance(exc, Exception): + mark_mcp_tools_executed(exc) diff --git a/tests/unit/mcp_client/test_mcp_client.py b/tests/unit/mcp_client/test_mcp_client.py index 43021d248a9..6acd2d48f8d 100644 --- a/tests/unit/mcp_client/test_mcp_client.py +++ b/tests/unit/mcp_client/test_mcp_client.py @@ -1035,6 +1035,36 @@ async def test_call_tool_does_not_log_arguments(): assert "gpt-5-mini" not in logged +@pytest.mark.asyncio +@pytest.mark.parametrize("session_opens", [True, False]) +async def test_call_tool_reports_dispatch_only_when_the_call_is_sent(session_opens): + """ + on_dispatch marks the moment a tool call can reach the server, so it fires right before the + session sends the call and never when the session could not be opened. + """ + from mcp.types import CallToolRequestParams + + events: list[str] = [] + + async def send(**_): + events.append("sent") + return MagicMock() + + async def run_with_session(operation, quiet_on_error=False): + if not session_opens: + raise ConnectionError("cannot reach the server") + return await operation(MagicMock(call_tool=send)) + + client = MCPClient(server_url="http://test-server") + client.run_with_session = run_with_session + + await client.call_tool( + CallToolRequestParams(name="search_tool", arguments={}), on_dispatch=lambda: events.append("dispatched") + ) + + assert events == (["dispatched", "sent"] if session_opens else []) + + @pytest.mark.asyncio async def test_get_prompt_does_not_log_arguments(): from mcp.types import GetPromptRequestParams diff --git a/tests/unit/proxy/mcp_server/test_mcp_max_concurrent_requests.py b/tests/unit/proxy/mcp_server/test_mcp_max_concurrent_requests.py index f1896757874..fd3357e1343 100644 --- a/tests/unit/proxy/mcp_server/test_mcp_max_concurrent_requests.py +++ b/tests/unit/proxy/mcp_server/test_mcp_max_concurrent_requests.py @@ -67,8 +67,9 @@ class _AdmissionProbeClient(MCPClient): raise_on_error: bool = False, allow_input_required: bool = False, persistent_session: PersistentMCPSession | None = None, + on_dispatch: Callable[[], None] | None = None, ) -> CallToolResult: - del call_tool_request_params, host_progress_callback, raise_on_error, allow_input_required + del call_tool_request_params, host_progress_callback, raise_on_error, allow_input_required, on_dispatch if self._submitted is not None: self._submitted.set() if persistent_session is None: @@ -121,6 +122,7 @@ def _patch_client_with_tracker(manager: MCPServerManager, tracker: _ConcurrencyT host_progress_callback=None, allow_input_required=False, persistent_session: PersistentMCPSession | None = None, + on_dispatch=None, ): tracker.enter(server.server_id) try: diff --git a/tests/unit/proxy/mcp_server/test_mcp_server_manager.py b/tests/unit/proxy/mcp_server/test_mcp_server_manager.py index 13e4704bd90..5d6b8ca43ba 100644 --- a/tests/unit/proxy/mcp_server/test_mcp_server_manager.py +++ b/tests/unit/proxy/mcp_server/test_mcp_server_manager.py @@ -7644,6 +7644,7 @@ class TestMCPServerManager: host_progress_callback=None, allow_input_required=False, persistent_session: PersistentMCPSession | None = None, + on_dispatch=None, ): # Return a mock CallToolResult result = MagicMock(spec=CallToolResult) @@ -7682,6 +7683,30 @@ class TestMCPServerManager: # Verify the MCP client call was awaited exactly once assert mock_client.call_tool.await_count == 1 + @pytest.mark.asyncio + async def test_call_tool_hands_on_dispatch_to_the_mcp_client(self): + """The MCP client fires on_dispatch when it sends the call, so call_tool must hand it down.""" + from mcp.types import CallToolResult + + auth = self._unrestricted_auth() + manager, proxy_logging_obj = self._manager_ready_for_call_tool( + [MCPTool(name="test_tool", inputSchema={"type": "object"})], + caller=ListedToolsCaller(user_api_key_auth=auth), + ) + manager.create_mcp_client = AsyncMock(return_value=_RetryFakeClient(result=CallToolResult(content=[]))) + dispatched = MagicMock() + + await manager.call_tool( + server_name="test-server", + name="test_tool", + arguments={}, + user_api_key_auth=auth, + proxy_logging_obj=proxy_logging_obj, + on_dispatch=dispatched, + ) + + dispatched.assert_called_once_with() + @staticmethod def _manager_ready_for_call_tool( listed_tools: list[MCPTool], caller: ListedToolsCaller | None = None @@ -12179,9 +12204,12 @@ class _RetryFakeClient: raise_on_error=False, allow_input_required=False, persistent_session: PersistentMCPSession | None = None, + on_dispatch=None, ): self.attempts += 1 self.persistent_sessions.append(persistent_session) + if on_dispatch is not None: + on_dispatch() if self._raises is not None: if raise_on_error: raise self._raises @@ -12319,6 +12347,56 @@ class TestOBOCallToolRetry: manager._cred_provider.invalidate_credentials.assert_awaited_once() assert first.attempts == 1 and retry.attempts == 1 + @pytest.mark.asyncio + @pytest.mark.parametrize("path", ["plain", "relay", "obo_retry"]) + async def test_call_regular_reports_every_sent_attempt_through_on_dispatch(self, path): + """on_dispatch tells the caller a tool may have run, so every attempt that is sent must reach + it: once on the plain and relay paths, twice when the OBO path retries after an upstream 401.""" + manager = self._manager() + success = CallToolResult(content=[], isError=False) + clients = [_RetryFakeClient(raises=_UpstreamAuthError(401)), _RetryFakeClient(result=success)] + manager.create_mcp_client = AsyncMock(side_effect=clients if path == "obo_retry" else clients[1:]) + servers = { + "plain": MCPServer( + server_id="plain-srv", name="plain", url="https://upstream.example/mcp", transport=MCPTransport.sse + ), + "relay": MCPServer( + server_id="relay-srv", + name="relay", + url="https://upstream.example/mcp", + transport=MCPTransport.sse, + auth_type=MCPAuth.true_passthrough, + ), + "obo_retry": MCPServer( + server_id="id-jag-srv", + name="id-jag", + url="https://upstream.example/mcp", + transport=MCPTransport.sse, + auth_type=MCPAuth.oauth2_id_jag, + client_id="gateway-client", + client_secret="gateway-secret", + token_exchange_endpoint="https://org-idp.example/oauth2/token", + id_jag_resource_token_endpoint="https://resource-as.example/oauth2/token", + ), + } + dispatched = MagicMock() + + result = await manager._call_regular_mcp_tool( + mcp_server=servers[path], + original_tool_name="tool", + arguments={}, + tasks=[], + mcp_auth_header=None, + mcp_server_auth_headers=None, + oauth2_headers={"Authorization": "Bearer caller-id-token"} if path == "obo_retry" else None, + raw_headers={"authorization": "Bearer caller-token"} if path == "relay" else None, + proxy_logging_obj=None, + on_dispatch=dispatched, + ) + + assert result is success + assert dispatched.call_count == (2 if path == "obo_retry" else 1) + @pytest.mark.asyncio async def test_non_auth_error_does_not_retry(self): manager = self._manager() @@ -12464,6 +12542,7 @@ class TestOBOConcurrencyLimit: raise_on_error=False, allow_input_required=False, persistent_session: PersistentMCPSession | None = None, + on_dispatch=None, ): inflight["current"] += 1 inflight["peak"] = max(inflight["peak"], inflight["current"]) @@ -12548,6 +12627,7 @@ class TestOBOConcurrencyLimit: raise_on_error: bool = False, allow_input_required: bool = False, persistent_session: PersistentMCPSession | None = None, + on_dispatch: Callable[[], None] | None = None, ) -> CallToolResult: sessions_seen.append(persistent_session) return CallToolResult(content=[], isError=False) diff --git a/tests/unit/test_router_retry_non_retryable_errors.py b/tests/unit/test_router_retry_non_retryable_errors.py index 98a7db7a079..65f32773fea 100644 --- a/tests/unit/test_router_retry_non_retryable_errors.py +++ b/tests/unit/test_router_retry_non_retryable_errors.py @@ -12,14 +12,19 @@ Regression tests for https://github.com/BerriAI/litellm/issues/21343 import asyncio import datetime -from collections.abc import Awaitable, Callable -from typing import Final +import json +from collections.abc import AsyncIterator, Awaitable, Callable +from typing import TYPE_CHECKING, Final, NamedTuple from unittest.mock import AsyncMock, patch +import httpx import pytest +import respx import litellm from litellm import Router +from litellm.exceptions import GuardrailRaisedException +from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( _union_duration_ms, @@ -27,8 +32,14 @@ from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( ) from litellm.litellm_core_utils.logging_utils import track_llm_api_timing from litellm.litellm_core_utils.rules import Rules +from litellm.router_utils.mcp_tool_execution import mark_mcp_tools_executed +from litellm.types.router import RetryPolicy from litellm.utils import function_setup +if TYPE_CHECKING: + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy._types import UserAPIKeyAuth + def _make_rate_limit_error(message="Rate limited"): """Create a RateLimitError for testing.""" @@ -66,7 +77,7 @@ def _make_not_found_error(message="Model not found"): ) -def _create_router(num_retries=2): +def _create_router(num_retries=2, retry_policy=None): """Create a Router with two deployments for testing.""" return Router( model_list=[ @@ -86,6 +97,7 @@ def _create_router(num_retries=2): }, ], num_retries=num_retries, + retry_policy=retry_policy, ) @@ -357,3 +369,512 @@ async def test_retry_attempts_accumulate_timing_in_shared_request_metadata(): assert timing_metrics["litellm_overhead_time_ms"] == pytest.approx( round(total_response_time_ms - union_duration_ms, 4) ) + + +def _make_follow_up_error(tools_executed: bool) -> litellm.InternalServerError: + error: Final = litellm.InternalServerError(message="follow-up failed", llm_provider="openai", model="gpt-4") + if tools_executed: + mark_mcp_tools_executed(error) + return error + + +@pytest.mark.asyncio +@pytest.mark.parametrize("retry_policy", [None, RetryPolicy(InternalServerErrorRetries=2)]) +@pytest.mark.parametrize("tools_executed", [True, False]) +async def test_500_after_mcp_tool_execution_is_not_retried(retry_policy, tools_executed): + """ + Regression test for #43153: a retry re-runs the whole MCP gateway loop, so a + failure raised after the loop executed tool calls must surface instead of + executing those tools again. A retry policy that retries 500s must not override + that, and a 500 raised before any tool ran must still be retried. + """ + router: Final = _create_router(num_retries=2, retry_policy=retry_policy) + error: Final = _make_follow_up_error(tools_executed) + attempt: Final = AsyncMock(side_effect=error) + + with pytest.raises(litellm.InternalServerError) as exc_info: + await router.async_function_with_retries(**{**_base_kwargs(), "original_function": attempt, "num_retries": 2}) + + assert exc_info.value is error + assert attempt.await_count == (1 if tools_executed else 3) + + +@pytest.mark.asyncio +async def test_500_after_mcp_tool_execution_stops_an_ongoing_retry_loop(): + """ + The first attempt fails before any tool ran and is retried; the retry executes + the tools and then fails, so the loop must stop there instead of running them + a second time. + """ + router: Final = _create_router(num_retries=3) + after_tools: Final = _make_follow_up_error(tools_executed=True) + attempt: Final = AsyncMock(side_effect=[_make_follow_up_error(tools_executed=False), after_tools]) + + with pytest.raises(litellm.InternalServerError) as exc_info: + await router.async_function_with_retries(**{**_base_kwargs(), "original_function": attempt, "num_retries": 3}) + + assert exc_info.value is after_tools + assert attempt.await_count == 2 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("tools_executed", [True, False]) +async def test_500_after_mcp_tool_execution_does_not_fall_back(tools_executed): + """ + A fallback re-runs the whole MCP gateway loop on the fallback group, so it must + not start once the primary attempt executed tool calls. + """ + router: Final = Router( + model_list=[ + {"model_name": "test-model", "litellm_params": {"model": "openai/gpt-4", "api_key": "fake-key-1"}}, + {"model_name": "backup-model", "litellm_params": {"model": "openai/gpt-4", "api_key": "fake-key-2"}}, + ], + num_retries=0, + fallbacks=[{"test-model": ["backup-model"]}], + ) + error: Final = _make_follow_up_error(tools_executed) + fallback_response: Final = litellm.ModelResponse() + + async def primary_fails(**kwargs): + if kwargs["model"] == "test-model": + raise error + return fallback_response + + attempt: Final = AsyncMock(side_effect=primary_fails) + call: Final = router.async_function_with_fallbacks( + **{**_base_kwargs(), "original_function": attempt, "num_retries": 0} + ) + + if tools_executed: + with pytest.raises(litellm.InternalServerError) as exc_info: + await call + assert exc_info.value is error + else: + assert await call is fallback_response + + attempted_models: Final = [awaited.kwargs["model"] for awaited in attempt.await_args_list] + assert attempted_models == (["test-model"] if tools_executed else ["test-model", "backup-model"]) + + +_UPSTREAM: Final = "https://upstream.example/v1" +_TOOL_NAME: Final = "ledger-commit_write" + + +def _ledger_tools() -> list[dict[str, str]]: + return [{"type": "mcp", "server_url": "litellm_proxy/ledger", "require_approval": "never"}] + + +def _ledger_caller() -> "UserAPIKeyAuth": + from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth + + return UserAPIKeyAuth( + object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="test", mcp_servers=["ledger"]) + ) + + +class _LedgerGateway(NamedTuple): + tool: AsyncMock + list_tools: AsyncMock + manager: "MCPServerManager" + + +@pytest.fixture +def ledger_gateway(monkeypatch: pytest.MonkeyPatch) -> _LedgerGateway: + """Serve one auto-approved MCP tool through the real MCP manager and tool registry.""" + from mcp.types import Tool + + from litellm.caching.caching import DualCache + from litellm.proxy import proxy_server + from litellm.proxy.mcp_server import mcp_server_manager, server, tool_registry + from litellm.proxy.mcp_server.faults.list_outcomes import AggregateToolListing + from litellm.proxy.utils import ProxyLogging + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + manager: Final = mcp_server_manager.MCPServerManager() + manager.registry = { + "ledger": MCPServer( + server_id="ledger", + name="ledger", + server_name="ledger", + transport="http", + url="https://ledger.example/mcp", + spec_path="ledger.json", + auth_type="none", + ) + } + manager.tool_name_to_mcp_server_name_mapping = {_TOOL_NAME: "ledger"} + tool: Final = AsyncMock(return_value={"written": True}) + registry: Final = tool_registry.MCPToolRegistry() + registry.register_tool(_TOOL_NAME, "Write one ledger entry", {"type": "object"}, tool) + list_tools: Final = AsyncMock( + return_value=AggregateToolListing(tools=[Tool(name=_TOOL_NAME, inputSchema={"type": "object"})], outcomes={}) + ) + monkeypatch.setattr(tool_registry, "global_mcp_tool_registry", registry) + monkeypatch.setattr(mcp_server_manager, "global_mcp_server_manager", manager) + monkeypatch.setattr(proxy_server, "proxy_logging_obj", ProxyLogging(user_api_key_cache=DualCache())) + monkeypatch.setattr(server, "_get_tools_from_mcp_servers", list_tools) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + return _LedgerGateway(tool=tool, list_tools=list_tools, manager=manager) + + +def _is_follow_up(request: httpx.Request) -> bool: + body: Final = json.loads(request.content) + messages: Final = body.get("messages") or [] + items: Final = body["input"] if isinstance(body.get("input"), list) else [] + return any(message.get("role") == "tool" for message in messages) or any( + item.get("type") == "function_call_output" for item in items + ) + + +def _chat_completion(message: dict[str, object], finish_reason: str) -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 0, + "model": "gpt-4.1-mini", + "choices": [{"index": 0, "message": message, "finish_reason": finish_reason}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + + +def _responses_api_response(output_item: dict[str, object]) -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": "resp_1", + "object": "response", + "created_at": 0, + "status": "completed", + "model": "gpt-4.1-mini", + "output": [output_item], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + }, + ) + + +def _tool_call(responses_api: bool) -> httpx.Response: + arguments: Final = json.dumps({"entry": "alpha"}) + if responses_api: + return _responses_api_response( + {"type": "function_call", "id": "fc_1", "call_id": "call_1", "name": _TOOL_NAME, "arguments": arguments} + ) + return _chat_completion( + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": _TOOL_NAME, "arguments": arguments}} + ], + }, + "tool_calls", + ) + + +def _final_answer(responses_api: bool) -> httpx.Response: + if responses_api: + return _responses_api_response( + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "Recorded alpha", "annotations": []}], + } + ) + return _chat_completion({"role": "assistant", "content": "Recorded alpha"}, "stop") + + +class _FakeModel: + """Answers both OpenAI routes: a tool call, then a final answer once the tool result arrives.""" + + def __init__(self, failing_initial_calls: int = 0, failing_follow_ups: int = 0) -> None: + self.calls: list[str] = [] + self._failures: Final = {"initial": failing_initial_calls, "follow_up": failing_follow_ups} + + def __call__(self, request: httpx.Request) -> httpx.Response: + kind: Final = "follow_up" if _is_follow_up(request) else "initial" + self.calls.append(kind) + if self.calls.count(kind) <= self._failures[kind]: + return httpx.Response(500, json={"error": {"message": "upstream failed", "type": "server_error"}}) + responses_api: Final = request.url.path.endswith("/responses") + return _tool_call(responses_api) if kind == "initial" else _final_answer(responses_api) + + +class _StreamBrokenBeforeFirstChunk(httpx.AsyncByteStream): + async def __aiter__(self) -> AsyncIterator[bytes]: + raise httpx.ReadError("connection dropped before the first chunk") + yield b"" + + +def _chat_tool_call_stream() -> httpx.Response: + tool_call: Final = { + "index": 0, + "id": "call_1", + "type": "function", + "function": {"name": _TOOL_NAME, "arguments": json.dumps({"entry": "alpha"})}, + } + chunks: Final = ( + {"role": "assistant", "content": None, "tool_calls": [tool_call]}, + {}, + ) + body: Final = "".join( + "data: " + + json.dumps( + { + "id": "chatcmpl-1", + "object": "chat.completion.chunk", + "created": 0, + "model": "gpt-4.1-mini", + "choices": [{"index": 0, "delta": delta, "finish_reason": None if delta else "tool_calls"}], + } + ) + + "\n\n" + for delta in chunks + ) + return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=f"{body}data: [DONE]\n\n") + + +def _serve(model: _FakeModel) -> None: + respx.post(f"{_UPSTREAM}/chat/completions").mock(side_effect=model) + respx.post(f"{_UPSTREAM}/responses").mock(side_effect=model) + + +def _mcp_router(num_retries: int, fallbacks: list[dict[str, list[str]]] | None = None) -> litellm.Router: + deployment: Final = {"model": "openai/gpt-4.1-mini", "api_key": "sk-test", "api_base": _UPSTREAM} + return litellm.Router( + model_list=[ + {"model_name": "repro-model", "litellm_params": deployment}, + {"model_name": "repro-model", "litellm_params": deployment}, + {"model_name": "backup-model", "litellm_params": deployment}, + ], + num_retries=num_retries, + fallbacks=fallbacks, + ) + + +def _chat(router: litellm.Router) -> Awaitable[object]: + return router.acompletion( + model="repro-model", + messages=[{"role": "user", "content": "Record entry alpha"}], + tools=_ledger_tools(), + metadata={"user_api_key_auth": _ledger_caller()}, + ) + + +def _responses(router: litellm.Router) -> Awaitable[object]: + return router.aresponses( + model="repro-model", + input="Record entry alpha", + tools=_ledger_tools(), + litellm_metadata={"user_api_key_auth": _ledger_caller()}, + ) + + +def _messages(router: litellm.Router) -> Awaitable[object]: + return router.aanthropic_messages( + model="repro-model", + max_tokens=64, + messages=[{"role": "user", "content": "Record entry alpha"}], + tools=_ledger_tools(), + litellm_metadata={"user_api_key_auth": _ledger_caller()}, + ) + + +_ENDPOINTS: Final = pytest.mark.parametrize( + "send", [_chat, _responses, _messages], ids=["chat_completions", "responses", "messages"] +) + + +@pytest.mark.asyncio +@_ENDPOINTS +@pytest.mark.parametrize( + "router_kwargs", + [{"num_retries": 2}, {"num_retries": 0, "fallbacks": [{"repro-model": ["backup-model"]}]}], + ids=["retries", "fallbacks"], +) +@respx.mock +async def test_router_does_not_replay_an_executed_mcp_tool( + ledger_gateway: _LedgerGateway, + send: Callable[[litellm.Router], Awaitable[object]], + router_kwargs: dict[str, object], +): + """ + Regression test for #43153: an auto-approved MCP tool must run once per request. + + Given: The model asks for the tool, the tool runs, and every follow-up model call returns a 500 + When: The router has retries or a fallback group configured + Then: The client gets the 500 after one tool execution and one follow-up call + + A retry or fallback re-runs the whole MCP loop, so before the fix every replay executed + the tool again (three writes with num_retries=2, two with one fallback). + """ + model: Final = _FakeModel(failing_follow_ups=3) + _serve(model) + + with pytest.raises(litellm.InternalServerError): + await send(_mcp_router(**router_kwargs)) + + assert ledger_gateway.tool.await_count == 1 + assert model.calls == ["initial", "follow_up"] + + +@pytest.mark.asyncio +@_ENDPOINTS +@respx.mock +async def test_router_retries_when_a_guardrail_blocked_every_mcp_tool( + ledger_gateway: _LedgerGateway, + send: Callable[[litellm.Router], Awaitable[object]], + monkeypatch: pytest.MonkeyPatch, +): + """ + Given: A pre-call guardrail blocks the tool call, so no tool runs + When: The follow-up model call fails once with a 500 + Then: The router still retries, because replaying the request cannot repeat a side effect + """ + + class BlockEveryToolCall(CustomGuardrail): + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + raise GuardrailRaisedException(message="tool calls are blocked", blocked_content=True) + + monkeypatch.setattr( + litellm, "callbacks", [BlockEveryToolCall(guardrail_name="block", event_hook="pre_mcp_call", default_on=True)] + ) + model: Final = _FakeModel(failing_follow_ups=1) + _serve(model) + + await send(_mcp_router(num_retries=2)) + + assert ledger_gateway.tool.await_count == 0 + assert model.calls == ["initial", "follow_up", "initial", "follow_up"] + + +@pytest.mark.asyncio +@_ENDPOINTS +@respx.mock +async def test_router_does_not_replay_a_tool_call_that_failed_after_it_was_sent( + ledger_gateway: _LedgerGateway, + send: Callable[[litellm.Router], Awaitable[object]], + monkeypatch: pytest.MonkeyPatch, +): + """ + Given: The tool runs, but its call fails afterwards, here because a during-call guardrail + rejects the result, which leaves the router in the same spot as a timeout after the write + When: Every follow-up model call returns a 500 + Then: The router does not replay the request, because the tool may already have written + """ + tool_ran: Final = asyncio.Event() + + def write(**_: object) -> dict[str, bool]: + tool_ran.set() + return {"written": True} + + class RejectToolResult(CustomGuardrail): + async def async_moderation_hook(self, data, user_api_key_dict, call_type): + await tool_ran.wait() + raise GuardrailRaisedException(message="tool result rejected", blocked_content=True) + + ledger_gateway.tool.side_effect = write + monkeypatch.setattr( + litellm, "callbacks", [RejectToolResult(guardrail_name="reject", event_hook="during_mcp_call", default_on=True)] + ) + model: Final = _FakeModel(failing_follow_ups=3) + _serve(model) + + with pytest.raises(litellm.InternalServerError): + await send(_mcp_router(num_retries=2)) + + assert ledger_gateway.tool.await_count == 1 + assert model.calls == ["initial", "follow_up"] + + +@pytest.mark.asyncio +@respx.mock +async def test_router_retries_when_an_mcp_tool_call_never_reached_its_server( + ledger_gateway: _LedgerGateway, + monkeypatch: pytest.MonkeyPatch, +): + """ + Given: The MCP server needs a per-user header the caller has not set, so the tool call fails + before anything is sent to the server + When: The follow-up model call fails once with a 500 + Then: The router still retries, because replaying the request cannot repeat a side effect + """ + registry: Final = ledger_gateway.manager.registry + monkeypatch.setitem( + registry, + "ledger", + registry["ledger"].model_copy( + update={ + "spec_path": None, + "static_headers": {"Authorization": "Bearer ${LEDGER_TOKEN}"}, + "env_vars": [{"name": "LEDGER_TOKEN", "scope": "user"}], + } + ), + ) + model: Final = _FakeModel(failing_follow_ups=1) + _serve(model) + + await _chat(_mcp_router(num_retries=2)) + + assert model.calls == ["initial", "follow_up", "initial", "follow_up"] + + +@pytest.mark.asyncio +@respx.mock +async def test_router_does_not_fall_back_when_a_streamed_follow_up_breaks_after_a_tool_ran( + ledger_gateway: _LedgerGateway, +): + """ + Given: A streaming Chat Completions request whose tool ran + When: The follow-up stream breaks before its first chunk and a fallback group is configured + Then: The client gets the error instead of a fallback that runs the tool again + """ + calls: Final[list[str]] = [] + + def stream_model(request: httpx.Request) -> httpx.Response: + kind: Final = "follow_up" if _is_follow_up(request) else "initial" + calls.append(kind) + if kind == "follow_up": + return httpx.Response( + 200, headers={"content-type": "text/event-stream"}, stream=_StreamBrokenBeforeFirstChunk() + ) + return _chat_tool_call_stream() + + respx.post(f"{_UPSTREAM}/chat/completions").mock(side_effect=stream_model) + stream: Final = await _mcp_router(num_retries=0, fallbacks=[{"repro-model": ["backup-model"]}]).acompletion( + model="repro-model", + messages=[{"role": "user", "content": "Record entry alpha"}], + tools=_ledger_tools(), + metadata={"user_api_key_auth": _ledger_caller()}, + stream=True, + ) + + with pytest.raises(litellm.APIConnectionError): + async for _ in stream: + pass + + assert ledger_gateway.tool.await_count == 1 + assert calls == ["initial", "follow_up"] + + +@pytest.mark.asyncio +@respx.mock +async def test_router_still_retries_a_failure_before_any_mcp_tool_ran(ledger_gateway: _LedgerGateway): + """ + Given: The initial model call fails once with a 500, then asks for the tool + When: The router retries + Then: The retry succeeds and the tool runs once + """ + model: Final = _FakeModel(failing_initial_calls=1) + _serve(model) + + await _chat(_mcp_router(num_retries=2)) + + assert ledger_gateway.tool.await_count == 1 + assert model.calls == ["initial", "initial", "follow_up"] From 04af109ee16f93acb534b8aa3c119d1a03269f40 Mon Sep 17 00:00:00 2001 From: Atharva-Kanherkar Date: Tue, 3 Nov 2026 15:34:07 +0900 Subject: [PATCH 15/15] refactor(mcp): move the replay guard out of router_utils Main's router surface check bans new litellm.router_utils imports from outside the router, and the MCP handlers set the guard. The module now lives in litellm_core_utils, which the router core and the handlers both import Signed-off-by: Atharva-Kanherkar --- .../mcp_tool_replay.py} | 0 litellm/llms/anthropic/messages/native/mcp_handler.py | 2 +- litellm/responses/main.py | 2 +- litellm/responses/mcp/chat_completions_handler.py | 2 +- litellm/router_python_core.py | 2 +- tests/unit/test_router_retry_non_retryable_errors.py | 2 +- 6 files changed, 5 insertions(+), 5 deletions(-) rename litellm/{router_utils/mcp_tool_execution.py => litellm_core_utils/mcp_tool_replay.py} (100%) diff --git a/litellm/router_utils/mcp_tool_execution.py b/litellm/litellm_core_utils/mcp_tool_replay.py similarity index 100% rename from litellm/router_utils/mcp_tool_execution.py rename to litellm/litellm_core_utils/mcp_tool_replay.py diff --git a/litellm/llms/anthropic/messages/native/mcp_handler.py b/litellm/llms/anthropic/messages/native/mcp_handler.py index 32b78766d53..fd50737d64b 100644 --- a/litellm/llms/anthropic/messages/native/mcp_handler.py +++ b/litellm/llms/anthropic/messages/native/mcp_handler.py @@ -12,8 +12,8 @@ from types import MappingProxyType from typing import Any, Final, NamedTuple from litellm._logging import verbose_logger +from litellm.litellm_core_utils.mcp_tool_replay import MCPToolReplayGuard from litellm.responses.mcp.request_context import MCPRequestContext -from litellm.router_utils.mcp_tool_execution import MCPToolReplayGuard from litellm.types.llms.anthropic import ( AnthropicMessagesTool, AnthropicMessagesToolResultParam, diff --git a/litellm/responses/main.py b/litellm/responses/main.py index f4382aeb9fc..d5124ee1709 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -37,6 +37,7 @@ from litellm.litellm_core_utils.get_llm_provider_logic import ( ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup +from litellm.litellm_core_utils.mcp_tool_replay import MCPToolReplayGuard from litellm.litellm_core_utils.prompt_templates.common_utils import ( update_responses_input_with_model_file_ids, update_responses_tools_with_model_file_ids, @@ -68,7 +69,6 @@ from litellm.responses.mcp.round_billing import ( ) from litellm.responses.mcp.tool_precheck import may_name_litellm_mcp_gateway from litellm.responses.utils import ResponsesAPIRequestUtils -from litellm.router_utils.mcp_tool_execution import MCPToolReplayGuard from litellm.types.llms.openai import ( PromptObject, Reasoning, diff --git a/litellm/responses/mcp/chat_completions_handler.py b/litellm/responses/mcp/chat_completions_handler.py index bc7b71b0e0f..9fb412f20d5 100644 --- a/litellm/responses/mcp/chat_completions_handler.py +++ b/litellm/responses/mcp/chat_completions_handler.py @@ -12,6 +12,7 @@ from typing_extensions import TypedDict, Unpack from litellm._internal_context import internal_sub_call from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.mcp_tool_replay import MCPToolReplayGuard from litellm.responses.mcp.chat_stream_round_billing import ChatStreamRoundBilling, engaged_round_billing from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, @@ -22,7 +23,6 @@ from litellm.responses.mcp.round_billing import ( first_round_billed_on_failure, summed_cost_breakdown, ) -from litellm.router_utils.mcp_tool_execution import MCPToolReplayGuard from litellm.types.utils import Message, ModelResponse, Usage from litellm.utils import CustomStreamWrapper diff --git a/litellm/router_python_core.py b/litellm/router_python_core.py index bdd94660f01..55afb6331bb 100644 --- a/litellm/router_python_core.py +++ b/litellm/router_python_core.py @@ -155,6 +155,7 @@ from litellm.litellm_core_utils.hidden_params import ( ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.llm_cost_calc.utils import SERVICE_TIER_COST_KEY_SUFFIXES +from litellm.litellm_core_utils.mcp_tool_replay import mcp_tools_executed from litellm.litellm_core_utils.pattern_match_deployments import PatternMatchRouter from litellm.litellm_core_utils.ptu_pricing import ( PTU_COST_ATTRIBUTION_ENV_VAR, @@ -366,7 +367,6 @@ from litellm.router_utils.kubernetes_pod_discovery import ( resolve_pods_after, resolve_pods_after_bound, ) -from litellm.router_utils.mcp_tool_execution import mcp_tools_executed from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( MID_STREAM_CONTINUATION_MARKER, RESPONSES_CONTINUATION_KWARG, diff --git a/tests/unit/test_router_retry_non_retryable_errors.py b/tests/unit/test_router_retry_non_retryable_errors.py index 65f32773fea..d7ec4290641 100644 --- a/tests/unit/test_router_retry_non_retryable_errors.py +++ b/tests/unit/test_router_retry_non_retryable_errors.py @@ -32,7 +32,7 @@ from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( ) from litellm.litellm_core_utils.logging_utils import track_llm_api_timing from litellm.litellm_core_utils.rules import Rules -from litellm.router_utils.mcp_tool_execution import mark_mcp_tools_executed +from litellm.litellm_core_utils.mcp_tool_replay import mark_mcp_tools_executed from litellm.types.router import RetryPolicy from litellm.utils import function_setup