From 3e4ceef8d583694110120b0be36c8ee594e7ccb2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 22:47:09 -0700 Subject: [PATCH] fix(mcp): give a proxy-rejected follow-up's failure row round 1's own cost breakdown --- litellm/proxy/utils.py | 3 ++ .../mcp/chat_stream_round_billing.py | 6 ++++ .../responses/mcp/parked_stream_dispatch.py | 12 ++++++- .../mcp/test_chat_completions_handler.py | 34 +++++++++++++++++++ 4 files changed, 54 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 0ddfc5ce62f..ecca45cccdd 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -91,6 +91,7 @@ from litellm.proxy.common_utils.openai_error_payload import ( ) from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error from litellm.responses.litellm_completion_transformation.compaction_items import messages_since_newest_compaction +from litellm.responses.mcp.parked_stream_dispatch import held_round_spend from litellm.types.guardrails import GuardrailEventHooks from litellm.types.proxy.model_listing import ModelInfoResponse from litellm.types.utils import MCP_GUARDRAIL_CALL_TYPES, CallTypes, CallTypesLiteral, ModelInfo, Usage @@ -4934,6 +4935,8 @@ class ProxyLogging: if not isinstance(error, _PROXY_ONLY_LLM_API_ERRORS): return False match _args: + case (ModelResponse(), *_) if (held := held_round_spend(logging_obj)) is not None: + logging_obj.record_billed_rounds_for_failure(held) case (ModelResponse() as assembled, *_): logging_obj.record_assembled_response_for_failure(assembled) case (BilledRoundsDispatch() as billed,): diff --git a/litellm/responses/mcp/chat_stream_round_billing.py b/litellm/responses/mcp/chat_stream_round_billing.py index 7623d42e295..370de663606 100644 --- a/litellm/responses/mcp/chat_stream_round_billing.py +++ b/litellm/responses/mcp/chat_stream_round_billing.py @@ -11,6 +11,7 @@ from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend, Uncach from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.responses.mcp.parked_stream_dispatch import ( DISPATCH_RELEASE, + HELD_ROUND_SPEND, IN_FLIGHT_ROUND, PARKED_DISPATCH, SERVED_FROM_CACHE, @@ -114,6 +115,7 @@ class ChatStreamRoundBilling: assembled: Final = _assembled_round(self._peek_parked()) if assembled is not None: self._first = _FirstRound(self._priced(assembled), assembled.parked, assembled.parked) + setattr(self._logging_obj, HELD_ROUND_SPEND, self._held_round_one_spend) def bank_first_round(self) -> BilledRoundsSpend | None: self._bank_apart_from_the_follow_up() @@ -237,6 +239,10 @@ class ChatStreamRoundBilling: first: Final = self._first return first if first is not None and self._holds_round_one_alone(first) else None + def _held_round_one_spend(self) -> BilledRoundsSpend | None: + first: Final = self._first_round_held_alone() + return None if first is None else first.priced.spend + def _holds_round_one_alone(self, first: _FirstRound) -> bool: parked: Final = self._peek_parked() return parked is first.parked or parked is first.logged_alone diff --git a/litellm/responses/mcp/parked_stream_dispatch.py b/litellm/responses/mcp/parked_stream_dispatch.py index 85bbc2d902a..792363f0bab 100644 --- a/litellm/responses/mcp/parked_stream_dispatch.py +++ b/litellm/responses/mcp/parked_stream_dispatch.py @@ -2,12 +2,13 @@ from collections.abc import Coroutine from dataclasses import dataclass from typing import Final, Protocol +from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend, UncachedPrice from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.litellm_core_utils.litellm_logging import UncachedPrice PARKED_DISPATCH: Final = "_deferred_stream_complete_args" DISPATCH_RELEASE: Final = "_on_deferred_stream_complete" IN_FLIGHT_ROUND: Final = "_bill_in_flight_round_on_disconnect" +HELD_ROUND_SPEND: Final = "_held_round_spend_for_failure" SERVED_FROM_CACHE: Final = UncachedPrice(cost=0.0, cost_breakdown=None) @@ -30,6 +31,10 @@ class InFlightRoundBiller(Protocol): def __call__(self) -> Coroutine[object, object, None]: ... +class HeldRoundSpend(Protocol): + def __call__(self) -> BilledRoundsSpend | None: ... + + def success_dispatcher(logger: LiteLLMLoggingObj) -> DispatchesSuccessHandlers: return logger @@ -52,6 +57,11 @@ def in_flight_round_biller(logging_obj: object) -> InFlightRoundBiller | None: return biller if isinstance(logging_obj, LiteLLMLoggingObj) and callable(biller) else None +def held_round_spend(logging_obj: object) -> BilledRoundsSpend | None: + held: Final[HeldRoundSpend | None] = getattr(logging_obj, HELD_ROUND_SPEND, None) + return held() if isinstance(logging_obj, LiteLLMLoggingObj) and callable(held) else None + + def parked_success(parked: tuple[object, ...]) -> ParkedSuccess | None: match parked: case (assembled, cache_hit): diff --git a/tests/unit/responses/mcp/test_chat_completions_handler.py b/tests/unit/responses/mcp/test_chat_completions_handler.py index 4529ab815d3..c0d32742e8a 100644 --- a/tests/unit/responses/mcp/test_chat_completions_handler.py +++ b/tests/unit/responses/mcp/test_chat_completions_handler.py @@ -1933,6 +1933,40 @@ async def test_stream_auto_execute_blocked_by_a_post_call_guardrail_bills_both_r assert recorder.failed[0]["response_cost"] == pytest.approx(charged_once) +@pytest.mark.asyncio +async def test_stream_auto_execute_rejected_by_the_proxy_during_the_follow_up_bills_round_one_with_its_own_breakdown( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + from fastapi import HTTPException + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.utils import ProxyLogging + + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}}) + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_CHAT_ROUND_TWO))) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-rejected-mid-follow-up", stream=True) + rejected: Final = HTTPException(status_code=400, detail="rejected by a stream hook") + + with _echo_tool_served(): + stream: Final = await _open_chat_echo_stream(logging_obj) + ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch( + response=stream, route_type="acompletion", user_api_key_dict=MagicMock(), logging_obj=logging_obj + ) + await _read_until(stream, "PING") + assert ProxyLogging._discard_deferred_stream_logging_for_failure({"litellm_logging_obj": logging_obj}, rejected) + await logging_obj.async_failure_handler(rejected, "rejected by a stream hook") + await recorder.settled(failed=1) + + assert recorder.logged == () + assert len(recorder.failed) == 1 + failed: Final = recorder.failed[0] + breakdown: Final = failed["cost_breakdown"] + assert (failed["prompt_tokens"], failed["completion_tokens"]) == (71, 19) + assert failed["response_cost"] == pytest.approx(_chat_round_cost(_CHAT_ROUND_ONE)) + assert breakdown is not None + assert breakdown["total_cost"] == pytest.approx(failed["response_cost"]) + + @pytest.mark.asyncio async def test_stream_auto_execute_answered_without_a_tool_call_logs_its_one_round_once_the_answer_finishes( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch