fix(mcp): give a proxy-rejected follow-up's failure row round 1's own cost breakdown

This commit is contained in:
mateo-berri 2026-10-29 22:47:09 -07:00
parent d433450ba5
commit 3e4ceef8d5
4 changed files with 54 additions and 1 deletions

View file

@ -91,6 +91,7 @@ from litellm.proxy.common_utils.openai_error_payload import (
)
from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
from litellm.responses.litellm_completion_transformation.compaction_items import messages_since_newest_compaction
from litellm.responses.mcp.parked_stream_dispatch import held_round_spend
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.proxy.model_listing import ModelInfoResponse
from litellm.types.utils import MCP_GUARDRAIL_CALL_TYPES, CallTypes, CallTypesLiteral, ModelInfo, Usage
@ -4934,6 +4935,8 @@ class ProxyLogging:
if not isinstance(error, _PROXY_ONLY_LLM_API_ERRORS):
return False
match _args:
case (ModelResponse(), *_) if (held := held_round_spend(logging_obj)) is not None:
logging_obj.record_billed_rounds_for_failure(held)
case (ModelResponse() as assembled, *_):
logging_obj.record_assembled_response_for_failure(assembled)
case (BilledRoundsDispatch() as billed,):

View file

@ -11,6 +11,7 @@ from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend, Uncach
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.responses.mcp.parked_stream_dispatch import (
DISPATCH_RELEASE,
HELD_ROUND_SPEND,
IN_FLIGHT_ROUND,
PARKED_DISPATCH,
SERVED_FROM_CACHE,
@ -114,6 +115,7 @@ class ChatStreamRoundBilling:
assembled: Final = _assembled_round(self._peek_parked())
if assembled is not None:
self._first = _FirstRound(self._priced(assembled), assembled.parked, assembled.parked)
setattr(self._logging_obj, HELD_ROUND_SPEND, self._held_round_one_spend)
def bank_first_round(self) -> BilledRoundsSpend | None:
self._bank_apart_from_the_follow_up()
@ -237,6 +239,10 @@ class ChatStreamRoundBilling:
first: Final = self._first
return first if first is not None and self._holds_round_one_alone(first) else None
def _held_round_one_spend(self) -> BilledRoundsSpend | None:
first: Final = self._first_round_held_alone()
return None if first is None else first.priced.spend
def _holds_round_one_alone(self, first: _FirstRound) -> bool:
parked: Final = self._peek_parked()
return parked is first.parked or parked is first.logged_alone

View file

@ -2,12 +2,13 @@ from collections.abc import Coroutine
from dataclasses import dataclass
from typing import Final, Protocol
from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend, UncachedPrice
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.litellm_logging import UncachedPrice
PARKED_DISPATCH: Final = "_deferred_stream_complete_args"
DISPATCH_RELEASE: Final = "_on_deferred_stream_complete"
IN_FLIGHT_ROUND: Final = "_bill_in_flight_round_on_disconnect"
HELD_ROUND_SPEND: Final = "_held_round_spend_for_failure"
SERVED_FROM_CACHE: Final = UncachedPrice(cost=0.0, cost_breakdown=None)
@ -30,6 +31,10 @@ class InFlightRoundBiller(Protocol):
def __call__(self) -> Coroutine[object, object, None]: ...
class HeldRoundSpend(Protocol):
def __call__(self) -> BilledRoundsSpend | None: ...
def success_dispatcher(logger: LiteLLMLoggingObj) -> DispatchesSuccessHandlers:
return logger
@ -52,6 +57,11 @@ def in_flight_round_biller(logging_obj: object) -> InFlightRoundBiller | None:
return biller if isinstance(logging_obj, LiteLLMLoggingObj) and callable(biller) else None
def held_round_spend(logging_obj: object) -> BilledRoundsSpend | None:
held: Final[HeldRoundSpend | None] = getattr(logging_obj, HELD_ROUND_SPEND, None)
return held() if isinstance(logging_obj, LiteLLMLoggingObj) and callable(held) else None
def parked_success(parked: tuple[object, ...]) -> ParkedSuccess | None:
match parked:
case (assembled, cache_hit):

View file

@ -1933,6 +1933,40 @@ async def test_stream_auto_execute_blocked_by_a_post_call_guardrail_bills_both_r
assert recorder.failed[0]["response_cost"] == pytest.approx(charged_once)
@pytest.mark.asyncio
async def test_stream_auto_execute_rejected_by_the_proxy_during_the_follow_up_bills_round_one_with_its_own_breakdown(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
from fastapi import HTTPException
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
from litellm.proxy.utils import ProxyLogging
monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}})
_serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_CHAT_ROUND_TWO)))
recorder: Final = _ChatRoundRecorder()
logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-rejected-mid-follow-up", stream=True)
rejected: Final = HTTPException(status_code=400, detail="rejected by a stream hook")
with _echo_tool_served():
stream: Final = await _open_chat_echo_stream(logging_obj)
ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch(
response=stream, route_type="acompletion", user_api_key_dict=MagicMock(), logging_obj=logging_obj
)
await _read_until(stream, "PING")
assert ProxyLogging._discard_deferred_stream_logging_for_failure({"litellm_logging_obj": logging_obj}, rejected)
await logging_obj.async_failure_handler(rejected, "rejected by a stream hook")
await recorder.settled(failed=1)
assert recorder.logged == ()
assert len(recorder.failed) == 1
failed: Final = recorder.failed[0]
breakdown: Final = failed["cost_breakdown"]
assert (failed["prompt_tokens"], failed["completion_tokens"]) == (71, 19)
assert failed["response_cost"] == pytest.approx(_chat_round_cost(_CHAT_ROUND_ONE))
assert breakdown is not None
assert breakdown["total_cost"] == pytest.approx(failed["response_cost"])
@pytest.mark.asyncio
async def test_stream_auto_execute_answered_without_a_tool_call_logs_its_one_round_once_the_answer_finishes(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch