mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(mcp): give a proxy-rejected follow-up's failure row round 1's own cost breakdown
This commit is contained in:
parent
d433450ba5
commit
3e4ceef8d5
4 changed files with 54 additions and 1 deletions
|
|
@ -91,6 +91,7 @@ from litellm.proxy.common_utils.openai_error_payload import (
|
|||
)
|
||||
from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
|
||||
from litellm.responses.litellm_completion_transformation.compaction_items import messages_since_newest_compaction
|
||||
from litellm.responses.mcp.parked_stream_dispatch import held_round_spend
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
from litellm.types.proxy.model_listing import ModelInfoResponse
|
||||
from litellm.types.utils import MCP_GUARDRAIL_CALL_TYPES, CallTypes, CallTypesLiteral, ModelInfo, Usage
|
||||
|
|
@ -4934,6 +4935,8 @@ class ProxyLogging:
|
|||
if not isinstance(error, _PROXY_ONLY_LLM_API_ERRORS):
|
||||
return False
|
||||
match _args:
|
||||
case (ModelResponse(), *_) if (held := held_round_spend(logging_obj)) is not None:
|
||||
logging_obj.record_billed_rounds_for_failure(held)
|
||||
case (ModelResponse() as assembled, *_):
|
||||
logging_obj.record_assembled_response_for_failure(assembled)
|
||||
case (BilledRoundsDispatch() as billed,):
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend, Uncach
|
|||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.responses.mcp.parked_stream_dispatch import (
|
||||
DISPATCH_RELEASE,
|
||||
HELD_ROUND_SPEND,
|
||||
IN_FLIGHT_ROUND,
|
||||
PARKED_DISPATCH,
|
||||
SERVED_FROM_CACHE,
|
||||
|
|
@ -114,6 +115,7 @@ class ChatStreamRoundBilling:
|
|||
assembled: Final = _assembled_round(self._peek_parked())
|
||||
if assembled is not None:
|
||||
self._first = _FirstRound(self._priced(assembled), assembled.parked, assembled.parked)
|
||||
setattr(self._logging_obj, HELD_ROUND_SPEND, self._held_round_one_spend)
|
||||
|
||||
def bank_first_round(self) -> BilledRoundsSpend | None:
|
||||
self._bank_apart_from_the_follow_up()
|
||||
|
|
@ -237,6 +239,10 @@ class ChatStreamRoundBilling:
|
|||
first: Final = self._first
|
||||
return first if first is not None and self._holds_round_one_alone(first) else None
|
||||
|
||||
def _held_round_one_spend(self) -> BilledRoundsSpend | None:
|
||||
first: Final = self._first_round_held_alone()
|
||||
return None if first is None else first.priced.spend
|
||||
|
||||
def _holds_round_one_alone(self, first: _FirstRound) -> bool:
|
||||
parked: Final = self._peek_parked()
|
||||
return parked is first.parked or parked is first.logged_alone
|
||||
|
|
|
|||
|
|
@ -2,12 +2,13 @@ from collections.abc import Coroutine
|
|||
from dataclasses import dataclass
|
||||
from typing import Final, Protocol
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend, UncachedPrice
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.litellm_logging import UncachedPrice
|
||||
|
||||
PARKED_DISPATCH: Final = "_deferred_stream_complete_args"
|
||||
DISPATCH_RELEASE: Final = "_on_deferred_stream_complete"
|
||||
IN_FLIGHT_ROUND: Final = "_bill_in_flight_round_on_disconnect"
|
||||
HELD_ROUND_SPEND: Final = "_held_round_spend_for_failure"
|
||||
SERVED_FROM_CACHE: Final = UncachedPrice(cost=0.0, cost_breakdown=None)
|
||||
|
||||
|
||||
|
|
@ -30,6 +31,10 @@ class InFlightRoundBiller(Protocol):
|
|||
def __call__(self) -> Coroutine[object, object, None]: ...
|
||||
|
||||
|
||||
class HeldRoundSpend(Protocol):
|
||||
def __call__(self) -> BilledRoundsSpend | None: ...
|
||||
|
||||
|
||||
def success_dispatcher(logger: LiteLLMLoggingObj) -> DispatchesSuccessHandlers:
|
||||
return logger
|
||||
|
||||
|
|
@ -52,6 +57,11 @@ def in_flight_round_biller(logging_obj: object) -> InFlightRoundBiller | None:
|
|||
return biller if isinstance(logging_obj, LiteLLMLoggingObj) and callable(biller) else None
|
||||
|
||||
|
||||
def held_round_spend(logging_obj: object) -> BilledRoundsSpend | None:
|
||||
held: Final[HeldRoundSpend | None] = getattr(logging_obj, HELD_ROUND_SPEND, None)
|
||||
return held() if isinstance(logging_obj, LiteLLMLoggingObj) and callable(held) else None
|
||||
|
||||
|
||||
def parked_success(parked: tuple[object, ...]) -> ParkedSuccess | None:
|
||||
match parked:
|
||||
case (assembled, cache_hit):
|
||||
|
|
|
|||
|
|
@ -1933,6 +1933,40 @@ async def test_stream_auto_execute_blocked_by_a_post_call_guardrail_bills_both_r
|
|||
assert recorder.failed[0]["response_cost"] == pytest.approx(charged_once)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_auto_execute_rejected_by_the_proxy_during_the_follow_up_bills_round_one_with_its_own_breakdown(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
from fastapi import HTTPException
|
||||
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
|
||||
monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}})
|
||||
_serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_CHAT_ROUND_TWO)))
|
||||
recorder: Final = _ChatRoundRecorder()
|
||||
logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-rejected-mid-follow-up", stream=True)
|
||||
rejected: Final = HTTPException(status_code=400, detail="rejected by a stream hook")
|
||||
|
||||
with _echo_tool_served():
|
||||
stream: Final = await _open_chat_echo_stream(logging_obj)
|
||||
ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch(
|
||||
response=stream, route_type="acompletion", user_api_key_dict=MagicMock(), logging_obj=logging_obj
|
||||
)
|
||||
await _read_until(stream, "PING")
|
||||
assert ProxyLogging._discard_deferred_stream_logging_for_failure({"litellm_logging_obj": logging_obj}, rejected)
|
||||
await logging_obj.async_failure_handler(rejected, "rejected by a stream hook")
|
||||
await recorder.settled(failed=1)
|
||||
|
||||
assert recorder.logged == ()
|
||||
assert len(recorder.failed) == 1
|
||||
failed: Final = recorder.failed[0]
|
||||
breakdown: Final = failed["cost_breakdown"]
|
||||
assert (failed["prompt_tokens"], failed["completion_tokens"]) == (71, 19)
|
||||
assert failed["response_cost"] == pytest.approx(_chat_round_cost(_CHAT_ROUND_ONE))
|
||||
assert breakdown is not None
|
||||
assert breakdown["total_cost"] == pytest.approx(failed["response_cost"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_auto_execute_answered_without_a_tool_call_logs_its_one_round_once_the_answer_finishes(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue