mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-11 08:38:24 +00:00
fix(mcp): bill both rounds of a streamed chat completions MCP auto-execute
A streamed /v1/chat/completions request with an MCP tool set to require_approval "never" runs two model calls on one logging object, and the final-stream dedupe kept only the first round's spend row. The final usage chunk also showed only the second round. The MCP stream iterator now holds the first round's assembled response, prices it, and logs one success row that carries the usage and cost of both rounds, with the fixed margin charged once. The final usage chunk shows the same summed usage. A single-round stream still logs its one row when the answer finishes. When the follow-up round fails, the first round is logged from a copy of the logging object taken before the follow-up opened, and the failure row carries only the follow-up's partial usage with no second fixed margin. With proxy post-call guardrails armed, only (assembled, cache_hit) tuples are parked for the proxy, and the deferred log is armed only while the iterator reads, so the proxy's orphaned-callback cleanup never sees it after the Router's first-chunk read
This commit is contained in:
mateo-berri
parent
e2d78b69cd
commit
b6ee181ec9
4 changed files with 684 additions and 22 deletions
|
|
@ -10,6 +10,7 @@ from typing_extensions import TypedDict, Unpack
|
|||
from litellm._internal_context import internal_sub_call
|
||||
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.responses.mcp.chat_stream_round_billing import ChatStreamRoundBilling, engaged_round_billing
|
||||
from litellm.responses.mcp.litellm_proxy_mcp_handler import (
|
||||
LiteLLM_Proxy_MCP_Handler,
|
||||
)
|
||||
|
|
@ -258,10 +259,19 @@ async def acompletion_with_mcp(
|
|||
self.follow_up_stream = None
|
||||
self.follow_up_iterator = None
|
||||
self.follow_up_exhausted = False
|
||||
self._round_billing: ChatStreamRoundBilling | None = engaged_round_billing(initial_stream.logging_obj)
|
||||
|
||||
def __aiter__(self):
|
||||
return self
|
||||
|
||||
def __del__(self) -> None:
|
||||
self.release_abandoned()
|
||||
|
||||
def release_abandoned(self) -> None:
|
||||
round_billing: Final[object] = getattr(self, "_round_billing", None)
|
||||
if isinstance(round_billing, ChatStreamRoundBilling):
|
||||
round_billing.release_abandoned()
|
||||
|
||||
def _add_mcp_list_tools_to_chunk(self, chunk: ModelResponseStream) -> ModelResponseStream:
|
||||
"""Add mcp_list_tools to the first chunk."""
|
||||
from litellm.types.utils import (
|
||||
|
|
@ -332,7 +342,32 @@ async def acompletion_with_mcp(
|
|||
"Error draining inner MCP stream after final chunk; spend logging may be incomplete"
|
||||
)
|
||||
|
||||
async def __anext__(self):
|
||||
async def _complete_first_round(self) -> None:
|
||||
await self._drain_inner_stream()
|
||||
if self._round_billing is not None:
|
||||
self._round_billing.complete_first_round()
|
||||
|
||||
async def _follow_up_or_release_first_round(self) -> None:
|
||||
if self.tool_results and self.complete_response:
|
||||
await self._prepare_follow_up_call()
|
||||
if self.follow_up_stream is None and self._round_billing is not None:
|
||||
self._round_billing.finish()
|
||||
|
||||
async def __anext__(self) -> ModelResponseStream:
|
||||
round_billing: Final = self._round_billing
|
||||
if round_billing is None:
|
||||
return await self._next_chunk()
|
||||
with round_billing.armed_while_reading():
|
||||
try:
|
||||
return await self._next_chunk()
|
||||
except StopAsyncIteration:
|
||||
round_billing.finish()
|
||||
raise
|
||||
except Exception:
|
||||
round_billing.fail()
|
||||
raise
|
||||
|
||||
async def _next_chunk(self) -> ModelResponseStream:
|
||||
# Phase 1: Collect and yield initial stream chunks
|
||||
if not self.stream_exhausted:
|
||||
# Get the iterator from the stream wrapper
|
||||
|
|
@ -362,29 +397,22 @@ async def acompletion_with_mcp(
|
|||
|
||||
if is_final:
|
||||
self.stream_exhausted = True
|
||||
await self._complete_first_round()
|
||||
await self._process_tool_calls()
|
||||
chunk = self._add_mcp_tool_metadata_to_final_chunk(chunk)
|
||||
if self.tool_results and self.complete_response:
|
||||
await self._prepare_follow_up_call()
|
||||
# Drain inner stream so CustomStreamWrapper fires its
|
||||
# end-of-stream handler (dispatch_success_handlers →
|
||||
# _ProxyDBLogger → LiteLLM_SpendLogs). The CSW may
|
||||
# yield one usage chunk before raising StopAsyncIteration.
|
||||
await self._drain_inner_stream()
|
||||
await self._follow_up_or_release_first_round()
|
||||
|
||||
return chunk
|
||||
except StopAsyncIteration:
|
||||
self.stream_exhausted = True
|
||||
await self._complete_first_round()
|
||||
# Process tool calls after stream is exhausted
|
||||
await self._process_tool_calls()
|
||||
# If we have chunks, yield the final one with metadata
|
||||
if self.collected_chunks:
|
||||
final_chunk = self.collected_chunks[-1]
|
||||
final_chunk = self._add_mcp_tool_metadata_to_final_chunk(final_chunk)
|
||||
# If we have tool results, prepare follow-up call
|
||||
if self.tool_results and self.complete_response:
|
||||
await self._prepare_follow_up_call()
|
||||
await self._drain_inner_stream()
|
||||
await self._follow_up_or_release_first_round()
|
||||
return final_chunk
|
||||
|
||||
# Phase 2: Yield follow-up stream chunks if available
|
||||
|
|
@ -400,7 +428,7 @@ async def acompletion_with_mcp(
|
|||
from litellm._logging import verbose_logger
|
||||
|
||||
verbose_logger.debug("Follow-up chunk yielded: %s", chunk)
|
||||
return chunk
|
||||
return chunk if self._round_billing is None else self._round_billing.client_chunk(chunk)
|
||||
except StopAsyncIteration:
|
||||
self.follow_up_exhausted = True
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -488,10 +516,13 @@ async def acompletion_with_mcp(
|
|||
# This ensures the patch works correctly in tests
|
||||
import litellm
|
||||
|
||||
earlier_rounds: Final = None if self._round_billing is None else self._round_billing.bank_first_round()
|
||||
follow_up_response: Final = await litellm.acompletion(**follow_up_call_args)
|
||||
|
||||
# Ensure follow-up response is a CustomStreamWrapper
|
||||
if isinstance(follow_up_response, CustomStreamWrapper):
|
||||
if earlier_rounds is not None:
|
||||
follow_up_response.bill_failure_after_earlier_rounds(earlier_rounds)
|
||||
self.follow_up_stream = follow_up_response
|
||||
from litellm._logging import verbose_logger
|
||||
|
||||
|
|
@ -548,6 +579,10 @@ async def acompletion_with_mcp(
|
|||
def __aiter__(self):
|
||||
return self._custom_iterator
|
||||
|
||||
async def aclose(self) -> None:
|
||||
self._custom_iterator.release_abandoned()
|
||||
await super().aclose()
|
||||
|
||||
def __iter__(self):
|
||||
# For synchronous iteration, create a sync wrapper
|
||||
if self._sync_iterator is None:
|
||||
|
|
|
|||
210
litellm/responses/mcp/chat_stream_round_billing.py
Normal file
210
litellm/responses/mcp/chat_stream_round_billing.py
Normal file
|
|
@ -0,0 +1,210 @@
|
|||
import asyncio
|
||||
from collections.abc import Generator
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, replace
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Protocol, TypeAlias
|
||||
|
||||
from litellm.litellm_core_utils.asyncify import is_event_loop_running
|
||||
from litellm.litellm_core_utils.hidden_params import get_hidden_params, served_from_cache, set_hidden_params
|
||||
from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend, UncachedPrice
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.responses.mcp.round_billing import billed_for_every_round, merged_chat_usage, summed_cost_breakdown
|
||||
from litellm.responses.streaming_iterator import spawn_logging_task
|
||||
from litellm.types.utils import ModelResponse, ModelResponseStream, Usage
|
||||
|
||||
Parked: TypeAlias = tuple[object, ...]
|
||||
|
||||
_PARKED_DISPATCH: Final = "_deferred_stream_complete_args"
|
||||
_DISPATCH_RELEASE: Final = "_on_deferred_stream_complete"
|
||||
_SERVED_FROM_CACHE: Final = UncachedPrice(cost=0.0, cost_breakdown=None)
|
||||
|
||||
|
||||
class _DispatchesSuccessHandlers(Protocol):
|
||||
async def dispatch_success_handlers(
|
||||
self,
|
||||
result: object = None,
|
||||
start_time: object = None,
|
||||
end_time: object = None,
|
||||
cache_hit: object = None,
|
||||
prefer_async_handlers: bool = False,
|
||||
) -> None: ...
|
||||
|
||||
|
||||
def _success_dispatcher(logger: LiteLLMLoggingObj) -> _DispatchesSuccessHandlers:
|
||||
return logger
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _AssembledRound:
|
||||
response: ModelResponse
|
||||
usage: Usage
|
||||
cache_hit: bool
|
||||
parked: Parked
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _PricedRound:
|
||||
response: ModelResponse
|
||||
usage: Usage
|
||||
price: UncachedPrice
|
||||
cache_hit: bool
|
||||
|
||||
@property
|
||||
def spend(self) -> BilledRoundsSpend:
|
||||
return BilledRoundsSpend(self.usage, self.price.cost or 0.0, self.price.cost_breakdown)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _FirstRound:
|
||||
priced: _PricedRound
|
||||
parked: Parked
|
||||
logged_alone: Parked
|
||||
banked: LiteLLMLoggingObj | None = None
|
||||
|
||||
|
||||
def _assembled_round(parked: Parked | None) -> _AssembledRound | None:
|
||||
match parked:
|
||||
case (ModelResponse() as response, cache_hit) if isinstance(usage := getattr(response, "usage", None), Usage):
|
||||
return _AssembledRound(response, usage, cache_hit is True, parked)
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def engaged_round_billing(logging_obj: object) -> "ChatStreamRoundBilling | None":
|
||||
return ChatStreamRoundBilling(logging_obj) if isinstance(logging_obj, LiteLLMLoggingObj) else None
|
||||
|
||||
|
||||
class ChatStreamRoundBilling:
|
||||
def __init__(self, logging_obj: LiteLLMLoggingObj) -> None:
|
||||
self._logging_obj: Final = logging_obj
|
||||
self._first: _FirstRound | None = None
|
||||
|
||||
@contextmanager
|
||||
def armed_while_reading(self) -> Generator[None]:
|
||||
armed: Final = getattr(self._logging_obj, _DISPATCH_RELEASE, None) is None
|
||||
if armed:
|
||||
setattr(self._logging_obj, _DISPATCH_RELEASE, self._log_success)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
if armed and self._peek_parked() is None:
|
||||
self._disarm()
|
||||
|
||||
def complete_first_round(self) -> None:
|
||||
assembled: Final = _assembled_round(self._peek_parked())
|
||||
if assembled is not None:
|
||||
self._first = _FirstRound(self._priced(assembled), assembled.parked, assembled.parked)
|
||||
|
||||
def bank_first_round(self) -> BilledRoundsSpend | None:
|
||||
self._bank_apart_from_the_follow_up()
|
||||
return None if self._first is None else self._first.priced.spend
|
||||
|
||||
def client_chunk(self, chunk: ModelResponseStream) -> ModelResponseStream:
|
||||
usage: Final[object] = getattr(chunk, "usage", None)
|
||||
if self._first is None or not isinstance(usage, Usage):
|
||||
return chunk
|
||||
return chunk.model_copy(update=MappingProxyType({"usage": merged_chat_usage(self._first.priced.usage, usage)}))
|
||||
|
||||
def finish(self) -> None:
|
||||
parked: Final = self._take_parked()
|
||||
first: Final = self._first
|
||||
self._first = None
|
||||
if parked is None:
|
||||
self._disarm()
|
||||
return
|
||||
if first is None or parked is first.logged_alone:
|
||||
self._release(parked)
|
||||
return
|
||||
final_round: Final = _assembled_round(parked)
|
||||
if final_round is None:
|
||||
self._release(first.logged_alone)
|
||||
return
|
||||
self._release(self._billed_for_both_rounds(first.priced, self._priced(final_round)))
|
||||
|
||||
def fail(self) -> None:
|
||||
self._bank_apart_from_the_follow_up()
|
||||
first: Final = self._first
|
||||
if first is None or first.banked is None or self._peek_parked() is not first.parked:
|
||||
self.finish()
|
||||
return
|
||||
self._first = None
|
||||
setattr(self._logging_obj, _PARKED_DISPATCH, None)
|
||||
self._log_apart((*first.parked, first.banked))
|
||||
|
||||
def release_abandoned(self) -> None:
|
||||
if not self._owns_release():
|
||||
return
|
||||
if is_event_loop_running():
|
||||
self.finish()
|
||||
return
|
||||
setattr(self._logging_obj, _PARKED_DISPATCH, None)
|
||||
self._first = None
|
||||
self._disarm()
|
||||
|
||||
def _bank_apart_from_the_follow_up(self) -> None:
|
||||
first: Final = self._first
|
||||
if first is None or first.banked is not None or self._peek_parked() is not first.parked:
|
||||
return
|
||||
banked: Final = self._logging_obj.attempt_scoped_copy()
|
||||
if not self._owns_release():
|
||||
self._first = replace(first, banked=banked)
|
||||
return
|
||||
logged_alone: Final = (*first.parked, banked)
|
||||
self._first = replace(first, logged_alone=logged_alone, banked=banked)
|
||||
setattr(self._logging_obj, _PARKED_DISPATCH, logged_alone)
|
||||
|
||||
def _billed_for_both_rounds(self, first: _PricedRound, final: _PricedRound) -> Parked:
|
||||
first_breakdown: Final = first.price.cost_breakdown
|
||||
final_breakdown: Final = final.price.cost_breakdown
|
||||
billed: Final = billed_for_every_round(first.response, first_breakdown, final.response, final_breakdown)
|
||||
cache_hit: Final = first.cache_hit and final.cache_hit
|
||||
set_hidden_params(billed, {**(get_hidden_params(billed) or {}), "cache_hit": cache_hit})
|
||||
self._logging_obj.cost_breakdown = summed_cost_breakdown(
|
||||
first.response, first_breakdown, final.response, final_breakdown
|
||||
)
|
||||
return billed, cache_hit
|
||||
|
||||
def _priced(self, assembled: _AssembledRound) -> _PricedRound:
|
||||
served: Final = assembled.cache_hit or served_from_cache(assembled.response)
|
||||
price: Final = _SERVED_FROM_CACHE if served else self._logging_obj.uncached_price(assembled.response)
|
||||
priced: Final = assembled.response.model_copy()
|
||||
set_hidden_params(
|
||||
priced,
|
||||
{**(get_hidden_params(assembled.response) or {}), "response_cost": price.cost, "cache_hit": served},
|
||||
)
|
||||
return _PricedRound(priced, assembled.usage, price, served)
|
||||
|
||||
def _peek_parked(self) -> Parked | None:
|
||||
parked: Final[Parked | None] = getattr(self._logging_obj, _PARKED_DISPATCH, None)
|
||||
return parked if isinstance(parked, tuple) else None
|
||||
|
||||
def _take_parked(self) -> Parked | None:
|
||||
parked: Final = self._peek_parked()
|
||||
setattr(self._logging_obj, _PARKED_DISPATCH, None)
|
||||
return parked
|
||||
|
||||
def _owns_release(self) -> bool:
|
||||
release: Final[object] = getattr(self._logging_obj, _DISPATCH_RELEASE, None)
|
||||
return release == self._log_success
|
||||
|
||||
def _release(self, parked: Parked) -> None:
|
||||
if getattr(self._logging_obj, _DISPATCH_RELEASE, None) is not None and not self._owns_release():
|
||||
setattr(self._logging_obj, _PARKED_DISPATCH, parked)
|
||||
return
|
||||
setattr(self._logging_obj, _DISPATCH_RELEASE, None)
|
||||
self._log_apart(parked)
|
||||
|
||||
def _log_apart(self, parked: Parked) -> None:
|
||||
loop: Final = asyncio.get_running_loop()
|
||||
spawn_logging_task(loop, self._log_success(*parked), task_name="Chat completions MCP stream success log")
|
||||
|
||||
def _disarm(self) -> None:
|
||||
if self._owns_release():
|
||||
setattr(self._logging_obj, _DISPATCH_RELEASE, None)
|
||||
|
||||
async def _log_success(self, assembled: object, cache_hit: object, banked: object = None) -> None:
|
||||
logger: Final = banked if isinstance(banked, LiteLLMLoggingObj) else self._logging_obj
|
||||
await _success_dispatcher(logger).dispatch_success_handlers(
|
||||
assembled, cache_hit=cache_hit, start_time=None, end_time=None, prefer_async_handlers=True
|
||||
)
|
||||
|
|
@ -950,6 +950,57 @@ def test_nonstream_failed_follow_up_bills_round_one_on_the_failure_row(
|
|||
)
|
||||
|
||||
|
||||
def _streamed_chunks(response: httpx.Response) -> tuple[dict[str, object], ...]:
|
||||
return tuple(
|
||||
json.loads(line.removeprefix("data: "))
|
||||
for line in response.text.splitlines()
|
||||
if line.startswith("data: ") and line != "data: [DONE]"
|
||||
)
|
||||
|
||||
|
||||
def test_streaming_chat_auto_execute_logs_the_answer_and_bills_both_rounds(gateway: Gateway) -> None:
|
||||
billing: Final = Billing(FIRST_ROUND, FOLLOW_UP_ROUND)
|
||||
with _rig(gateway, "chat", billing=billing, prices=PRICES) as rig:
|
||||
response: Final = rig.send(_granted_key(rig), [AUTO], stream=True, stream_options={"include_usage": True})
|
||||
assert response.status_code == 200, response.text
|
||||
chunks: Final = _streamed_chunks(response)
|
||||
text: Final = "".join(
|
||||
str(chunk["choices"][0]["delta"].get("content") or "") for chunk in chunks if chunk.get("choices")
|
||||
)
|
||||
assert text == ANSWER, response.text
|
||||
usages: Final = tuple(chunk["usage"] for chunk in chunks if chunk.get("usage"))
|
||||
assert usages, response.text
|
||||
rows: Final = _spend_rows(response.headers["x-litellm-call-id"], "acompletion", "success")
|
||||
assert len(rows) == 1, rows
|
||||
row: Final = rows[0]
|
||||
assert row["request_id"] == chunks[-1]["id"], (row, chunks[-1])
|
||||
assert (row["prompt_tokens"], row["completion_tokens"]) == (
|
||||
FIRST_ROUND[0] + FOLLOW_UP_ROUND[0],
|
||||
FIRST_ROUND[1] + FOLLOW_UP_ROUND[1],
|
||||
), row
|
||||
assert float(str(row["spend"])) == pytest.approx(_price(FIRST_ROUND, FOLLOW_UP_ROUND)), row
|
||||
final_usage: Final = usages[-1]
|
||||
assert (final_usage["prompt_tokens"], final_usage["completion_tokens"], final_usage["total_tokens"]) == (
|
||||
row["prompt_tokens"],
|
||||
row["completion_tokens"],
|
||||
sum((*FIRST_ROUND, *FOLLOW_UP_ROUND)),
|
||||
), usages
|
||||
|
||||
|
||||
def test_streaming_chat_auto_execute_whose_follow_up_fails_bills_round_one_once(gateway: Gateway) -> None:
|
||||
billing: Final = Billing(FIRST_ROUND, FOLLOW_UP_ROUND, follow_up_fails=True)
|
||||
with _rig(gateway, "chat", billing=billing, prices=PRICES) as rig:
|
||||
response: Final = rig.send(_granted_key(rig), [AUTO], stream=True, stream_options={"include_usage": True})
|
||||
assert FOLLOW_UP_OUTAGE in response.text, response.text
|
||||
call_id: Final = response.headers["x-litellm-call-id"]
|
||||
succeeded: Final = _spend_rows(call_id, "acompletion", "success")
|
||||
assert len(succeeded) == 1, succeeded
|
||||
assert (succeeded[0]["prompt_tokens"], succeeded[0]["completion_tokens"]) == FIRST_ROUND, succeeded
|
||||
assert float(str(succeeded[0]["spend"])) == pytest.approx(_price(FIRST_ROUND)), succeeded
|
||||
failed: Final = _spend_rows(call_id, "acompletion", "failure")
|
||||
assert [float(str(row["spend"])) for row in failed] == [0.0], failed
|
||||
|
||||
|
||||
def _tool_rows(call_id: str, expected: int) -> list[dict[str, JsonValue]]:
|
||||
return eventually(
|
||||
lambda: read_rows(
|
||||
|
|
|
|||
|
|
@ -1,13 +1,18 @@
|
|||
import asyncio
|
||||
import gc
|
||||
import json
|
||||
import queue
|
||||
import sys
|
||||
import types
|
||||
from collections.abc import Mapping, Sequence
|
||||
import warnings
|
||||
from collections.abc import AsyncIterator, Callable, Iterator, Mapping, Sequence
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime
|
||||
from types import MappingProxyType
|
||||
from typing import Final, cast
|
||||
from unittest.mock import AsyncMock, patch
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from httpx import Response
|
||||
|
|
@ -25,7 +30,8 @@ from litellm.responses.mcp.litellm_proxy_mcp_handler import (
|
|||
LiteLLM_Proxy_MCP_Handler,
|
||||
)
|
||||
from litellm.responses.utils import ResponsesAPIRequestUtils
|
||||
from litellm.types.utils import ModelResponse, StandardLoggingPayload
|
||||
from litellm.types.utils import ModelResponse, ModelResponseStream, StandardLoggingPayload, Usage
|
||||
from litellm.utils import CustomStreamWrapper
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -1600,18 +1606,28 @@ async def _settled() -> None:
|
|||
await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10)
|
||||
|
||||
|
||||
async def _auto_execute_chat_echo(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, rounds: Sequence[Response], call_id: str
|
||||
) -> tuple[_ChatRoundRecorder, ModelResponse | Exception]:
|
||||
async def _settled_once(logged: Callable[[], bool]) -> None:
|
||||
for _ in range(200):
|
||||
if logged():
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
await _settled()
|
||||
|
||||
|
||||
def _serve_chat_rounds(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, rounds: Sequence[Response]
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
respx_mock.post("https://mcp-chat-rounds.example.com/v1/chat/completions").mock(side_effect=list(rounds))
|
||||
recorder: Final = _ChatRoundRecorder()
|
||||
logging_obj: Final = Logging(
|
||||
|
||||
|
||||
def _chat_rounds_logging_obj(recorder: _ChatRoundRecorder, call_id: str, stream: bool) -> Logging:
|
||||
return Logging(
|
||||
model="openai/gpt-5.6",
|
||||
messages=[{"role": "user", "content": "Call echo with PING"}],
|
||||
stream=False,
|
||||
stream=stream,
|
||||
call_type="acompletion",
|
||||
start_time=datetime(2026, 1, 1),
|
||||
litellm_call_id=call_id,
|
||||
|
|
@ -1619,6 +1635,10 @@ async def _auto_execute_chat_echo(
|
|||
dynamic_async_success_callbacks=[recorder],
|
||||
dynamic_async_failure_callbacks=[recorder],
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _echo_tool_served() -> Iterator[None]:
|
||||
echo_tool: Final = MCPTool.model_validate(
|
||||
{"name": "echo", "inputSchema": {"type": "object", "properties": {"value": {"type": "string"}}}},
|
||||
by_name=False,
|
||||
|
|
@ -1635,6 +1655,16 @@ async def _auto_execute_chat_echo(
|
|||
AsyncMock(return_value=[{"tool_call_id": "call_echo", "result": "PING", "name": "echo"}]),
|
||||
),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
async def _auto_execute_chat_echo(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, rounds: Sequence[Response], call_id: str
|
||||
) -> tuple[_ChatRoundRecorder, ModelResponse | Exception]:
|
||||
_serve_chat_rounds(respx_mock, monkeypatch, rounds)
|
||||
recorder: Final = _ChatRoundRecorder()
|
||||
logging_obj: Final = _chat_rounds_logging_obj(recorder, call_id, stream=False)
|
||||
with _echo_tool_served():
|
||||
try:
|
||||
response: Final = await litellm.acompletion(
|
||||
model="openai/gpt-5.6",
|
||||
|
|
@ -1712,3 +1742,339 @@ async def test_non_stream_auto_execute_bills_the_paid_first_round_when_the_follo
|
|||
assert (recorder.failed[0]["prompt_tokens"], recorder.failed[0]["completion_tokens"]) == (71, 19)
|
||||
assert recorder.failed[0]["response_cost"] == pytest.approx(_chat_round_cost(_CHAT_ROUND_ONE))
|
||||
assert recorder.failed[0]["response_cost"] > 0
|
||||
|
||||
|
||||
_FIXED_MARGIN: Final = 0.001
|
||||
|
||||
|
||||
def _chat_stream_events(body: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
|
||||
completed: Final = json.loads(json.dumps(body, default=dict))
|
||||
choice: Final = completed["choices"][0]
|
||||
message: Final = choice["message"]
|
||||
calls: Final = message.get("tool_calls") or []
|
||||
delta: Final = (
|
||||
{**message, "tool_calls": [{**call, "index": index} for index, call in enumerate(calls)]} if calls else message
|
||||
)
|
||||
head: Final = {"id": completed["id"], "object": "chat.completion.chunk", "created": 1, "model": completed["model"]}
|
||||
return (
|
||||
{**head, "choices": [{"index": 0, "delta": delta, "finish_reason": None}]},
|
||||
{**head, "choices": [{"index": 0, "delta": {}, "finish_reason": choice["finish_reason"]}]},
|
||||
{**head, "choices": [], "usage": completed["usage"]},
|
||||
)
|
||||
|
||||
|
||||
def _sse(events: Sequence[Mapping[str, object]]) -> bytes:
|
||||
return b"".join(f"data: {json.dumps(event)}\n\n".encode() for event in events)
|
||||
|
||||
|
||||
def _streamed(body: Mapping[str, object]) -> Response:
|
||||
return Response(
|
||||
200,
|
||||
content=_sse(_chat_stream_events(body)) + b"data: [DONE]\n\n",
|
||||
headers={"content-type": "text/event-stream"},
|
||||
)
|
||||
|
||||
|
||||
class _CutOnceTheAnswerStarts(httpx.AsyncByteStream):
|
||||
async def __aiter__(self) -> AsyncIterator[bytes]:
|
||||
yield _sse(_chat_stream_events(_CHAT_ROUND_TWO)[:1])
|
||||
raise httpx.ReadError("connection reset mid-stream")
|
||||
|
||||
|
||||
def _cut_once_the_answer_starts() -> Response:
|
||||
return Response(200, stream=_CutOnceTheAnswerStarts(), headers={"content-type": "text/event-stream"})
|
||||
|
||||
|
||||
async def _open_chat_echo_stream(logging_obj: Logging) -> CustomStreamWrapper:
|
||||
stream: Final = await litellm.acompletion(
|
||||
model="openai/gpt-5.6",
|
||||
messages=[{"role": "user", "content": "Call echo with PING"}],
|
||||
tools=[{"type": "mcp", "server_url": "litellm_proxy/mcp/demo", "require_approval": "never"}],
|
||||
api_key="sk-mcp-chat-rounds",
|
||||
api_base="https://mcp-chat-rounds.example.com/v1",
|
||||
stream=True,
|
||||
stream_options={"include_usage": True},
|
||||
litellm_logging_obj=logging_obj,
|
||||
)
|
||||
assert isinstance(stream, CustomStreamWrapper)
|
||||
return stream
|
||||
|
||||
|
||||
async def _read_all(stream: CustomStreamWrapper) -> tuple[ModelResponseStream, ...]:
|
||||
return tuple([chunk async for chunk in stream])
|
||||
|
||||
|
||||
async def _read_until(stream: CustomStreamWrapper, content: str) -> None:
|
||||
async for chunk in stream:
|
||||
if chunk.choices and chunk.choices[0].delta.content == content:
|
||||
return
|
||||
pytest.fail(f"the stream never sent {content!r}")
|
||||
|
||||
|
||||
def _streamed_usage(chunks: Sequence[ModelResponseStream]) -> Usage:
|
||||
usages: Final = tuple(usage for chunk in chunks if isinstance(usage := getattr(chunk, "usage", None), Usage))
|
||||
assert len(usages) == 1, usages
|
||||
return usages[0]
|
||||
|
||||
|
||||
def _assert_one_stream_row_bills_both_rounds_once(
|
||||
logged: Sequence[StandardLoggingPayload], chunks: Sequence[ModelResponseStream]
|
||||
) -> None:
|
||||
charged_once: Final = _chat_round_cost(_CHAT_ROUND_ONE) + _chat_round_cost(_CHAT_ROUND_TWO) - _FIXED_MARGIN
|
||||
assert len(logged) == 1
|
||||
row: Final = logged[0]
|
||||
assert row["id"] == chunks[-1].id
|
||||
assert (row["prompt_tokens"], row["completion_tokens"]) == (71 + 149, 19 + 5)
|
||||
assert row["metadata"]["usage_object"]["prompt_tokens_details"]["cached_tokens"] == 32 + 64
|
||||
assert row["response_cost"] == pytest.approx(charged_once)
|
||||
assert row["cost_breakdown"]["total_cost"] == pytest.approx(charged_once)
|
||||
usage: Final = _streamed_usage(chunks)
|
||||
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (71 + 149, 19 + 5, 90 + 154)
|
||||
|
||||
|
||||
def _assert_one_row_bills_round_one(logged: Sequence[StandardLoggingPayload]) -> None:
|
||||
assert len(logged) == 1
|
||||
assert (logged[0]["prompt_tokens"], logged[0]["completion_tokens"]) == (71, 19)
|
||||
assert logged[0]["response_cost"] == pytest.approx(_chat_round_cost(_CHAT_ROUND_ONE))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_auto_execute_deferred_for_a_post_call_guardrail_logs_both_rounds_once_when_the_proxy_releases(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
|
||||
monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}})
|
||||
_serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_CHAT_ROUND_TWO)))
|
||||
recorder: Final = _ChatRoundRecorder()
|
||||
logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-rounds-deferred", stream=True)
|
||||
|
||||
with _echo_tool_served():
|
||||
stream: Final = await _open_chat_echo_stream(logging_obj)
|
||||
ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch(
|
||||
response=stream, route_type="acompletion", user_api_key_dict=MagicMock(), logging_obj=logging_obj
|
||||
)
|
||||
chunks: Final = await _read_all(stream)
|
||||
await _settled()
|
||||
logged_before_release: Final = recorder.logged
|
||||
ProxyLogging.fire_deferred_stream_logging({"litellm_logging_obj": logging_obj})
|
||||
await _settled_once(lambda: len(recorder.logged) > 0)
|
||||
|
||||
assert logged_before_release == ()
|
||||
_assert_one_stream_row_bills_both_rounds_once(recorder.logged, chunks)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_auto_execute_blocked_by_a_post_call_guardrail_bills_both_rounds_on_the_failure_row(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
from litellm.exceptions import GuardrailRaisedException
|
||||
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
|
||||
monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}})
|
||||
_serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_CHAT_ROUND_TWO)))
|
||||
recorder: Final = _ChatRoundRecorder()
|
||||
logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-rounds-blocked", stream=True)
|
||||
blocked: Final = GuardrailRaisedException(guardrail_name="output-policy", message="blocked")
|
||||
|
||||
with _echo_tool_served():
|
||||
stream: Final = await _open_chat_echo_stream(logging_obj)
|
||||
ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch(
|
||||
response=stream, route_type="acompletion", user_api_key_dict=MagicMock(), logging_obj=logging_obj
|
||||
)
|
||||
await _read_all(stream)
|
||||
assert ProxyLogging._discard_deferred_stream_logging_for_failure({"litellm_logging_obj": logging_obj}, blocked)
|
||||
await logging_obj.async_failure_handler(blocked, "blocked by output-policy")
|
||||
await _settled_once(lambda: len(recorder.failed) > 0)
|
||||
|
||||
charged_once: Final = _chat_round_cost(_CHAT_ROUND_ONE) + _chat_round_cost(_CHAT_ROUND_TWO) - _FIXED_MARGIN
|
||||
assert recorder.logged == ()
|
||||
assert len(recorder.failed) == 1
|
||||
assert (recorder.failed[0]["prompt_tokens"], recorder.failed[0]["completion_tokens"]) == (71 + 149, 19 + 5)
|
||||
assert recorder.failed[0]["response_cost"] == pytest.approx(charged_once)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_auto_execute_answered_without_a_tool_call_logs_its_one_round_once_the_answer_finishes(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
_serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_TWO),))
|
||||
recorder: Final = _ChatRoundRecorder()
|
||||
logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-one-round", stream=True)
|
||||
|
||||
with _echo_tool_served():
|
||||
stream: Final = await _open_chat_echo_stream(logging_obj)
|
||||
async for chunk in stream:
|
||||
if chunk.choices and chunk.choices[0].finish_reason is not None:
|
||||
break
|
||||
await _settled_once(lambda: len(recorder.logged) > 0)
|
||||
logged_once_the_answer_finished: Final = recorder.logged
|
||||
await _read_all(stream)
|
||||
await _settled()
|
||||
|
||||
assert len(logged_once_the_answer_finished) == 1
|
||||
assert recorder.logged == logged_once_the_answer_finished
|
||||
assert recorder.logged[0]["id"] == _CHAT_ROUND_TWO["id"]
|
||||
assert (recorder.logged[0]["prompt_tokens"], recorder.logged[0]["completion_tokens"]) == (149, 5)
|
||||
assert recorder.logged[0]["response_cost"] == pytest.approx(_chat_round_cost(_CHAT_ROUND_TWO))
|
||||
assert recorder.failed == ()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("closes", (True, False), ids=("aclose", "dropped"))
|
||||
async def test_stream_auto_execute_bills_the_finished_first_round_when_the_caller_stops_reading_the_follow_up(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, closes: bool
|
||||
):
|
||||
_serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_CHAT_ROUND_TWO)))
|
||||
recorder: Final = _ChatRoundRecorder()
|
||||
logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-caller-stops", stream=True)
|
||||
|
||||
async def stop_reading_once_the_follow_up_answers() -> None:
|
||||
stream: Final = await _open_chat_echo_stream(logging_obj)
|
||||
await _read_until(stream, "PING")
|
||||
if closes:
|
||||
await stream.aclose()
|
||||
|
||||
with _echo_tool_served():
|
||||
await stop_reading_once_the_follow_up_answers()
|
||||
await _settled_once(lambda: len(recorder.logged) > 0)
|
||||
|
||||
_assert_one_row_bills_round_one(recorder.logged)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("deferred_for_a_post_call_guardrail", [False, True])
|
||||
async def test_stream_auto_execute_bills_a_broken_follow_up_on_its_own_failure_row_with_no_second_fixed_margin(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, deferred_for_a_post_call_guardrail: bool
|
||||
):
|
||||
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
|
||||
from litellm.proxy.utils import ProxyLogging, _failure_fields_to_lift
|
||||
|
||||
monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}})
|
||||
_serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _cut_once_the_answer_starts()))
|
||||
recorder: Final = _ChatRoundRecorder()
|
||||
logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-follow-up-breaks", stream=True)
|
||||
request_data: Final = MappingProxyType({"litellm_logging_obj": logging_obj})
|
||||
|
||||
with _echo_tool_served():
|
||||
stream: Final = await _open_chat_echo_stream(logging_obj)
|
||||
if deferred_for_a_post_call_guardrail:
|
||||
ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch(
|
||||
response=stream, route_type="acompletion", user_api_key_dict=MagicMock(), logging_obj=logging_obj
|
||||
)
|
||||
with pytest.raises(litellm.ServiceUnavailableError) as broken:
|
||||
await _read_all(stream)
|
||||
if not ProxyLogging._discard_deferred_stream_logging_for_failure(dict(request_data), broken.value):
|
||||
ProxyLogging.fire_deferred_stream_logging(dict(request_data))
|
||||
await _settled_once(lambda: len(recorder.logged) > 0 and len(recorder.failed) > 0)
|
||||
|
||||
_assert_one_row_bills_round_one(recorder.logged)
|
||||
assert len(recorder.failed) == 1
|
||||
failed: Final = recorder.failed[0]
|
||||
unmargined: Final = (
|
||||
litellm.completion_cost(
|
||||
completion_response=ModelResponse(
|
||||
model="gpt-5.6",
|
||||
usage=Usage(
|
||||
prompt_tokens=failed["prompt_tokens"],
|
||||
completion_tokens=failed["completion_tokens"],
|
||||
total_tokens=failed["total_tokens"],
|
||||
),
|
||||
),
|
||||
model="gpt-5.6",
|
||||
custom_llm_provider="openai",
|
||||
)
|
||||
- _FIXED_MARGIN
|
||||
)
|
||||
assert failed["completion_tokens"] > 0
|
||||
assert failed["response_cost"] == pytest.approx(unmargined)
|
||||
request_failure: Final = _failure_fields_to_lift(request_data)
|
||||
assert cast(StandardLoggingPayload, request_failure["standard_logging_object"])["id"] != recorder.logged[0]["id"]
|
||||
assert request_failure["response_cost"] == pytest.approx(unmargined)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_auto_execute_leaves_no_deferred_log_armed_for_the_proxy_to_orphan_after_its_first_chunk(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}})
|
||||
_serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_CHAT_ROUND_TWO)))
|
||||
recorder: Final = _ChatRoundRecorder()
|
||||
logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-first-chunk-read-ahead", stream=True)
|
||||
|
||||
with _echo_tool_served():
|
||||
stream: Final = await _open_chat_echo_stream(logging_obj)
|
||||
rest: Final = stream.__aiter__()
|
||||
first_chunk: Final = await rest.__anext__()
|
||||
armed_after_the_first_chunk: Final = getattr(logging_obj, "_on_deferred_stream_complete", None)
|
||||
chunks: Final = (first_chunk, *[chunk async for chunk in rest])
|
||||
await _settled_once(lambda: len(recorder.logged) > 0)
|
||||
|
||||
assert armed_after_the_first_chunk is None
|
||||
_assert_one_stream_row_bills_both_rounds_once(recorder.logged, chunks)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_auto_execute_whose_follow_up_streams_nothing_still_bills_the_first_round_once(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
nothing_streamed: Final = Response(200, content=b"data: [DONE]\n\n", headers={"content-type": "text/event-stream"})
|
||||
_serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), nothing_streamed))
|
||||
recorder: Final = _ChatRoundRecorder()
|
||||
logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-empty-follow-up", stream=True)
|
||||
|
||||
with _echo_tool_served():
|
||||
await _read_all(await _open_chat_echo_stream(logging_obj))
|
||||
await _settled_once(lambda: len(recorder.logged) > 0)
|
||||
|
||||
_assert_one_row_bills_round_one(recorder.logged)
|
||||
|
||||
|
||||
async def _stream_read_until_the_follow_up_answers(
|
||||
logging_obj: Logging, opened: "queue.SimpleQueue[CustomStreamWrapper]"
|
||||
) -> None:
|
||||
stream: Final = await _open_chat_echo_stream(logging_obj)
|
||||
await _read_until(stream, "PING")
|
||||
opened.put(stream)
|
||||
|
||||
|
||||
_CHAT_STREAM_BILLING_NAMES: Final = ("MCPStreamingIterator", "ChatStreamRoundBilling", "_log_success")
|
||||
|
||||
|
||||
def _names_chat_stream_billing(text: str) -> bool:
|
||||
return any(name in text for name in _CHAT_STREAM_BILLING_NAMES)
|
||||
|
||||
|
||||
class _UnraisableRecorder:
|
||||
def __init__(self) -> None:
|
||||
self.raised: tuple[str, ...] = ()
|
||||
|
||||
def __call__(self, unraisable: "sys.UnraisableHookArgs") -> None:
|
||||
self.raised = (*self.raised, f"{unraisable.object!r}: {unraisable.exc_value!r}")
|
||||
|
||||
|
||||
def test_stream_auto_execute_dropped_after_its_event_loop_closed_discards_its_parked_log_cleanly(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
_serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_CHAT_ROUND_TWO)))
|
||||
recorder: Final = _ChatRoundRecorder()
|
||||
unraisable: Final = _UnraisableRecorder()
|
||||
opened: Final = queue.SimpleQueue[CustomStreamWrapper]()
|
||||
gc.collect()
|
||||
monkeypatch.setattr(sys, "unraisablehook", unraisable)
|
||||
|
||||
with _echo_tool_served(), warnings.catch_warnings(record=True) as caught:
|
||||
warnings.simplefilter("always")
|
||||
asyncio.run(
|
||||
_stream_read_until_the_follow_up_answers(
|
||||
_chat_rounds_logging_obj(recorder, "mcp-chat-stream-loop-closed", stream=True), opened
|
||||
)
|
||||
)
|
||||
opened.get_nowait()
|
||||
gc.collect()
|
||||
|
||||
assert recorder.logged == ()
|
||||
assert [raised for raised in unraisable.raised if _names_chat_stream_billing(raised)] == []
|
||||
assert [str(warning.message) for warning in caught if _names_chat_stream_billing(str(warning.message))] == []
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue