fix(proxy): bill the provider usage on a stream a post_call guardrail blocks mid-stream

A streamed /v1/chat/completions or /v1/completions request that a post_call guardrail cut mid-stream wrote its
failure spend row with 0 tokens and $0, while the same block at the end of the stream billed the provider's usage.
When the proxy cuts a stream before the deferred success dispatch was parked, both the iterator hook and the
per-chunk hook now rebuild the usage and cost from the chunks already streamed (the provider's usage chunk when it
arrived, else a token count) before the failure row is written. /v1/completions on a chat model reaches the inner
stream through TextCompletionStreamWrapper.source_stream
This commit is contained in:
mateo-berri 2026-10-30 07:03:46 -07:00
parent 02ef44c388
commit de0fcff544
9 changed files with 315 additions and 22 deletions

View file

@ -2468,7 +2468,7 @@ class CustomStreamWrapper:
self._handle_stream_fallback_error(e)
def _dispatch_stream_failure(self, e: Exception, traceback_exception: str) -> None:
self._record_partial_usage_for_failure()
self.record_partial_usage_for_failure()
self._failure_logging = asyncio.create_task(
self.logging_obj.dispatch_attempt_failure_handlers(e, traceback_exception)
)
@ -2486,7 +2486,7 @@ class CustomStreamWrapper:
def bill_failure_after_earlier_rounds(self, earlier: "BilledRoundsSpend") -> None:
self._earlier_rounds_spend = earlier
def _record_partial_usage_for_failure(self) -> None:
def record_partial_usage_for_failure(self) -> None:
"""
A stream that breaks mid-flight still billed the provider for the chunks
already delivered. Recover that partial usage from the chunks seen so

View file

@ -7988,6 +7988,9 @@ async def atext_completion(*args, **kwargs) -> TextCompletionResponse | TextComp
stream_options=kwargs.get("stream_options"),
usage_injected_by_proxy="_litellm_strip_stream_usage" in kwargs
and kwargs["_litellm_strip_stream_usage"] is True,
source_stream=(
response if isinstance(response, (CustomStreamWrapper, TextCompletionStreamWrapper)) else None
),
)
else:
## OpenAI / Azure Text Completion Returns here

View file

@ -953,6 +953,7 @@ from litellm.proxy.ui_crud_endpoints.user_banner_endpoints import (
router as user_banner_endpoints_router,
)
from litellm.proxy.utils import ( # noqa: F401, RUF100 # legacy module exports
USAGE_RECOVERABLE_STREAMS,
PrismaClient,
ProxyLogging,
ProxyUpdateSpend,
@ -10250,15 +10251,20 @@ async def _apply_streaming_chunk_hooks(
*,
chunk: Any,
user_api_key_dict: UserAPIKeyAuth,
request_data: dict,
request_data: dict[str, object],
str_so_far: str,
stream: object,
) -> tuple[object, str]:
chunk = await proxy_logging_obj.async_post_call_streaming_hook(
user_api_key_dict=user_api_key_dict,
response=chunk,
data=request_data,
str_so_far=str_so_far if str_so_far else None,
)
try:
chunk = await proxy_logging_obj.async_post_call_streaming_hook(
user_api_key_dict=user_api_key_dict,
response=chunk,
data=request_data,
str_so_far=str_so_far if str_so_far else None,
)
except Exception as e:
ProxyLogging.bill_stream_ended_by_proxy_error(request_data, stream, e)
raise
if isinstance(chunk, (ModelResponse, ModelResponseStream)):
response_str: Final = litellm.get_response_string(response_obj=chunk)
@ -10502,7 +10508,7 @@ def _make_keepalive_resolver(request_data: Mapping[str, object]) -> Callable[[ob
async def async_data_generator(
response,
user_api_key_dict: UserAPIKeyAuth,
request_data: dict,
request_data: dict[str, object],
request: Request | None = None,
*,
responses_stream_errors: bool = False,
@ -10562,6 +10568,7 @@ async def async_data_generator(
# happens to start with it off.
resolve_keepalive_seconds: Final = _make_keepalive_resolver(request_data)
initial_keepalive_seconds: Final = resolve_keepalive_seconds(response)
usage_recoverable_stream: Final = response if isinstance(response, USAGE_RECOVERABLE_STREAMS) else None
stream_source = (
_iter_with_keepalive(
stream_iterator.__aiter__(),
@ -10592,6 +10599,7 @@ async def async_data_generator(
user_api_key_dict=user_api_key_dict,
request_data=request_data,
str_so_far=_str_so_far,
stream=usage_recoverable_stream,
)
stream_started = True

View file

@ -168,6 +168,7 @@ from litellm.litellm_core_utils.served_output_texts import (
record_served_output_texts,
served_stream_output_texts,
)
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.litellm_core_utils.token_counter import offload_token_count, usage_tokenizer_model
from litellm.llms import load_guardrail_translation_mappings
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
@ -276,9 +277,11 @@ from litellm.repositories.user_repository import UserRepository
from litellm.repositories.verification_token_repository import (
VerificationTokenRepository,
)
from litellm.router_utils.add_retry_fallback_headers import HiddenParamsAsyncIteratorWrapper
from litellm.router_utils.common_utils import resolve_model_group_alias
from litellm.router_utils.deployment_pricing import pricing_models
from litellm.secret_managers.main import str_to_bool
from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper
from litellm.types.integrations.slack_alerting import DEFAULT_ALERT_TYPES
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.mcp import (
@ -1495,6 +1498,7 @@ object rather than ``True`` because ``request_data`` is the client body: JSON ca
client can neither hide a guardrail block nor force a row by sending the key or a ``litellm_call_id`` itself."""
_PROXY_ONLY_LLM_API_ERRORS: Final = (HTTPException, ProxyException, GuardrailRaisedException)
USAGE_RECOVERABLE_STREAMS: Final = (CustomStreamWrapper, TextCompletionStreamWrapper, HiddenParamsAsyncIteratorWrapper)
_PROXY_ONLY_ERROR_LOGGED_FIELDS: Final = frozenset(
{"model", "messages", "input", "prompt", "user", "stream", "litellm_call_id"}
)
@ -4555,7 +4559,8 @@ class ProxyLogging:
except (GeneratorExit, asyncio.CancelledError):
raise
except Exception as e:
if not ProxyLogging._discard_deferred_stream_logging_for_failure(request_data, e):
cut_stream: Final = response if isinstance(response, USAGE_RECOVERABLE_STREAMS) else None
if not ProxyLogging.bill_stream_ended_by_proxy_error(request_data, cut_stream, e):
ProxyLogging.fire_deferred_stream_logging(request_data)
raise
ProxyLogging.fire_deferred_stream_logging(request_data)
@ -4644,7 +4649,8 @@ class ProxyLogging:
raise
except Exception as e:
await ProxyLogging._record_served_stream_output(request_data, served_chunks)
if not ProxyLogging._discard_deferred_stream_logging_for_failure(request_data, e):
cut_provider_stream: Final = response if isinstance(response, USAGE_RECOVERABLE_STREAMS) else None
if not ProxyLogging.bill_stream_ended_by_proxy_error(request_data, cut_provider_stream, e):
ProxyLogging.fire_deferred_stream_logging(request_data)
raise
@ -4931,6 +4937,36 @@ class ProxyLogging:
_fire_deferred_stream_logging = fire_deferred_stream_logging
@staticmethod
def bill_stream_ended_by_proxy_error(request_data: Mapping[str, object], stream: object, error: Exception) -> bool:
if ProxyLogging._discard_deferred_stream_logging_for_failure(request_data, error):
return True
ProxyLogging._record_cut_stream_usage_for_failure(request_data, stream, error)
return False
@staticmethod
def _record_cut_stream_usage_for_failure(
request_data: Mapping[str, object], stream: object, error: Exception
) -> None:
logging_obj: Final = request_data.get("litellm_logging_obj")
if not isinstance(error, _PROXY_ONLY_LLM_API_ERRORS) or not isinstance(logging_obj, Logging):
return
if getattr(logging_obj, "_on_deferred_stream_complete", None) is None:
return
if getattr(logging_obj, "_deferred_stream_complete_args", None) is not None:
return
ProxyLogging._record_partial_stream_usage(stream)
@staticmethod
def _record_partial_stream_usage(stream: object) -> None:
match stream:
case CustomStreamWrapper() | TextCompletionStreamWrapper():
stream.record_partial_usage_for_failure()
case HiddenParamsAsyncIteratorWrapper(inner=inner):
ProxyLogging._record_partial_stream_usage(inner)
case _:
return
@staticmethod
def _discard_deferred_stream_logging_for_failure(request_data: Mapping[str, object], error: Exception) -> bool:
"""Drop the parked success dispatch for an assembled chat stream or a stream that billed several rounds when it

View file

@ -1,8 +1,11 @@
from typing import Final
from typing import TYPE_CHECKING, Final
from litellm.litellm_core_utils.stream_usage_chunks import is_injected_stream_usage_artifact
from litellm.types.utils import Choices, ModelResponse, ModelResponseStream, TextChoices, TextCompletionResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
class TextCompletionStreamWrapper:
def __init__(
@ -12,8 +15,10 @@ class TextCompletionStreamWrapper:
stream_options: dict | None = None,
custom_llm_provider: str | None = None,
usage_injected_by_proxy: bool = False,
source_stream: "CustomStreamWrapper | TextCompletionStreamWrapper | None" = None,
):
self.completion_stream = completion_stream
self.source_stream: Final = source_stream
self.model = model
self.stream_options = stream_options
self.custom_llm_provider = custom_llm_provider
@ -24,6 +29,11 @@ class TextCompletionStreamWrapper:
and stream_options.get("include_usage", False) is True
)
def record_partial_usage_for_failure(self) -> None:
if self.source_stream is None:
return
self.source_stream.record_partial_usage_for_failure()
def __iter__(self):
return self

View file

@ -2,7 +2,7 @@ import contextlib
import json
import re
import uuid
from collections.abc import Iterator, Mapping, Sequence
from collections.abc import Callable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from typing import Final
@ -36,6 +36,8 @@ _REPLACE: Final = "replace-client-error"
_REPLACEMENT_MESSAGE: Final = "Request blocked by policy"
_CRASH_MESSAGE: Final = "Guardrail service crashed while scanning"
_RAISED_BLOCK: Final = "Raised block: policy"
_INPUT_COST_PER_TOKEN: Final = 0.001
_OUTPUT_COST_PER_TOKEN: Final = 0.002
_SPAN: Final = "spanword"
_LEAD: Final = "The lighthouse on the northern cliff has guided ships for two hundred years, and "
_TEXTS: Final[Mapping[str, tuple[str, ...]]] = {
@ -187,6 +189,8 @@ def _config(upstream_url: str) -> dict[str, JsonValue]:
"api_base": f"{upstream_url}/v1",
"api_key": "synthetic-stream-key",
"allow_client_keepalive_override": True,
"input_cost_per_token": _INPUT_COST_PER_TOKEN,
"output_cost_per_token": _OUTPUT_COST_PER_TOKEN,
},
}
],
@ -424,6 +428,63 @@ def test_mid_stream_block_releases_the_parallel_request_slot_and_logs_one_failur
assert [row["status"] for row in rows] == ["failure"], rows
def _spend_row(rig: Rig, call_id: str) -> dict[str, JsonValue]:
(row,) = eventually(
lambda: rig.proxy.gateway.rows(
'SELECT status, prompt_tokens, completion_tokens, total_tokens, spend FROM "LiteLLM_SpendLogs" '
"WHERE litellm_call_id = %s",
(call_id,),
),
lambda found: len(found) == 1,
seconds=70,
)
return row
def _completion_body(scenario: str, marker: str, guardrails: Sequence[str]) -> dict[str, JsonValue]:
return {
"model": _MODEL,
"stream": True,
"prompt": _prompt(scenario, marker),
"guardrails": list(guardrails),
"num_retries": 0,
"cache": {"no-cache": True},
}
@pytest.mark.parametrize(
("path", "body", "scenario", "guardrail"),
[
pytest.param("/v1/chat/completions", _body, "filter-late", _FILTER, id="chat_stream_iterator_hook"),
pytest.param("/v1/chat/completions", _body, "span-late", _SPAN_POST, id="chat_per_chunk_hook"),
pytest.param(
"/v1/completions", _completion_body, "filter-late", _FILTER, id="completions_stream_iterator_hook"
),
],
)
def test_mid_stream_block_bills_what_the_provider_streamed_on_the_failure_row(
rig: Rig, path: str, body: Callable[[str, str, Sequence[str]], dict[str, JsonValue]], scenario: str, guardrail: str
) -> None:
marker: Final = _marker()
served: Final = rig.proxy.gateway.request("POST", path, body(scenario, marker, ()))
assert served.status_code == 200, served.text
assert all("error" not in payload for payload in _payloads(served.text)), served.text
blocked: Final = rig.proxy.gateway.request("POST", path, body(scenario, marker, (guardrail,)))
assert blocked.status_code == 200 and "error" in _payloads(blocked.text)[-1], blocked.text
served_row: Final = _spend_row(rig, served.headers["x-litellm-call-id"])
failed_row: Final = _spend_row(rig, blocked.headers["x-litellm-call-id"])
assert (served_row["status"], failed_row["status"]) == ("success", "failure"), (served_row, failed_row)
prompt_tokens: Final = int(str(failed_row["prompt_tokens"]))
completion_tokens: Final = int(str(failed_row["completion_tokens"]))
assert prompt_tokens == served_row["prompt_tokens"] and prompt_tokens > 0, (served_row, failed_row)
assert 0 < completion_tokens <= int(str(served_row["completion_tokens"])), (served_row, failed_row)
assert failed_row["total_tokens"] == prompt_tokens + completion_tokens, failed_row
assert float(str(failed_row["spend"])) == pytest.approx(
prompt_tokens * _INPUT_COST_PER_TOKEN + completion_tokens * _OUTPUT_COST_PER_TOKEN
), failed_row
def test_failure_hook_replacement_hides_a_non_http_failure_mid_stream(rig: Rig) -> None:
marker: Final = _marker()
response: Final = _stream_chat(rig, _body("clean", marker, (_RAISING,), "raise-builtin-after-1", _REPLACE))

View file

@ -4985,7 +4985,7 @@ def test_record_partial_usage_for_failure_stashes_usage_and_cost():
)
]
wrapper._record_partial_usage_for_failure()
wrapper.record_partial_usage_for_failure()
stashed = logging_obj.model_call_details["combined_usage_object"]
assert stashed.prompt_tokens == 30
@ -5077,7 +5077,7 @@ def test_record_partial_usage_for_failure_noop_without_chunks():
)
wrapper.chunks = []
wrapper._record_partial_usage_for_failure()
wrapper.record_partial_usage_for_failure()
assert "combined_usage_object" not in logging_obj.model_call_details
@ -5135,7 +5135,7 @@ def test_record_partial_usage_for_failure_prices_alias_restamped_chunks_at_real_
)
assert "bedrock/bedrock-claude-opus-5" not in litellm.model_cost
wrapper._record_partial_usage_for_failure()
wrapper.record_partial_usage_for_failure()
stashed = logging_obj.model_call_details["combined_usage_object"]
assert stashed.completion_tokens == 5
@ -5147,7 +5147,7 @@ def test_record_partial_usage_for_failure_prices_alias_restamped_chunks_at_real_
def test_record_partial_usage_for_failure_counts_prompt_tokens_from_request_messages():
wrapper, logging_obj = _wrapper_with_partial_chunks(chunk_model="my-public-alias")
wrapper._record_partial_usage_for_failure()
wrapper.record_partial_usage_for_failure()
stashed = logging_obj.model_call_details["combined_usage_object"]
assert stashed.prompt_tokens > 0
@ -5156,7 +5156,7 @@ def test_record_partial_usage_for_failure_counts_prompt_tokens_from_request_mess
def test_record_partial_usage_for_failure_backfills_missing_cache_fields():
wrapper, logging_obj = _wrapper_with_partial_chunks(chunk_model="gpt-4o-mini")
wrapper._record_partial_usage_for_failure()
wrapper.record_partial_usage_for_failure()
stashed = logging_obj.model_call_details["combined_usage_object"]
assert stashed.cache_creation_input_tokens == 0
@ -5173,7 +5173,7 @@ def test_record_partial_usage_for_failure_prices_corrected_model_not_chunk_model
custom_llm_provider="openai",
)
wrapper._record_partial_usage_for_failure()
wrapper.record_partial_usage_for_failure()
rates = litellm.model_cost["gpt-4o-mini"]
expected = 40 * rates["input_cost_per_token"] + 5 * rates["output_cost_per_token"]
@ -5191,7 +5191,7 @@ def test_record_partial_usage_for_failure_carries_up_openai_style_cached_tokens(
chunk_model="gpt-4o-mini", usage=recovered
)
wrapper._record_partial_usage_for_failure()
wrapper.record_partial_usage_for_failure()
stashed = logging_obj.model_call_details["combined_usage_object"]
assert stashed.cache_read_input_tokens == 500
@ -5210,7 +5210,7 @@ def test_record_partial_usage_for_failure_keeps_cache_values_recovered_from_chun
chunk_model="gpt-4o-mini", usage=recovered
)
wrapper._record_partial_usage_for_failure()
wrapper.record_partial_usage_for_failure()
stashed = logging_obj.model_call_details["combined_usage_object"]
assert stashed.cache_read_input_tokens == 7

View file

@ -32,6 +32,7 @@ import litellm
import litellm.proxy.proxy_server as ps
from litellm.constants import RETURN_RAW_MODEL_NAME_METADATA_KEY
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
from litellm.proxy.common_request_processing import create_response
from litellm.proxy.proxy_server import (
@ -602,6 +603,7 @@ async def test_apply_streaming_chunk_hooks_appends_to_str_so_far(monkeypatch):
user_api_key_dict=_user_auth(),
request_data={},
str_so_far="prior:",
stream=None,
)
observed = {
@ -629,9 +631,46 @@ async def test_apply_streaming_chunk_hooks_hook_raises_exception(monkeypatch):
user_api_key_dict=_user_auth(),
request_data={},
str_so_far="",
stream=None,
)
@pytest.mark.asyncio
async def test_apply_streaming_chunk_hooks_block_bills_what_the_provider_already_streamed(monkeypatch):
stream: Final = await litellm.acompletion(
model="gpt-4o-mini",
messages=[{"role": "user", "content": "Tell me about the lighthouse keeper"}],
stream=True,
mock_response="The lighthouse keeper lights the lamp at dusk.",
)
assert isinstance(stream, CustomStreamWrapper)
success_dispatches: Final[list[object]] = []
async def _dispatch_success(*args: object) -> None:
success_dispatches.append(args)
stream.logging_obj._on_deferred_stream_complete = _dispatch_success
first_chunk: Final = await anext(stream)
async def _block(*, user_api_key_dict, response, data, str_so_far=None):
raise HTTPException(status_code=400, detail="blocked")
monkeypatch.setattr(ps.proxy_logging_obj, "async_post_call_streaming_hook", _block)
with pytest.raises(HTTPException):
await _apply_streaming_chunk_hooks(
chunk=first_chunk,
user_api_key_dict=_user_auth(),
request_data={"litellm_logging_obj": stream.logging_obj},
str_so_far="",
stream=stream,
)
usage: Final = stream.logging_obj.model_call_details["combined_usage_object"]
assert (usage.prompt_tokens > 0, usage.completion_tokens > 0, success_dispatches) == (True, True, [])
assert stream.logging_obj.model_call_details["response_cost"] > 0
# ---------------------------------------------------------------------------
# _format_streaming_sse_chunk
# ---------------------------------------------------------------------------

View file

@ -23,11 +23,14 @@ from litellm.exceptions import GuardrailRaisedException
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.llms.anthropic.messages.native.streaming_iterator import (
BaseAnthropicMessagesStreamingIterator,
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.utils import ProxyLogging
from litellm.router_utils.add_retry_fallback_headers import prepare_response_for_header_attachment
from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.utils import Usage
@ -735,6 +738,139 @@ async def test_chat_stream_generic_callback_error_after_stream_end_still_flushes
assert snapshot == {"events": ["success_dispatched"], "args_cleared": True, "failure_usage_recorded": False}
_CUT_PROMPT: Final = "Tell me about the lighthouse keeper"
_CUT_REPLY: Final = "The lighthouse keeper lights the lamp at dusk and keeps it burning until dawn."
async def _mock_provider_stream(endpoint: str) -> tuple[object, LiteLLMLoggingObj]:
match endpoint:
case "chat":
chat: Final = await litellm.acompletion(
model="gpt-4o-mini",
messages=[{"role": "user", "content": _CUT_PROMPT}],
stream=True,
mock_response=_CUT_REPLY,
)
assert isinstance(chat, CustomStreamWrapper)
return chat, chat.logging_obj
case _:
text: Final = await litellm.atext_completion(
model="gpt-4o-mini", prompt=_CUT_PROMPT, stream=True, mock_response=_CUT_REPLY
)
assert isinstance(text, TextCompletionStreamWrapper)
assert isinstance(text.source_stream, CustomStreamWrapper)
return prepare_response_for_header_attachment(text), text.source_stream.logging_obj
def _arm_success_dispatch(logging_obj: LiteLLMLoggingObj, events: list[str]) -> None:
async def _dispatch_success(*args: object) -> None:
events.append("success_dispatched")
logging_obj._on_deferred_stream_complete = _dispatch_success
class _BlocksOnSecondChunk(CustomLogger):
def __init__(self, error: Exception) -> None:
super().__init__()
self.error: Final = error
async def async_post_call_streaming_iterator_hook(
self, user_api_key_dict: UserAPIKeyAuth, response: AsyncIterator[object], request_data: dict[str, object]
) -> AsyncGenerator[object, None]:
yield await anext(response)
await anext(response)
raise self.error
async def _raw_provider_stream() -> AsyncGenerator[dict[str, object], None]:
yield {"id": "c1", "choices": [{"index": 0, "delta": {"content": "The"}, "finish_reason": None}]}
yield {"id": "c1", "choices": [{"index": 0, "delta": {"content": " keeper"}, "finish_reason": None}]}
def _client_stream(shape: str, provider_stream: object) -> object:
match shape:
case "raw":
return _raw_provider_stream()
case "text_without_provider_stream":
return TextCompletionStreamWrapper(completion_stream=_raw_provider_stream(), model="gpt-4o-mini")
case _:
return provider_stream
async def _drain(
proxy_logging: ProxyLogging, stream: object, user_api_key_dict: UserAPIKeyAuth, logging_obj: LiteLLMLoggingObj
) -> None:
async for _ in proxy_logging.async_post_call_streaming_iterator_hook(
response=stream,
user_api_key_dict=user_api_key_dict,
request_data={"metadata": {}, "litellm_logging_obj": logging_obj},
):
pass
@pytest.mark.asyncio
@pytest.mark.parametrize("endpoint", ["chat", "text_completion"])
async def test_stream_blocked_mid_stream_bills_what_the_provider_already_streamed(
proxy_logging: ProxyLogging, make_user_api_key_auth, monkeypatch: pytest.MonkeyPatch, endpoint: str
) -> None:
events: Final[list[str]] = []
stream, logging_obj = await _mock_provider_stream(endpoint)
_arm_success_dispatch(logging_obj, events)
monkeypatch.setattr(litellm, "callbacks", [_BlocksOnSecondChunk(HTTPException(status_code=400, detail="blocked"))])
with pytest.raises(HTTPException):
await _drain(proxy_logging, stream, make_user_api_key_auth(), logging_obj)
await asyncio.sleep(0)
usage: Final = logging_obj.model_call_details["combined_usage_object"]
rates: Final = litellm.model_cost["gpt-4o-mini"]
assert events == []
assert usage.prompt_tokens > 0
assert 0 < usage.completion_tokens < litellm.token_counter(model="gpt-4o-mini", text=_CUT_REPLY)
assert logging_obj.model_call_details["response_cost"] == pytest.approx(
usage.prompt_tokens * rates["input_cost_per_token"] + usage.completion_tokens * rates["output_cost_per_token"]
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("armed", "parked", "error", "shape", "expected_events"),
[
pytest.param(False, None, HTTPException(status_code=400), "chat", [], id="success_dispatch_not_deferred"),
pytest.param(True, None, RuntimeError("callback crashed"), "chat", [], id="not_a_proxy_error"),
pytest.param(True, None, HTTPException(status_code=400), "raw", [], id="stream_without_recoverable_chunks"),
pytest.param(
True, None, HTTPException(status_code=400), "text_without_provider_stream", [], id="text_without_source"
),
pytest.param(
True, (None, False), HTTPException(status_code=400), "chat", ["success_dispatched"], id="stream_completed"
),
],
)
async def test_stream_error_leaves_failure_usage_unset_unless_the_proxy_cut_a_recoverable_stream(
proxy_logging: ProxyLogging,
make_user_api_key_auth,
monkeypatch: pytest.MonkeyPatch,
armed: bool,
parked: tuple[None, bool] | None,
error: Exception,
shape: str,
expected_events: list[str],
) -> None:
events: Final[list[str]] = []
stream, logging_obj = await _mock_provider_stream("chat")
if armed:
_arm_success_dispatch(logging_obj, events)
logging_obj._deferred_stream_complete_args = parked
monkeypatch.setattr(litellm, "callbacks", [_BlocksOnSecondChunk(error)])
with pytest.raises(type(error)):
await _drain(proxy_logging, _client_stream(shape, stream), make_user_api_key_auth(), logging_obj)
await asyncio.sleep(0)
assert (events, "combined_usage_object" in logging_obj.model_call_details) == (expected_events, False)
# ---------------------------------------------------------------------------
# _fire_deferred_stream_logging
# ---------------------------------------------------------------------------