diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 682b2a2f538..e4ceaa8c870 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -2468,7 +2468,7 @@ class CustomStreamWrapper: self._handle_stream_fallback_error(e) def _dispatch_stream_failure(self, e: Exception, traceback_exception: str) -> None: - self._record_partial_usage_for_failure() + self.record_partial_usage_for_failure() self._failure_logging = asyncio.create_task( self.logging_obj.dispatch_attempt_failure_handlers(e, traceback_exception) ) @@ -2486,7 +2486,7 @@ class CustomStreamWrapper: def bill_failure_after_earlier_rounds(self, earlier: "BilledRoundsSpend") -> None: self._earlier_rounds_spend = earlier - def _record_partial_usage_for_failure(self) -> None: + def record_partial_usage_for_failure(self) -> None: """ A stream that breaks mid-flight still billed the provider for the chunks already delivered. Recover that partial usage from the chunks seen so diff --git a/litellm/main.py b/litellm/main.py index 54afe654cec..74f996c8360 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -7988,6 +7988,9 @@ async def atext_completion(*args, **kwargs) -> TextCompletionResponse | TextComp stream_options=kwargs.get("stream_options"), usage_injected_by_proxy="_litellm_strip_stream_usage" in kwargs and kwargs["_litellm_strip_stream_usage"] is True, + source_stream=( + response if isinstance(response, (CustomStreamWrapper, TextCompletionStreamWrapper)) else None + ), ) else: ## OpenAI / Azure Text Completion Returns here diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f078267c6b2..e692ac43e11 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -953,6 +953,7 @@ from litellm.proxy.ui_crud_endpoints.user_banner_endpoints import ( router as user_banner_endpoints_router, ) from litellm.proxy.utils import ( # noqa: F401, RUF100 # legacy module exports + USAGE_RECOVERABLE_STREAMS, PrismaClient, ProxyLogging, ProxyUpdateSpend, @@ -10250,15 +10251,20 @@ async def _apply_streaming_chunk_hooks( *, chunk: Any, user_api_key_dict: UserAPIKeyAuth, - request_data: dict, + request_data: dict[str, object], str_so_far: str, + stream: object, ) -> tuple[object, str]: - chunk = await proxy_logging_obj.async_post_call_streaming_hook( - user_api_key_dict=user_api_key_dict, - response=chunk, - data=request_data, - str_so_far=str_so_far if str_so_far else None, - ) + try: + chunk = await proxy_logging_obj.async_post_call_streaming_hook( + user_api_key_dict=user_api_key_dict, + response=chunk, + data=request_data, + str_so_far=str_so_far if str_so_far else None, + ) + except Exception as e: + ProxyLogging.bill_stream_ended_by_proxy_error(request_data, stream, e) + raise if isinstance(chunk, (ModelResponse, ModelResponseStream)): response_str: Final = litellm.get_response_string(response_obj=chunk) @@ -10502,7 +10508,7 @@ def _make_keepalive_resolver(request_data: Mapping[str, object]) -> Callable[[ob async def async_data_generator( response, user_api_key_dict: UserAPIKeyAuth, - request_data: dict, + request_data: dict[str, object], request: Request | None = None, *, responses_stream_errors: bool = False, @@ -10562,6 +10568,7 @@ async def async_data_generator( # happens to start with it off. resolve_keepalive_seconds: Final = _make_keepalive_resolver(request_data) initial_keepalive_seconds: Final = resolve_keepalive_seconds(response) + usage_recoverable_stream: Final = response if isinstance(response, USAGE_RECOVERABLE_STREAMS) else None stream_source = ( _iter_with_keepalive( stream_iterator.__aiter__(), @@ -10592,6 +10599,7 @@ async def async_data_generator( user_api_key_dict=user_api_key_dict, request_data=request_data, str_so_far=_str_so_far, + stream=usage_recoverable_stream, ) stream_started = True diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index b1055bb9124..32d8ed87587 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -168,6 +168,7 @@ from litellm.litellm_core_utils.served_output_texts import ( record_served_output_texts, served_stream_output_texts, ) +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.litellm_core_utils.token_counter import offload_token_count, usage_tokenizer_model from litellm.llms import load_guardrail_translation_mappings from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler @@ -276,9 +277,11 @@ from litellm.repositories.user_repository import UserRepository from litellm.repositories.verification_token_repository import ( VerificationTokenRepository, ) +from litellm.router_utils.add_retry_fallback_headers import HiddenParamsAsyncIteratorWrapper from litellm.router_utils.common_utils import resolve_model_group_alias from litellm.router_utils.deployment_pricing import pricing_models from litellm.secret_managers.main import str_to_bool +from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper from litellm.types.integrations.slack_alerting import DEFAULT_ALERT_TYPES from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.mcp import ( @@ -1495,6 +1498,7 @@ object rather than ``True`` because ``request_data`` is the client body: JSON ca client can neither hide a guardrail block nor force a row by sending the key or a ``litellm_call_id`` itself.""" _PROXY_ONLY_LLM_API_ERRORS: Final = (HTTPException, ProxyException, GuardrailRaisedException) +USAGE_RECOVERABLE_STREAMS: Final = (CustomStreamWrapper, TextCompletionStreamWrapper, HiddenParamsAsyncIteratorWrapper) _PROXY_ONLY_ERROR_LOGGED_FIELDS: Final = frozenset( {"model", "messages", "input", "prompt", "user", "stream", "litellm_call_id"} ) @@ -4555,7 +4559,8 @@ class ProxyLogging: except (GeneratorExit, asyncio.CancelledError): raise except Exception as e: - if not ProxyLogging._discard_deferred_stream_logging_for_failure(request_data, e): + cut_stream: Final = response if isinstance(response, USAGE_RECOVERABLE_STREAMS) else None + if not ProxyLogging.bill_stream_ended_by_proxy_error(request_data, cut_stream, e): ProxyLogging.fire_deferred_stream_logging(request_data) raise ProxyLogging.fire_deferred_stream_logging(request_data) @@ -4644,7 +4649,8 @@ class ProxyLogging: raise except Exception as e: await ProxyLogging._record_served_stream_output(request_data, served_chunks) - if not ProxyLogging._discard_deferred_stream_logging_for_failure(request_data, e): + cut_provider_stream: Final = response if isinstance(response, USAGE_RECOVERABLE_STREAMS) else None + if not ProxyLogging.bill_stream_ended_by_proxy_error(request_data, cut_provider_stream, e): ProxyLogging.fire_deferred_stream_logging(request_data) raise @@ -4931,6 +4937,36 @@ class ProxyLogging: _fire_deferred_stream_logging = fire_deferred_stream_logging + @staticmethod + def bill_stream_ended_by_proxy_error(request_data: Mapping[str, object], stream: object, error: Exception) -> bool: + if ProxyLogging._discard_deferred_stream_logging_for_failure(request_data, error): + return True + ProxyLogging._record_cut_stream_usage_for_failure(request_data, stream, error) + return False + + @staticmethod + def _record_cut_stream_usage_for_failure( + request_data: Mapping[str, object], stream: object, error: Exception + ) -> None: + logging_obj: Final = request_data.get("litellm_logging_obj") + if not isinstance(error, _PROXY_ONLY_LLM_API_ERRORS) or not isinstance(logging_obj, Logging): + return + if getattr(logging_obj, "_on_deferred_stream_complete", None) is None: + return + if getattr(logging_obj, "_deferred_stream_complete_args", None) is not None: + return + ProxyLogging._record_partial_stream_usage(stream) + + @staticmethod + def _record_partial_stream_usage(stream: object) -> None: + match stream: + case CustomStreamWrapper() | TextCompletionStreamWrapper(): + stream.record_partial_usage_for_failure() + case HiddenParamsAsyncIteratorWrapper(inner=inner): + ProxyLogging._record_partial_stream_usage(inner) + case _: + return + @staticmethod def _discard_deferred_stream_logging_for_failure(request_data: Mapping[str, object], error: Exception) -> bool: """Drop the parked success dispatch for an assembled chat stream or a stream that billed several rounds when it diff --git a/litellm/text_completions/streaming_iterator.py b/litellm/text_completions/streaming_iterator.py index 61e47e9d7af..26bf5d24753 100644 --- a/litellm/text_completions/streaming_iterator.py +++ b/litellm/text_completions/streaming_iterator.py @@ -1,8 +1,11 @@ -from typing import Final +from typing import TYPE_CHECKING, Final from litellm.litellm_core_utils.stream_usage_chunks import is_injected_stream_usage_artifact from litellm.types.utils import Choices, ModelResponse, ModelResponseStream, TextChoices, TextCompletionResponse +if TYPE_CHECKING: + from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper + class TextCompletionStreamWrapper: def __init__( @@ -12,8 +15,10 @@ class TextCompletionStreamWrapper: stream_options: dict | None = None, custom_llm_provider: str | None = None, usage_injected_by_proxy: bool = False, + source_stream: "CustomStreamWrapper | TextCompletionStreamWrapper | None" = None, ): self.completion_stream = completion_stream + self.source_stream: Final = source_stream self.model = model self.stream_options = stream_options self.custom_llm_provider = custom_llm_provider @@ -24,6 +29,11 @@ class TextCompletionStreamWrapper: and stream_options.get("include_usage", False) is True ) + def record_partial_usage_for_failure(self) -> None: + if self.source_stream is None: + return + self.source_stream.record_partial_usage_for_failure() + def __iter__(self): return self diff --git a/tests/integration/streaming/test_chat_stream_guardrail_error_events.py b/tests/integration/streaming/test_chat_stream_guardrail_error_events.py index a8ebe510b8e..e3f01ada68c 100644 --- a/tests/integration/streaming/test_chat_stream_guardrail_error_events.py +++ b/tests/integration/streaming/test_chat_stream_guardrail_error_events.py @@ -2,7 +2,7 @@ import contextlib import json import re import uuid -from collections.abc import Iterator, Mapping, Sequence +from collections.abc import Callable, Iterator, Mapping, Sequence from dataclasses import dataclass from pathlib import Path from typing import Final @@ -36,6 +36,8 @@ _REPLACE: Final = "replace-client-error" _REPLACEMENT_MESSAGE: Final = "Request blocked by policy" _CRASH_MESSAGE: Final = "Guardrail service crashed while scanning" _RAISED_BLOCK: Final = "Raised block: policy" +_INPUT_COST_PER_TOKEN: Final = 0.001 +_OUTPUT_COST_PER_TOKEN: Final = 0.002 _SPAN: Final = "spanword" _LEAD: Final = "The lighthouse on the northern cliff has guided ships for two hundred years, and " _TEXTS: Final[Mapping[str, tuple[str, ...]]] = { @@ -187,6 +189,8 @@ def _config(upstream_url: str) -> dict[str, JsonValue]: "api_base": f"{upstream_url}/v1", "api_key": "synthetic-stream-key", "allow_client_keepalive_override": True, + "input_cost_per_token": _INPUT_COST_PER_TOKEN, + "output_cost_per_token": _OUTPUT_COST_PER_TOKEN, }, } ], @@ -424,6 +428,63 @@ def test_mid_stream_block_releases_the_parallel_request_slot_and_logs_one_failur assert [row["status"] for row in rows] == ["failure"], rows +def _spend_row(rig: Rig, call_id: str) -> dict[str, JsonValue]: + (row,) = eventually( + lambda: rig.proxy.gateway.rows( + 'SELECT status, prompt_tokens, completion_tokens, total_tokens, spend FROM "LiteLLM_SpendLogs" ' + "WHERE litellm_call_id = %s", + (call_id,), + ), + lambda found: len(found) == 1, + seconds=70, + ) + return row + + +def _completion_body(scenario: str, marker: str, guardrails: Sequence[str]) -> dict[str, JsonValue]: + return { + "model": _MODEL, + "stream": True, + "prompt": _prompt(scenario, marker), + "guardrails": list(guardrails), + "num_retries": 0, + "cache": {"no-cache": True}, + } + + +@pytest.mark.parametrize( + ("path", "body", "scenario", "guardrail"), + [ + pytest.param("/v1/chat/completions", _body, "filter-late", _FILTER, id="chat_stream_iterator_hook"), + pytest.param("/v1/chat/completions", _body, "span-late", _SPAN_POST, id="chat_per_chunk_hook"), + pytest.param( + "/v1/completions", _completion_body, "filter-late", _FILTER, id="completions_stream_iterator_hook" + ), + ], +) +def test_mid_stream_block_bills_what_the_provider_streamed_on_the_failure_row( + rig: Rig, path: str, body: Callable[[str, str, Sequence[str]], dict[str, JsonValue]], scenario: str, guardrail: str +) -> None: + marker: Final = _marker() + served: Final = rig.proxy.gateway.request("POST", path, body(scenario, marker, ())) + assert served.status_code == 200, served.text + assert all("error" not in payload for payload in _payloads(served.text)), served.text + blocked: Final = rig.proxy.gateway.request("POST", path, body(scenario, marker, (guardrail,))) + assert blocked.status_code == 200 and "error" in _payloads(blocked.text)[-1], blocked.text + + served_row: Final = _spend_row(rig, served.headers["x-litellm-call-id"]) + failed_row: Final = _spend_row(rig, blocked.headers["x-litellm-call-id"]) + assert (served_row["status"], failed_row["status"]) == ("success", "failure"), (served_row, failed_row) + prompt_tokens: Final = int(str(failed_row["prompt_tokens"])) + completion_tokens: Final = int(str(failed_row["completion_tokens"])) + assert prompt_tokens == served_row["prompt_tokens"] and prompt_tokens > 0, (served_row, failed_row) + assert 0 < completion_tokens <= int(str(served_row["completion_tokens"])), (served_row, failed_row) + assert failed_row["total_tokens"] == prompt_tokens + completion_tokens, failed_row + assert float(str(failed_row["spend"])) == pytest.approx( + prompt_tokens * _INPUT_COST_PER_TOKEN + completion_tokens * _OUTPUT_COST_PER_TOKEN + ), failed_row + + def test_failure_hook_replacement_hides_a_non_http_failure_mid_stream(rig: Rig) -> None: marker: Final = _marker() response: Final = _stream_chat(rig, _body("clean", marker, (_RAISING,), "raise-builtin-after-1", _REPLACE)) diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index b9c8feb223d..3cfde28d866 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -4985,7 +4985,7 @@ def test_record_partial_usage_for_failure_stashes_usage_and_cost(): ) ] - wrapper._record_partial_usage_for_failure() + wrapper.record_partial_usage_for_failure() stashed = logging_obj.model_call_details["combined_usage_object"] assert stashed.prompt_tokens == 30 @@ -5077,7 +5077,7 @@ def test_record_partial_usage_for_failure_noop_without_chunks(): ) wrapper.chunks = [] - wrapper._record_partial_usage_for_failure() + wrapper.record_partial_usage_for_failure() assert "combined_usage_object" not in logging_obj.model_call_details @@ -5135,7 +5135,7 @@ def test_record_partial_usage_for_failure_prices_alias_restamped_chunks_at_real_ ) assert "bedrock/bedrock-claude-opus-5" not in litellm.model_cost - wrapper._record_partial_usage_for_failure() + wrapper.record_partial_usage_for_failure() stashed = logging_obj.model_call_details["combined_usage_object"] assert stashed.completion_tokens == 5 @@ -5147,7 +5147,7 @@ def test_record_partial_usage_for_failure_prices_alias_restamped_chunks_at_real_ def test_record_partial_usage_for_failure_counts_prompt_tokens_from_request_messages(): wrapper, logging_obj = _wrapper_with_partial_chunks(chunk_model="my-public-alias") - wrapper._record_partial_usage_for_failure() + wrapper.record_partial_usage_for_failure() stashed = logging_obj.model_call_details["combined_usage_object"] assert stashed.prompt_tokens > 0 @@ -5156,7 +5156,7 @@ def test_record_partial_usage_for_failure_counts_prompt_tokens_from_request_mess def test_record_partial_usage_for_failure_backfills_missing_cache_fields(): wrapper, logging_obj = _wrapper_with_partial_chunks(chunk_model="gpt-4o-mini") - wrapper._record_partial_usage_for_failure() + wrapper.record_partial_usage_for_failure() stashed = logging_obj.model_call_details["combined_usage_object"] assert stashed.cache_creation_input_tokens == 0 @@ -5173,7 +5173,7 @@ def test_record_partial_usage_for_failure_prices_corrected_model_not_chunk_model custom_llm_provider="openai", ) - wrapper._record_partial_usage_for_failure() + wrapper.record_partial_usage_for_failure() rates = litellm.model_cost["gpt-4o-mini"] expected = 40 * rates["input_cost_per_token"] + 5 * rates["output_cost_per_token"] @@ -5191,7 +5191,7 @@ def test_record_partial_usage_for_failure_carries_up_openai_style_cached_tokens( chunk_model="gpt-4o-mini", usage=recovered ) - wrapper._record_partial_usage_for_failure() + wrapper.record_partial_usage_for_failure() stashed = logging_obj.model_call_details["combined_usage_object"] assert stashed.cache_read_input_tokens == 500 @@ -5210,7 +5210,7 @@ def test_record_partial_usage_for_failure_keeps_cache_values_recovered_from_chun chunk_model="gpt-4o-mini", usage=recovered ) - wrapper._record_partial_usage_for_failure() + wrapper.record_partial_usage_for_failure() stashed = logging_obj.model_call_details["combined_usage_object"] assert stashed.cache_read_input_tokens == 7 diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index c37475bd539..6e93cf536f6 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -32,6 +32,7 @@ import litellm import litellm.proxy.proxy_server as ps from litellm.constants import RETURN_RAW_MODEL_NAME_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.proxy._types import ProxyException, UserAPIKeyAuth from litellm.proxy.common_request_processing import create_response from litellm.proxy.proxy_server import ( @@ -602,6 +603,7 @@ async def test_apply_streaming_chunk_hooks_appends_to_str_so_far(monkeypatch): user_api_key_dict=_user_auth(), request_data={}, str_so_far="prior:", + stream=None, ) observed = { @@ -629,9 +631,46 @@ async def test_apply_streaming_chunk_hooks_hook_raises_exception(monkeypatch): user_api_key_dict=_user_auth(), request_data={}, str_so_far="", + stream=None, ) +@pytest.mark.asyncio +async def test_apply_streaming_chunk_hooks_block_bills_what_the_provider_already_streamed(monkeypatch): + stream: Final = await litellm.acompletion( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "Tell me about the lighthouse keeper"}], + stream=True, + mock_response="The lighthouse keeper lights the lamp at dusk.", + ) + assert isinstance(stream, CustomStreamWrapper) + success_dispatches: Final[list[object]] = [] + + async def _dispatch_success(*args: object) -> None: + success_dispatches.append(args) + + stream.logging_obj._on_deferred_stream_complete = _dispatch_success + first_chunk: Final = await anext(stream) + + async def _block(*, user_api_key_dict, response, data, str_so_far=None): + raise HTTPException(status_code=400, detail="blocked") + + monkeypatch.setattr(ps.proxy_logging_obj, "async_post_call_streaming_hook", _block) + + with pytest.raises(HTTPException): + await _apply_streaming_chunk_hooks( + chunk=first_chunk, + user_api_key_dict=_user_auth(), + request_data={"litellm_logging_obj": stream.logging_obj}, + str_so_far="", + stream=stream, + ) + + usage: Final = stream.logging_obj.model_call_details["combined_usage_object"] + assert (usage.prompt_tokens > 0, usage.completion_tokens > 0, success_dispatches) == (True, True, []) + assert stream.logging_obj.model_call_details["response_cost"] > 0 + + # --------------------------------------------------------------------------- # _format_streaming_sse_chunk # --------------------------------------------------------------------------- diff --git a/tests/unit/proxy/utils/proxy_logging/test_streaming_hooks.py b/tests/unit/proxy/utils/proxy_logging/test_streaming_hooks.py index 8f8f10565e9..fa1ff0abdec 100644 --- a/tests/unit/proxy/utils/proxy_logging/test_streaming_hooks.py +++ b/tests/unit/proxy/utils/proxy_logging/test_streaming_hooks.py @@ -23,11 +23,14 @@ from litellm.exceptions import GuardrailRaisedException from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.llms.anthropic.messages.native.streaming_iterator import ( BaseAnthropicMessagesStreamingIterator, ) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.utils import ProxyLogging +from litellm.router_utils.add_retry_fallback_headers import prepare_response_for_header_attachment +from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper from litellm.types.guardrails import GuardrailEventHooks from litellm.types.utils import Usage @@ -735,6 +738,139 @@ async def test_chat_stream_generic_callback_error_after_stream_end_still_flushes assert snapshot == {"events": ["success_dispatched"], "args_cleared": True, "failure_usage_recorded": False} +_CUT_PROMPT: Final = "Tell me about the lighthouse keeper" +_CUT_REPLY: Final = "The lighthouse keeper lights the lamp at dusk and keeps it burning until dawn." + + +async def _mock_provider_stream(endpoint: str) -> tuple[object, LiteLLMLoggingObj]: + match endpoint: + case "chat": + chat: Final = await litellm.acompletion( + model="gpt-4o-mini", + messages=[{"role": "user", "content": _CUT_PROMPT}], + stream=True, + mock_response=_CUT_REPLY, + ) + assert isinstance(chat, CustomStreamWrapper) + return chat, chat.logging_obj + case _: + text: Final = await litellm.atext_completion( + model="gpt-4o-mini", prompt=_CUT_PROMPT, stream=True, mock_response=_CUT_REPLY + ) + assert isinstance(text, TextCompletionStreamWrapper) + assert isinstance(text.source_stream, CustomStreamWrapper) + return prepare_response_for_header_attachment(text), text.source_stream.logging_obj + + +def _arm_success_dispatch(logging_obj: LiteLLMLoggingObj, events: list[str]) -> None: + async def _dispatch_success(*args: object) -> None: + events.append("success_dispatched") + + logging_obj._on_deferred_stream_complete = _dispatch_success + + +class _BlocksOnSecondChunk(CustomLogger): + def __init__(self, error: Exception) -> None: + super().__init__() + self.error: Final = error + + async def async_post_call_streaming_iterator_hook( + self, user_api_key_dict: UserAPIKeyAuth, response: AsyncIterator[object], request_data: dict[str, object] + ) -> AsyncGenerator[object, None]: + yield await anext(response) + await anext(response) + raise self.error + + +async def _raw_provider_stream() -> AsyncGenerator[dict[str, object], None]: + yield {"id": "c1", "choices": [{"index": 0, "delta": {"content": "The"}, "finish_reason": None}]} + yield {"id": "c1", "choices": [{"index": 0, "delta": {"content": " keeper"}, "finish_reason": None}]} + + +def _client_stream(shape: str, provider_stream: object) -> object: + match shape: + case "raw": + return _raw_provider_stream() + case "text_without_provider_stream": + return TextCompletionStreamWrapper(completion_stream=_raw_provider_stream(), model="gpt-4o-mini") + case _: + return provider_stream + + +async def _drain( + proxy_logging: ProxyLogging, stream: object, user_api_key_dict: UserAPIKeyAuth, logging_obj: LiteLLMLoggingObj +) -> None: + async for _ in proxy_logging.async_post_call_streaming_iterator_hook( + response=stream, + user_api_key_dict=user_api_key_dict, + request_data={"metadata": {}, "litellm_logging_obj": logging_obj}, + ): + pass + + +@pytest.mark.asyncio +@pytest.mark.parametrize("endpoint", ["chat", "text_completion"]) +async def test_stream_blocked_mid_stream_bills_what_the_provider_already_streamed( + proxy_logging: ProxyLogging, make_user_api_key_auth, monkeypatch: pytest.MonkeyPatch, endpoint: str +) -> None: + events: Final[list[str]] = [] + stream, logging_obj = await _mock_provider_stream(endpoint) + _arm_success_dispatch(logging_obj, events) + monkeypatch.setattr(litellm, "callbacks", [_BlocksOnSecondChunk(HTTPException(status_code=400, detail="blocked"))]) + + with pytest.raises(HTTPException): + await _drain(proxy_logging, stream, make_user_api_key_auth(), logging_obj) + await asyncio.sleep(0) + + usage: Final = logging_obj.model_call_details["combined_usage_object"] + rates: Final = litellm.model_cost["gpt-4o-mini"] + assert events == [] + assert usage.prompt_tokens > 0 + assert 0 < usage.completion_tokens < litellm.token_counter(model="gpt-4o-mini", text=_CUT_REPLY) + assert logging_obj.model_call_details["response_cost"] == pytest.approx( + usage.prompt_tokens * rates["input_cost_per_token"] + usage.completion_tokens * rates["output_cost_per_token"] + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("armed", "parked", "error", "shape", "expected_events"), + [ + pytest.param(False, None, HTTPException(status_code=400), "chat", [], id="success_dispatch_not_deferred"), + pytest.param(True, None, RuntimeError("callback crashed"), "chat", [], id="not_a_proxy_error"), + pytest.param(True, None, HTTPException(status_code=400), "raw", [], id="stream_without_recoverable_chunks"), + pytest.param( + True, None, HTTPException(status_code=400), "text_without_provider_stream", [], id="text_without_source" + ), + pytest.param( + True, (None, False), HTTPException(status_code=400), "chat", ["success_dispatched"], id="stream_completed" + ), + ], +) +async def test_stream_error_leaves_failure_usage_unset_unless_the_proxy_cut_a_recoverable_stream( + proxy_logging: ProxyLogging, + make_user_api_key_auth, + monkeypatch: pytest.MonkeyPatch, + armed: bool, + parked: tuple[None, bool] | None, + error: Exception, + shape: str, + expected_events: list[str], +) -> None: + events: Final[list[str]] = [] + stream, logging_obj = await _mock_provider_stream("chat") + if armed: + _arm_success_dispatch(logging_obj, events) + logging_obj._deferred_stream_complete_args = parked + monkeypatch.setattr(litellm, "callbacks", [_BlocksOnSecondChunk(error)]) + + with pytest.raises(type(error)): + await _drain(proxy_logging, _client_stream(shape, stream), make_user_api_key_auth(), logging_obj) + await asyncio.sleep(0) + + assert (events, "combined_usage_object" in logging_obj.model_call_details) == (expected_events, False) + + # --------------------------------------------------------------------------- # _fire_deferred_stream_logging # ---------------------------------------------------------------------------