mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(proxy): bill the provider usage on a stream a post_call guardrail blocks mid-stream
A streamed /v1/chat/completions or /v1/completions request that a post_call guardrail cut mid-stream wrote its failure spend row with 0 tokens and $0, while the same block at the end of the stream billed the provider's usage. When the proxy cuts a stream before the deferred success dispatch was parked, both the iterator hook and the per-chunk hook now rebuild the usage and cost from the chunks already streamed (the provider's usage chunk when it arrived, else a token count) before the failure row is written. /v1/completions on a chat model reaches the inner stream through TextCompletionStreamWrapper.source_stream
This commit is contained in:
parent
02ef44c388
commit
de0fcff544
9 changed files with 315 additions and 22 deletions
|
|
@ -2468,7 +2468,7 @@ class CustomStreamWrapper:
|
|||
self._handle_stream_fallback_error(e)
|
||||
|
||||
def _dispatch_stream_failure(self, e: Exception, traceback_exception: str) -> None:
|
||||
self._record_partial_usage_for_failure()
|
||||
self.record_partial_usage_for_failure()
|
||||
self._failure_logging = asyncio.create_task(
|
||||
self.logging_obj.dispatch_attempt_failure_handlers(e, traceback_exception)
|
||||
)
|
||||
|
|
@ -2486,7 +2486,7 @@ class CustomStreamWrapper:
|
|||
def bill_failure_after_earlier_rounds(self, earlier: "BilledRoundsSpend") -> None:
|
||||
self._earlier_rounds_spend = earlier
|
||||
|
||||
def _record_partial_usage_for_failure(self) -> None:
|
||||
def record_partial_usage_for_failure(self) -> None:
|
||||
"""
|
||||
A stream that breaks mid-flight still billed the provider for the chunks
|
||||
already delivered. Recover that partial usage from the chunks seen so
|
||||
|
|
|
|||
|
|
@ -7988,6 +7988,9 @@ async def atext_completion(*args, **kwargs) -> TextCompletionResponse | TextComp
|
|||
stream_options=kwargs.get("stream_options"),
|
||||
usage_injected_by_proxy="_litellm_strip_stream_usage" in kwargs
|
||||
and kwargs["_litellm_strip_stream_usage"] is True,
|
||||
source_stream=(
|
||||
response if isinstance(response, (CustomStreamWrapper, TextCompletionStreamWrapper)) else None
|
||||
),
|
||||
)
|
||||
else:
|
||||
## OpenAI / Azure Text Completion Returns here
|
||||
|
|
|
|||
|
|
@ -953,6 +953,7 @@ from litellm.proxy.ui_crud_endpoints.user_banner_endpoints import (
|
|||
router as user_banner_endpoints_router,
|
||||
)
|
||||
from litellm.proxy.utils import ( # noqa: F401, RUF100 # legacy module exports
|
||||
USAGE_RECOVERABLE_STREAMS,
|
||||
PrismaClient,
|
||||
ProxyLogging,
|
||||
ProxyUpdateSpend,
|
||||
|
|
@ -10250,15 +10251,20 @@ async def _apply_streaming_chunk_hooks(
|
|||
*,
|
||||
chunk: Any,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
request_data: dict,
|
||||
request_data: dict[str, object],
|
||||
str_so_far: str,
|
||||
stream: object,
|
||||
) -> tuple[object, str]:
|
||||
chunk = await proxy_logging_obj.async_post_call_streaming_hook(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
response=chunk,
|
||||
data=request_data,
|
||||
str_so_far=str_so_far if str_so_far else None,
|
||||
)
|
||||
try:
|
||||
chunk = await proxy_logging_obj.async_post_call_streaming_hook(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
response=chunk,
|
||||
data=request_data,
|
||||
str_so_far=str_so_far if str_so_far else None,
|
||||
)
|
||||
except Exception as e:
|
||||
ProxyLogging.bill_stream_ended_by_proxy_error(request_data, stream, e)
|
||||
raise
|
||||
|
||||
if isinstance(chunk, (ModelResponse, ModelResponseStream)):
|
||||
response_str: Final = litellm.get_response_string(response_obj=chunk)
|
||||
|
|
@ -10502,7 +10508,7 @@ def _make_keepalive_resolver(request_data: Mapping[str, object]) -> Callable[[ob
|
|||
async def async_data_generator(
|
||||
response,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
request_data: dict,
|
||||
request_data: dict[str, object],
|
||||
request: Request | None = None,
|
||||
*,
|
||||
responses_stream_errors: bool = False,
|
||||
|
|
@ -10562,6 +10568,7 @@ async def async_data_generator(
|
|||
# happens to start with it off.
|
||||
resolve_keepalive_seconds: Final = _make_keepalive_resolver(request_data)
|
||||
initial_keepalive_seconds: Final = resolve_keepalive_seconds(response)
|
||||
usage_recoverable_stream: Final = response if isinstance(response, USAGE_RECOVERABLE_STREAMS) else None
|
||||
stream_source = (
|
||||
_iter_with_keepalive(
|
||||
stream_iterator.__aiter__(),
|
||||
|
|
@ -10592,6 +10599,7 @@ async def async_data_generator(
|
|||
user_api_key_dict=user_api_key_dict,
|
||||
request_data=request_data,
|
||||
str_so_far=_str_so_far,
|
||||
stream=usage_recoverable_stream,
|
||||
)
|
||||
stream_started = True
|
||||
|
||||
|
|
|
|||
|
|
@ -168,6 +168,7 @@ from litellm.litellm_core_utils.served_output_texts import (
|
|||
record_served_output_texts,
|
||||
served_stream_output_texts,
|
||||
)
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.litellm_core_utils.token_counter import offload_token_count, usage_tokenizer_model
|
||||
from litellm.llms import load_guardrail_translation_mappings
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
|
|
@ -276,9 +277,11 @@ from litellm.repositories.user_repository import UserRepository
|
|||
from litellm.repositories.verification_token_repository import (
|
||||
VerificationTokenRepository,
|
||||
)
|
||||
from litellm.router_utils.add_retry_fallback_headers import HiddenParamsAsyncIteratorWrapper
|
||||
from litellm.router_utils.common_utils import resolve_model_group_alias
|
||||
from litellm.router_utils.deployment_pricing import pricing_models
|
||||
from litellm.secret_managers.main import str_to_bool
|
||||
from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper
|
||||
from litellm.types.integrations.slack_alerting import DEFAULT_ALERT_TYPES
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
from litellm.types.mcp import (
|
||||
|
|
@ -1495,6 +1498,7 @@ object rather than ``True`` because ``request_data`` is the client body: JSON ca
|
|||
client can neither hide a guardrail block nor force a row by sending the key or a ``litellm_call_id`` itself."""
|
||||
|
||||
_PROXY_ONLY_LLM_API_ERRORS: Final = (HTTPException, ProxyException, GuardrailRaisedException)
|
||||
USAGE_RECOVERABLE_STREAMS: Final = (CustomStreamWrapper, TextCompletionStreamWrapper, HiddenParamsAsyncIteratorWrapper)
|
||||
_PROXY_ONLY_ERROR_LOGGED_FIELDS: Final = frozenset(
|
||||
{"model", "messages", "input", "prompt", "user", "stream", "litellm_call_id"}
|
||||
)
|
||||
|
|
@ -4555,7 +4559,8 @@ class ProxyLogging:
|
|||
except (GeneratorExit, asyncio.CancelledError):
|
||||
raise
|
||||
except Exception as e:
|
||||
if not ProxyLogging._discard_deferred_stream_logging_for_failure(request_data, e):
|
||||
cut_stream: Final = response if isinstance(response, USAGE_RECOVERABLE_STREAMS) else None
|
||||
if not ProxyLogging.bill_stream_ended_by_proxy_error(request_data, cut_stream, e):
|
||||
ProxyLogging.fire_deferred_stream_logging(request_data)
|
||||
raise
|
||||
ProxyLogging.fire_deferred_stream_logging(request_data)
|
||||
|
|
@ -4644,7 +4649,8 @@ class ProxyLogging:
|
|||
raise
|
||||
except Exception as e:
|
||||
await ProxyLogging._record_served_stream_output(request_data, served_chunks)
|
||||
if not ProxyLogging._discard_deferred_stream_logging_for_failure(request_data, e):
|
||||
cut_provider_stream: Final = response if isinstance(response, USAGE_RECOVERABLE_STREAMS) else None
|
||||
if not ProxyLogging.bill_stream_ended_by_proxy_error(request_data, cut_provider_stream, e):
|
||||
ProxyLogging.fire_deferred_stream_logging(request_data)
|
||||
raise
|
||||
|
||||
|
|
@ -4931,6 +4937,36 @@ class ProxyLogging:
|
|||
|
||||
_fire_deferred_stream_logging = fire_deferred_stream_logging
|
||||
|
||||
@staticmethod
|
||||
def bill_stream_ended_by_proxy_error(request_data: Mapping[str, object], stream: object, error: Exception) -> bool:
|
||||
if ProxyLogging._discard_deferred_stream_logging_for_failure(request_data, error):
|
||||
return True
|
||||
ProxyLogging._record_cut_stream_usage_for_failure(request_data, stream, error)
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _record_cut_stream_usage_for_failure(
|
||||
request_data: Mapping[str, object], stream: object, error: Exception
|
||||
) -> None:
|
||||
logging_obj: Final = request_data.get("litellm_logging_obj")
|
||||
if not isinstance(error, _PROXY_ONLY_LLM_API_ERRORS) or not isinstance(logging_obj, Logging):
|
||||
return
|
||||
if getattr(logging_obj, "_on_deferred_stream_complete", None) is None:
|
||||
return
|
||||
if getattr(logging_obj, "_deferred_stream_complete_args", None) is not None:
|
||||
return
|
||||
ProxyLogging._record_partial_stream_usage(stream)
|
||||
|
||||
@staticmethod
|
||||
def _record_partial_stream_usage(stream: object) -> None:
|
||||
match stream:
|
||||
case CustomStreamWrapper() | TextCompletionStreamWrapper():
|
||||
stream.record_partial_usage_for_failure()
|
||||
case HiddenParamsAsyncIteratorWrapper(inner=inner):
|
||||
ProxyLogging._record_partial_stream_usage(inner)
|
||||
case _:
|
||||
return
|
||||
|
||||
@staticmethod
|
||||
def _discard_deferred_stream_logging_for_failure(request_data: Mapping[str, object], error: Exception) -> bool:
|
||||
"""Drop the parked success dispatch for an assembled chat stream or a stream that billed several rounds when it
|
||||
|
|
|
|||
|
|
@ -1,8 +1,11 @@
|
|||
from typing import Final
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from litellm.litellm_core_utils.stream_usage_chunks import is_injected_stream_usage_artifact
|
||||
from litellm.types.utils import Choices, ModelResponse, ModelResponseStream, TextChoices, TextCompletionResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
|
||||
|
||||
class TextCompletionStreamWrapper:
|
||||
def __init__(
|
||||
|
|
@ -12,8 +15,10 @@ class TextCompletionStreamWrapper:
|
|||
stream_options: dict | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
usage_injected_by_proxy: bool = False,
|
||||
source_stream: "CustomStreamWrapper | TextCompletionStreamWrapper | None" = None,
|
||||
):
|
||||
self.completion_stream = completion_stream
|
||||
self.source_stream: Final = source_stream
|
||||
self.model = model
|
||||
self.stream_options = stream_options
|
||||
self.custom_llm_provider = custom_llm_provider
|
||||
|
|
@ -24,6 +29,11 @@ class TextCompletionStreamWrapper:
|
|||
and stream_options.get("include_usage", False) is True
|
||||
)
|
||||
|
||||
def record_partial_usage_for_failure(self) -> None:
|
||||
if self.source_stream is None:
|
||||
return
|
||||
self.source_stream.record_partial_usage_for_failure()
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import contextlib
|
|||
import json
|
||||
import re
|
||||
import uuid
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
from collections.abc import Callable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
|
@ -36,6 +36,8 @@ _REPLACE: Final = "replace-client-error"
|
|||
_REPLACEMENT_MESSAGE: Final = "Request blocked by policy"
|
||||
_CRASH_MESSAGE: Final = "Guardrail service crashed while scanning"
|
||||
_RAISED_BLOCK: Final = "Raised block: policy"
|
||||
_INPUT_COST_PER_TOKEN: Final = 0.001
|
||||
_OUTPUT_COST_PER_TOKEN: Final = 0.002
|
||||
_SPAN: Final = "spanword"
|
||||
_LEAD: Final = "The lighthouse on the northern cliff has guided ships for two hundred years, and "
|
||||
_TEXTS: Final[Mapping[str, tuple[str, ...]]] = {
|
||||
|
|
@ -187,6 +189,8 @@ def _config(upstream_url: str) -> dict[str, JsonValue]:
|
|||
"api_base": f"{upstream_url}/v1",
|
||||
"api_key": "synthetic-stream-key",
|
||||
"allow_client_keepalive_override": True,
|
||||
"input_cost_per_token": _INPUT_COST_PER_TOKEN,
|
||||
"output_cost_per_token": _OUTPUT_COST_PER_TOKEN,
|
||||
},
|
||||
}
|
||||
],
|
||||
|
|
@ -424,6 +428,63 @@ def test_mid_stream_block_releases_the_parallel_request_slot_and_logs_one_failur
|
|||
assert [row["status"] for row in rows] == ["failure"], rows
|
||||
|
||||
|
||||
def _spend_row(rig: Rig, call_id: str) -> dict[str, JsonValue]:
|
||||
(row,) = eventually(
|
||||
lambda: rig.proxy.gateway.rows(
|
||||
'SELECT status, prompt_tokens, completion_tokens, total_tokens, spend FROM "LiteLLM_SpendLogs" '
|
||||
"WHERE litellm_call_id = %s",
|
||||
(call_id,),
|
||||
),
|
||||
lambda found: len(found) == 1,
|
||||
seconds=70,
|
||||
)
|
||||
return row
|
||||
|
||||
|
||||
def _completion_body(scenario: str, marker: str, guardrails: Sequence[str]) -> dict[str, JsonValue]:
|
||||
return {
|
||||
"model": _MODEL,
|
||||
"stream": True,
|
||||
"prompt": _prompt(scenario, marker),
|
||||
"guardrails": list(guardrails),
|
||||
"num_retries": 0,
|
||||
"cache": {"no-cache": True},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("path", "body", "scenario", "guardrail"),
|
||||
[
|
||||
pytest.param("/v1/chat/completions", _body, "filter-late", _FILTER, id="chat_stream_iterator_hook"),
|
||||
pytest.param("/v1/chat/completions", _body, "span-late", _SPAN_POST, id="chat_per_chunk_hook"),
|
||||
pytest.param(
|
||||
"/v1/completions", _completion_body, "filter-late", _FILTER, id="completions_stream_iterator_hook"
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_mid_stream_block_bills_what_the_provider_streamed_on_the_failure_row(
|
||||
rig: Rig, path: str, body: Callable[[str, str, Sequence[str]], dict[str, JsonValue]], scenario: str, guardrail: str
|
||||
) -> None:
|
||||
marker: Final = _marker()
|
||||
served: Final = rig.proxy.gateway.request("POST", path, body(scenario, marker, ()))
|
||||
assert served.status_code == 200, served.text
|
||||
assert all("error" not in payload for payload in _payloads(served.text)), served.text
|
||||
blocked: Final = rig.proxy.gateway.request("POST", path, body(scenario, marker, (guardrail,)))
|
||||
assert blocked.status_code == 200 and "error" in _payloads(blocked.text)[-1], blocked.text
|
||||
|
||||
served_row: Final = _spend_row(rig, served.headers["x-litellm-call-id"])
|
||||
failed_row: Final = _spend_row(rig, blocked.headers["x-litellm-call-id"])
|
||||
assert (served_row["status"], failed_row["status"]) == ("success", "failure"), (served_row, failed_row)
|
||||
prompt_tokens: Final = int(str(failed_row["prompt_tokens"]))
|
||||
completion_tokens: Final = int(str(failed_row["completion_tokens"]))
|
||||
assert prompt_tokens == served_row["prompt_tokens"] and prompt_tokens > 0, (served_row, failed_row)
|
||||
assert 0 < completion_tokens <= int(str(served_row["completion_tokens"])), (served_row, failed_row)
|
||||
assert failed_row["total_tokens"] == prompt_tokens + completion_tokens, failed_row
|
||||
assert float(str(failed_row["spend"])) == pytest.approx(
|
||||
prompt_tokens * _INPUT_COST_PER_TOKEN + completion_tokens * _OUTPUT_COST_PER_TOKEN
|
||||
), failed_row
|
||||
|
||||
|
||||
def test_failure_hook_replacement_hides_a_non_http_failure_mid_stream(rig: Rig) -> None:
|
||||
marker: Final = _marker()
|
||||
response: Final = _stream_chat(rig, _body("clean", marker, (_RAISING,), "raise-builtin-after-1", _REPLACE))
|
||||
|
|
|
|||
|
|
@ -4985,7 +4985,7 @@ def test_record_partial_usage_for_failure_stashes_usage_and_cost():
|
|||
)
|
||||
]
|
||||
|
||||
wrapper._record_partial_usage_for_failure()
|
||||
wrapper.record_partial_usage_for_failure()
|
||||
|
||||
stashed = logging_obj.model_call_details["combined_usage_object"]
|
||||
assert stashed.prompt_tokens == 30
|
||||
|
|
@ -5077,7 +5077,7 @@ def test_record_partial_usage_for_failure_noop_without_chunks():
|
|||
)
|
||||
wrapper.chunks = []
|
||||
|
||||
wrapper._record_partial_usage_for_failure()
|
||||
wrapper.record_partial_usage_for_failure()
|
||||
|
||||
assert "combined_usage_object" not in logging_obj.model_call_details
|
||||
|
||||
|
|
@ -5135,7 +5135,7 @@ def test_record_partial_usage_for_failure_prices_alias_restamped_chunks_at_real_
|
|||
)
|
||||
assert "bedrock/bedrock-claude-opus-5" not in litellm.model_cost
|
||||
|
||||
wrapper._record_partial_usage_for_failure()
|
||||
wrapper.record_partial_usage_for_failure()
|
||||
|
||||
stashed = logging_obj.model_call_details["combined_usage_object"]
|
||||
assert stashed.completion_tokens == 5
|
||||
|
|
@ -5147,7 +5147,7 @@ def test_record_partial_usage_for_failure_prices_alias_restamped_chunks_at_real_
|
|||
def test_record_partial_usage_for_failure_counts_prompt_tokens_from_request_messages():
|
||||
wrapper, logging_obj = _wrapper_with_partial_chunks(chunk_model="my-public-alias")
|
||||
|
||||
wrapper._record_partial_usage_for_failure()
|
||||
wrapper.record_partial_usage_for_failure()
|
||||
|
||||
stashed = logging_obj.model_call_details["combined_usage_object"]
|
||||
assert stashed.prompt_tokens > 0
|
||||
|
|
@ -5156,7 +5156,7 @@ def test_record_partial_usage_for_failure_counts_prompt_tokens_from_request_mess
|
|||
def test_record_partial_usage_for_failure_backfills_missing_cache_fields():
|
||||
wrapper, logging_obj = _wrapper_with_partial_chunks(chunk_model="gpt-4o-mini")
|
||||
|
||||
wrapper._record_partial_usage_for_failure()
|
||||
wrapper.record_partial_usage_for_failure()
|
||||
|
||||
stashed = logging_obj.model_call_details["combined_usage_object"]
|
||||
assert stashed.cache_creation_input_tokens == 0
|
||||
|
|
@ -5173,7 +5173,7 @@ def test_record_partial_usage_for_failure_prices_corrected_model_not_chunk_model
|
|||
custom_llm_provider="openai",
|
||||
)
|
||||
|
||||
wrapper._record_partial_usage_for_failure()
|
||||
wrapper.record_partial_usage_for_failure()
|
||||
|
||||
rates = litellm.model_cost["gpt-4o-mini"]
|
||||
expected = 40 * rates["input_cost_per_token"] + 5 * rates["output_cost_per_token"]
|
||||
|
|
@ -5191,7 +5191,7 @@ def test_record_partial_usage_for_failure_carries_up_openai_style_cached_tokens(
|
|||
chunk_model="gpt-4o-mini", usage=recovered
|
||||
)
|
||||
|
||||
wrapper._record_partial_usage_for_failure()
|
||||
wrapper.record_partial_usage_for_failure()
|
||||
|
||||
stashed = logging_obj.model_call_details["combined_usage_object"]
|
||||
assert stashed.cache_read_input_tokens == 500
|
||||
|
|
@ -5210,7 +5210,7 @@ def test_record_partial_usage_for_failure_keeps_cache_values_recovered_from_chun
|
|||
chunk_model="gpt-4o-mini", usage=recovered
|
||||
)
|
||||
|
||||
wrapper._record_partial_usage_for_failure()
|
||||
wrapper.record_partial_usage_for_failure()
|
||||
|
||||
stashed = logging_obj.model_call_details["combined_usage_object"]
|
||||
assert stashed.cache_read_input_tokens == 7
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ import litellm
|
|||
import litellm.proxy.proxy_server as ps
|
||||
from litellm.constants import RETURN_RAW_MODEL_NAME_METADATA_KEY
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
|
||||
from litellm.proxy.common_request_processing import create_response
|
||||
from litellm.proxy.proxy_server import (
|
||||
|
|
@ -602,6 +603,7 @@ async def test_apply_streaming_chunk_hooks_appends_to_str_so_far(monkeypatch):
|
|||
user_api_key_dict=_user_auth(),
|
||||
request_data={},
|
||||
str_so_far="prior:",
|
||||
stream=None,
|
||||
)
|
||||
|
||||
observed = {
|
||||
|
|
@ -629,9 +631,46 @@ async def test_apply_streaming_chunk_hooks_hook_raises_exception(monkeypatch):
|
|||
user_api_key_dict=_user_auth(),
|
||||
request_data={},
|
||||
str_so_far="",
|
||||
stream=None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_streaming_chunk_hooks_block_bills_what_the_provider_already_streamed(monkeypatch):
|
||||
stream: Final = await litellm.acompletion(
|
||||
model="gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": "Tell me about the lighthouse keeper"}],
|
||||
stream=True,
|
||||
mock_response="The lighthouse keeper lights the lamp at dusk.",
|
||||
)
|
||||
assert isinstance(stream, CustomStreamWrapper)
|
||||
success_dispatches: Final[list[object]] = []
|
||||
|
||||
async def _dispatch_success(*args: object) -> None:
|
||||
success_dispatches.append(args)
|
||||
|
||||
stream.logging_obj._on_deferred_stream_complete = _dispatch_success
|
||||
first_chunk: Final = await anext(stream)
|
||||
|
||||
async def _block(*, user_api_key_dict, response, data, str_so_far=None):
|
||||
raise HTTPException(status_code=400, detail="blocked")
|
||||
|
||||
monkeypatch.setattr(ps.proxy_logging_obj, "async_post_call_streaming_hook", _block)
|
||||
|
||||
with pytest.raises(HTTPException):
|
||||
await _apply_streaming_chunk_hooks(
|
||||
chunk=first_chunk,
|
||||
user_api_key_dict=_user_auth(),
|
||||
request_data={"litellm_logging_obj": stream.logging_obj},
|
||||
str_so_far="",
|
||||
stream=stream,
|
||||
)
|
||||
|
||||
usage: Final = stream.logging_obj.model_call_details["combined_usage_object"]
|
||||
assert (usage.prompt_tokens > 0, usage.completion_tokens > 0, success_dispatches) == (True, True, [])
|
||||
assert stream.logging_obj.model_call_details["response_cost"] > 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _format_streaming_sse_chunk
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -23,11 +23,14 @@ from litellm.exceptions import GuardrailRaisedException
|
|||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.llms.anthropic.messages.native.streaming_iterator import (
|
||||
BaseAnthropicMessagesStreamingIterator,
|
||||
)
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
from litellm.router_utils.add_retry_fallback_headers import prepare_response_for_header_attachment
|
||||
from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
|
|
@ -735,6 +738,139 @@ async def test_chat_stream_generic_callback_error_after_stream_end_still_flushes
|
|||
assert snapshot == {"events": ["success_dispatched"], "args_cleared": True, "failure_usage_recorded": False}
|
||||
|
||||
|
||||
_CUT_PROMPT: Final = "Tell me about the lighthouse keeper"
|
||||
_CUT_REPLY: Final = "The lighthouse keeper lights the lamp at dusk and keeps it burning until dawn."
|
||||
|
||||
|
||||
async def _mock_provider_stream(endpoint: str) -> tuple[object, LiteLLMLoggingObj]:
|
||||
match endpoint:
|
||||
case "chat":
|
||||
chat: Final = await litellm.acompletion(
|
||||
model="gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": _CUT_PROMPT}],
|
||||
stream=True,
|
||||
mock_response=_CUT_REPLY,
|
||||
)
|
||||
assert isinstance(chat, CustomStreamWrapper)
|
||||
return chat, chat.logging_obj
|
||||
case _:
|
||||
text: Final = await litellm.atext_completion(
|
||||
model="gpt-4o-mini", prompt=_CUT_PROMPT, stream=True, mock_response=_CUT_REPLY
|
||||
)
|
||||
assert isinstance(text, TextCompletionStreamWrapper)
|
||||
assert isinstance(text.source_stream, CustomStreamWrapper)
|
||||
return prepare_response_for_header_attachment(text), text.source_stream.logging_obj
|
||||
|
||||
|
||||
def _arm_success_dispatch(logging_obj: LiteLLMLoggingObj, events: list[str]) -> None:
|
||||
async def _dispatch_success(*args: object) -> None:
|
||||
events.append("success_dispatched")
|
||||
|
||||
logging_obj._on_deferred_stream_complete = _dispatch_success
|
||||
|
||||
|
||||
class _BlocksOnSecondChunk(CustomLogger):
|
||||
def __init__(self, error: Exception) -> None:
|
||||
super().__init__()
|
||||
self.error: Final = error
|
||||
|
||||
async def async_post_call_streaming_iterator_hook(
|
||||
self, user_api_key_dict: UserAPIKeyAuth, response: AsyncIterator[object], request_data: dict[str, object]
|
||||
) -> AsyncGenerator[object, None]:
|
||||
yield await anext(response)
|
||||
await anext(response)
|
||||
raise self.error
|
||||
|
||||
|
||||
async def _raw_provider_stream() -> AsyncGenerator[dict[str, object], None]:
|
||||
yield {"id": "c1", "choices": [{"index": 0, "delta": {"content": "The"}, "finish_reason": None}]}
|
||||
yield {"id": "c1", "choices": [{"index": 0, "delta": {"content": " keeper"}, "finish_reason": None}]}
|
||||
|
||||
|
||||
def _client_stream(shape: str, provider_stream: object) -> object:
|
||||
match shape:
|
||||
case "raw":
|
||||
return _raw_provider_stream()
|
||||
case "text_without_provider_stream":
|
||||
return TextCompletionStreamWrapper(completion_stream=_raw_provider_stream(), model="gpt-4o-mini")
|
||||
case _:
|
||||
return provider_stream
|
||||
|
||||
|
||||
async def _drain(
|
||||
proxy_logging: ProxyLogging, stream: object, user_api_key_dict: UserAPIKeyAuth, logging_obj: LiteLLMLoggingObj
|
||||
) -> None:
|
||||
async for _ in proxy_logging.async_post_call_streaming_iterator_hook(
|
||||
response=stream,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
request_data={"metadata": {}, "litellm_logging_obj": logging_obj},
|
||||
):
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("endpoint", ["chat", "text_completion"])
|
||||
async def test_stream_blocked_mid_stream_bills_what_the_provider_already_streamed(
|
||||
proxy_logging: ProxyLogging, make_user_api_key_auth, monkeypatch: pytest.MonkeyPatch, endpoint: str
|
||||
) -> None:
|
||||
events: Final[list[str]] = []
|
||||
stream, logging_obj = await _mock_provider_stream(endpoint)
|
||||
_arm_success_dispatch(logging_obj, events)
|
||||
monkeypatch.setattr(litellm, "callbacks", [_BlocksOnSecondChunk(HTTPException(status_code=400, detail="blocked"))])
|
||||
|
||||
with pytest.raises(HTTPException):
|
||||
await _drain(proxy_logging, stream, make_user_api_key_auth(), logging_obj)
|
||||
await asyncio.sleep(0)
|
||||
|
||||
usage: Final = logging_obj.model_call_details["combined_usage_object"]
|
||||
rates: Final = litellm.model_cost["gpt-4o-mini"]
|
||||
assert events == []
|
||||
assert usage.prompt_tokens > 0
|
||||
assert 0 < usage.completion_tokens < litellm.token_counter(model="gpt-4o-mini", text=_CUT_REPLY)
|
||||
assert logging_obj.model_call_details["response_cost"] == pytest.approx(
|
||||
usage.prompt_tokens * rates["input_cost_per_token"] + usage.completion_tokens * rates["output_cost_per_token"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("armed", "parked", "error", "shape", "expected_events"),
|
||||
[
|
||||
pytest.param(False, None, HTTPException(status_code=400), "chat", [], id="success_dispatch_not_deferred"),
|
||||
pytest.param(True, None, RuntimeError("callback crashed"), "chat", [], id="not_a_proxy_error"),
|
||||
pytest.param(True, None, HTTPException(status_code=400), "raw", [], id="stream_without_recoverable_chunks"),
|
||||
pytest.param(
|
||||
True, None, HTTPException(status_code=400), "text_without_provider_stream", [], id="text_without_source"
|
||||
),
|
||||
pytest.param(
|
||||
True, (None, False), HTTPException(status_code=400), "chat", ["success_dispatched"], id="stream_completed"
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_stream_error_leaves_failure_usage_unset_unless_the_proxy_cut_a_recoverable_stream(
|
||||
proxy_logging: ProxyLogging,
|
||||
make_user_api_key_auth,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
armed: bool,
|
||||
parked: tuple[None, bool] | None,
|
||||
error: Exception,
|
||||
shape: str,
|
||||
expected_events: list[str],
|
||||
) -> None:
|
||||
events: Final[list[str]] = []
|
||||
stream, logging_obj = await _mock_provider_stream("chat")
|
||||
if armed:
|
||||
_arm_success_dispatch(logging_obj, events)
|
||||
logging_obj._deferred_stream_complete_args = parked
|
||||
monkeypatch.setattr(litellm, "callbacks", [_BlocksOnSecondChunk(error)])
|
||||
|
||||
with pytest.raises(type(error)):
|
||||
await _drain(proxy_logging, _client_stream(shape, stream), make_user_api_key_auth(), logging_obj)
|
||||
await asyncio.sleep(0)
|
||||
|
||||
assert (events, "combined_usage_object" in logging_obj.model_call_details) == (expected_events, False)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _fire_deferred_stream_logging
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue