diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 6ab55772394..01bffd7d3f3 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -990,11 +990,11 @@ class Logging(LiteLLMLoggingBaseClass): self.llm_caching_handler = value @property - def _on_detached_stream_failure(self) -> Callable[[Exception], Awaitable[None]] | None: + def _on_detached_stream_failure(self) -> Callable[["Logging", Exception], Awaitable[None]] | None: return self.on_detached_stream_failure @_on_detached_stream_failure.setter - def _on_detached_stream_failure(self, value: Callable[[Exception], Awaitable[None]] | None) -> None: + def _on_detached_stream_failure(self, value: Callable[["Logging", Exception], Awaitable[None]] | None) -> None: self.on_detached_stream_failure = value @property @@ -1175,8 +1175,10 @@ class Logging(LiteLLMLoggingBaseClass): self._async_success_scheduled: bool = False self._async_success_enqueued: bool = False self._sync_success_scheduled: bool = False - self.on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = None + self.on_detached_stream_failure: Callable[[Logging, Exception], Awaitable[None]] | None = None self._stream_failure_claimed: bool = False + self._holding_stream_failure: bool = False + self._held_stream_failure: Exception | None = None self._stash_owed_on_success: bool = False self._stash_billed_by_continuation: bool = False self._failed_attempts_owed_on_success: tuple[ServedRound, ...] = () @@ -3172,6 +3174,30 @@ class Logging(LiteLLMLoggingBaseClass): if self.cost_breakdown is not None: self.cost_breakdown = _with_failed_attempts_cost(self.cost_breakdown, own_cost, failed_attempts_cost) + async def report_detached_stream_failure(self, exc: Exception) -> None: + """Fires the proxy's detached-stream failure hook, if one is armed, with this logging object. + While the proxy's client stream holds failures, the first one is kept for it instead.""" + hook: Final = self.on_detached_stream_failure + if hook is None: + return + if self._holding_stream_failure: + if self._held_stream_failure is None: + self._held_stream_failure = exc + return + await hook(self, exc) + + def hold_stream_failures(self) -> None: + """Defer detached-stream failure reports until the client stream ends, so a failure the + client stream reports itself claims the report first.""" + self._holding_stream_failure = True + + def release_held_stream_failure(self) -> Exception | None: + """Stop holding, and return the failure held meanwhile for the caller to report.""" + self._holding_stream_failure = False + held: Final = self._held_stream_failure + self._held_stream_failure = None + return held + def claim_stream_failure_report(self) -> bool: if self._stream_failure_claimed: return False diff --git a/litellm/llms/anthropic/messages/native/streaming_iterator.py b/litellm/llms/anthropic/messages/native/streaming_iterator.py index bd0de523ab5..006cdec9c69 100644 --- a/litellm/llms/anthropic/messages/native/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/native/streaming_iterator.py @@ -899,11 +899,8 @@ class BaseAnthropicMessagesStreamingIterator: async def _fire_detached_failure_hook(self, exc: Exception) -> None: from litellm._logging import verbose_proxy_logger - on_detached_failure: Final = getattr(self.litellm_logging_obj, "on_detached_stream_failure", None) - if on_detached_failure is None: - return try: - await on_detached_failure(exc) + await self.litellm_logging_obj.report_detached_stream_failure(exc) except Exception as hook_failure: # noqa: BLE001 # a failing proxy hook must not crash the detached pump verbose_proxy_logger.warning( "async_sse_wrapper detached failure hook raised: %s(%s)", type(hook_failure).__name__, hook_failure diff --git a/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py index d51360b4216..8bffab03e12 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py @@ -2,7 +2,7 @@ ## Translates OpenAI call to Anthropic `/v1/messages` format import asyncio from collections import deque -from collections.abc import AsyncGenerator, AsyncIterable, Awaitable, Callable, Iterable, Iterator, Mapping, Sequence +from collections.abc import AsyncGenerator, AsyncIterable, Iterable, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Final, Protocol, runtime_checkable import anyio @@ -486,14 +486,12 @@ class AnthropicResponsesStreamWrapper: if failure is None: return self._unreported_failure = None - on_failure: Final[Callable[[Exception], Awaitable[None]] | None] = getattr( - self.litellm_logging_obj, "_on_detached_stream_failure", None - ) - if on_failure is None: + logging_obj: Final = self.litellm_logging_obj + if logging_obj is None: return try: with anyio.CancelScope(shield=True): - await on_failure(failure) + await logging_obj.report_detached_stream_failure(failure) except Exception as hook_failure: # noqa: BLE001 # a failing proxy hook must not hide the error event from the client verbose_logger.warning( "AnthropicResponsesStreamWrapper: stream failure hook raised: %s(%s)", diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 1fdb51deb2c..5ecfcb71a02 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -5,7 +5,6 @@ import json import logging import math import time -import weakref from collections.abc import ( AsyncGenerator, AsyncIterator, @@ -97,6 +96,7 @@ from litellm.litellm_core_utils.core_helpers import ( redact_nested_match_and_regex_keys, require_attributes, stream_cost_injection_requested, + without, ) from litellm.litellm_core_utils.dd_tracing import NullTracer, tracer from litellm.litellm_core_utils.get_supported_openai_params import ( @@ -2728,47 +2728,35 @@ def _content_text(fields: Mapping[str, object]) -> str: return str(fields.get("content", "")) -async def _no_stream_failure_to_release() -> None: - return None +def _hold_stream_failures(logging_obj: LiteLLMLoggingObj | None) -> None: + if logging_obj is not None: + logging_obj.hold_stream_failures() -@dataclass(slots=True) -class _ClientStreamFailureReport: - logging_obj: LiteLLMLoggingObj - report: Callable[[Exception], Awaitable[None]] - held: Exception | None = None - - @classmethod - def hold_while_streaming(cls, logging_obj: LiteLLMLoggingObj | None) -> Callable[[], Awaitable[None]]: - if logging_obj is None or logging_obj.on_detached_stream_failure is None: - return _no_stream_failure_to_release - return cls(logging_obj, logging_obj.on_detached_stream_failure)._start_holding() - - def _start_holding(self) -> Callable[[], Awaitable[None]]: - self.logging_obj.on_detached_stream_failure = self.hold - return self.release - - async def hold(self, failure: Exception) -> None: - if self.held is None: - self.held = failure - - async def release(self) -> None: - self.logging_obj.on_detached_stream_failure = self.report - if self.held is None: - return - await asyncio.shield( - spawn_logging_task( - asyncio.get_running_loop(), self._report_held(self.held), task_name="stream failure report" - ) +async def _release_held_stream_failure(logging_obj: LiteLLMLoggingObj | None) -> None: + """Report the detached-stream failure held while the client stream ran, if the stream did not + report one itself.""" + if logging_obj is None: + return + held: Final = logging_obj.release_held_stream_failure() + if held is None: + return + await asyncio.shield( + spawn_logging_task( + asyncio.get_running_loop(), + _report_held_stream_failure(logging_obj, held), + task_name="stream failure report", ) + ) - async def _report_held(self, failure: Exception) -> None: - try: - await self.report(failure) - except Exception as hook_failure: # noqa: BLE001 # a failing failure hook must not skip the stream cleanup - verbose_proxy_logger.warning( - "Stream failure hook raised: %s(%s)", type(hook_failure).__name__, sanitize_for_log(hook_failure) - ) + +async def _report_held_stream_failure(logging_obj: LiteLLMLoggingObj, failure: Exception) -> None: + try: + await logging_obj.report_detached_stream_failure(failure) + except Exception as hook_failure: # noqa: BLE001 # a failing failure hook must not skip the stream cleanup + verbose_proxy_logger.warning( + "Stream failure hook raised: %s(%s)", type(hook_failure).__name__, sanitize_for_log(hook_failure) + ) def _metadata_logged_on(logged_on: LiteLLMLoggingObj) -> Mapping[str, object]: @@ -4480,22 +4468,19 @@ class ProxyBaseLLMRequestProcessing: a replacement error the hook raises has no client left to reach. """ # The closure lives on the logging object until the logging object dies, so it must not - # hold the logging object, or a request stays alive until a full GC pass (#51277) - logging_obj_ref: Final = weakref.ref(logging_obj) - request_data: Final = { - key: value for key, value in self._request_fields().items() if key != "litellm_logging_obj" - } + # hold the logging object, or a request stays alive until a full GC pass (#51277). The + # caller passes the logging object in when it fires. + request_data: Final = without(self._request_fields(), "litellm_logging_obj") route_type: Final = self.route_type - async def _on_detached_stream_failure(exc: Exception) -> None: - live_logging_obj: Final = logging_obj_ref() - if live_logging_obj is None or not live_logging_obj.claim_stream_failure_report(): + async def _on_detached_stream_failure(failed_logging_obj: LiteLLMLoggingObj, exc: Exception) -> None: + if not failed_logging_obj.claim_stream_failure_report(): return try: await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=exc, - request_data={**request_data, "litellm_logging_obj": live_logging_obj}, + request_data={**request_data, "litellm_logging_obj": failed_logging_obj}, dispatched_call_type=route_type, ) except HTTPException: @@ -5536,7 +5521,7 @@ class ProxyBaseLLMRequestProcessing: request_data=request_data, ) timing_sink: Final = stream_logging_obj if isinstance(stream_logging_obj, LiteLLMLoggingObj) else None - release_stream_failure: Final = _ClientStreamFailureReport.hold_while_streaming(timing_sink) + _hold_stream_failures(timing_sink) try: str_so_far = "" async for chunk in guarded_stream: @@ -5648,7 +5633,7 @@ class ProxyBaseLLMRequestProcessing: yield seal + error_frame if seal else error_frame finally: try: - await release_stream_failure() + await _release_held_stream_failure(timing_sink) finally: await ProxyBaseLLMRequestProcessing.finalize_streaming_generator_cleanup( request=request, diff --git a/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_mid_stream_error.py b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_mid_stream_error.py index 2cc0d86a3a8..2753e74c19a 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_mid_stream_error.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_mid_stream_error.py @@ -282,7 +282,7 @@ def _make_logging_obj( ) -async def _proxy_boundary_hook(exc: Exception) -> None: +async def _proxy_boundary_hook(_logging_obj: object, exc: Exception) -> None: return None diff --git a/tests/unit/llms/anthropic/messages/native/test_streaming_iterator.py b/tests/unit/llms/anthropic/messages/native/test_streaming_iterator.py index ded599512d0..59a4a882624 100644 --- a/tests/unit/llms/anthropic/messages/native/test_streaming_iterator.py +++ b/tests/unit/llms/anthropic/messages/native/test_streaming_iterator.py @@ -581,7 +581,7 @@ class _DetachedFailureRecorder: def __init__(self): self.exceptions = [] - async def __call__(self, exc: Exception) -> None: + async def __call__(self, _logging_obj: object, exc: Exception) -> None: self.exceptions.append(exc) diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_streaming_iterator.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_streaming_iterator.py index cb994fd612f..1e22ce1fc97 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_streaming_iterator.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_streaming_iterator.py @@ -707,11 +707,13 @@ class _FailureHookRecorder: def __init__(self) -> None: self.received: tuple[Exception, ...] = () - async def __call__(self, failure: Exception) -> None: + async def __call__(self, _logging_obj: object, failure: Exception) -> None: self.received = (*self.received, failure) -def _logging_obj_with_failure_hook(on_failure: Callable[[Exception], Awaitable[None]]) -> LitellmLogging: +def _logging_obj_with_failure_hook( + on_failure: Callable[[LitellmLogging, Exception], Awaitable[None]], +) -> LitellmLogging: logging_obj = LitellmLogging( model="m", messages=[{"role": "user", "content": "hi"}], @@ -740,7 +742,7 @@ class _DeliveryCountingConsumer: self.delivered: tuple[dict[str, object], ...] = () self.delivered_when_hook_fired: tuple[int, ...] = () - async def hook(self, failure: Exception) -> None: + async def hook(self, _logging_obj: object, failure: Exception) -> None: self.delivered_when_hook_fired = (*self.delivered_when_hook_fired, len(self.delivered)) async def drain(self, wrapper: AnthropicResponsesStreamWrapper) -> None: @@ -806,7 +808,7 @@ class TestUpstreamFailureReachesTheProxyFailureHook: assert "Rate limit reached for gpt-5.5, try again in 20s." in failure.message def test_a_raising_hook_never_hides_the_error_event_from_the_client(self): - async def _exploding_hook(failure: Exception) -> None: + async def _exploding_hook(_logging_obj: object, failure: Exception) -> None: raise RuntimeError("spend writer is down") logging_obj = _logging_obj_with_failure_hook(_exploding_hook) @@ -872,7 +874,7 @@ class TestUpstreamFailureReachesTheProxyFailureHook: hook_released = anyio.Event() finished: list[Exception] = [] - async def _held_hook(failure: Exception) -> None: + async def _held_hook(_logging_obj: object, failure: Exception) -> None: hook_started.set() await hook_released.wait() finished.append(failure) diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index fcf5f276516..c6c3a077b9f 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -13525,7 +13525,7 @@ class TestDetachedStreamFailureHook: ) failure = RuntimeError("upstream died after the client left") - await logging_obj.on_detached_stream_failure(failure) + await logging_obj.on_detached_stream_failure(logging_obj, failure) assert recorder.calls == [ { @@ -13550,7 +13550,7 @@ class TestDetachedStreamFailureHook: ) failure = RuntimeError("upstream died after the client left") - await logging_obj.on_detached_stream_failure(failure) + await logging_obj.on_detached_stream_failure(logging_obj, failure) assert [call["original_exception"] for call in recorder.calls] == [failure] @@ -13593,7 +13593,7 @@ class TestDetachedStreamFailureHook: def iterator_hook_for(logging_obj): async def iterator_hook(**_kwargs): yield b"event: message_start\n\n" - await logging_obj.on_detached_stream_failure(upstream_failure) + await logging_obj.report_detached_stream_failure(upstream_failure) yield b"event: error\n\n" yield b"event: message_stop\n\n" if after_report is not None: @@ -13696,7 +13696,7 @@ class TestDetachedStreamFailureHook: logging_obj, stream = self._armed_stream(recorder, lambda _logging_obj: iterator_hook) frames = [frame async for frame in stream] - await logging_obj.on_detached_stream_failure(upstream_failure) + await logging_obj.report_detached_stream_failure(upstream_failure) assert frames == ["data: chunk\n\n"] assert [call["original_exception"] for call in recorder.calls] == [upstream_failure] @@ -13733,7 +13733,7 @@ class TestDetachedStreamFailureHook: logging_obj, stream = self._armed_stream(recorder, lambda _logging_obj: iterator_hook) frames = [frame async for frame in stream] - await logging_obj.on_detached_stream_failure(upstream_failure) + await logging_obj.report_detached_stream_failure(upstream_failure) assert frames == ["data: chunk\n\n", "data: error\n\n"] assert [call["original_exception"] for call in recorder.calls] == [hook_failure]