refactor(proxy): keep the detached stream failure hook state on the logging object (#51962)

Follow-up to #51318. The detached stream failure hook lives on the logging object, so it must not
hold that object (#51277). Instead of a weakref, Logging.report_detached_stream_failure passes itself
to the hook, so the Anthropic stream iterators call it rather than reading the hook attribute. The
client stream's hold-until-it-ends latch moves onto Logging too (hold_stream_failures,
release_held_stream_failure), replacing _ClientStreamFailureReport and its swapping of the hook
attribute, and the hook's request data is copied with the without helper
This commit is contained in:
Nate Armstrong 2026-11-15 23:16:05 -08:00 • committed by GitHub
parent d034fb681c
commit 2f448a9ee8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 82 additions and 74 deletions

View file

@ -990,11 +990,11 @@ class Logging(LiteLLMLoggingBaseClass):
self.llm_caching_handler = value
@property
def _on_detached_stream_failure(self) -> Callable[[Exception], Awaitable[None]] | None:
def _on_detached_stream_failure(self) -> Callable[["Logging", Exception], Awaitable[None]] | None:
return self.on_detached_stream_failure
@_on_detached_stream_failure.setter
def _on_detached_stream_failure(self, value: Callable[[Exception], Awaitable[None]] | None) -> None:
def _on_detached_stream_failure(self, value: Callable[["Logging", Exception], Awaitable[None]] | None) -> None:
self.on_detached_stream_failure = value
@property
@ -1175,8 +1175,10 @@ class Logging(LiteLLMLoggingBaseClass):
self._async_success_scheduled: bool = False
self._async_success_enqueued: bool = False
self._sync_success_scheduled: bool = False
self.on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = None
self.on_detached_stream_failure: Callable[[Logging, Exception], Awaitable[None]] | None = None
self._stream_failure_claimed: bool = False
self._holding_stream_failure: bool = False
self._held_stream_failure: Exception | None = None
self._stash_owed_on_success: bool = False
self._stash_billed_by_continuation: bool = False
self._failed_attempts_owed_on_success: tuple[ServedRound, ...] = ()
@ -3172,6 +3174,30 @@ class Logging(LiteLLMLoggingBaseClass):
if self.cost_breakdown is not None:
self.cost_breakdown = _with_failed_attempts_cost(self.cost_breakdown, own_cost, failed_attempts_cost)
async def report_detached_stream_failure(self, exc: Exception) -> None:
"""Fires the proxy's detached-stream failure hook, if one is armed, with this logging object.
While the proxy's client stream holds failures, the first one is kept for it instead."""
hook: Final = self.on_detached_stream_failure
if hook is None:
return
if self._holding_stream_failure:
if self._held_stream_failure is None:
self._held_stream_failure = exc
return
await hook(self, exc)
def hold_stream_failures(self) -> None:
"""Defer detached-stream failure reports until the client stream ends, so a failure the
client stream reports itself claims the report first."""
self._holding_stream_failure = True
def release_held_stream_failure(self) -> Exception | None:
"""Stop holding, and return the failure held meanwhile for the caller to report."""
self._holding_stream_failure = False
held: Final = self._held_stream_failure
self._held_stream_failure = None
return held
def claim_stream_failure_report(self) -> bool:
if self._stream_failure_claimed:
return False

View file

@ -899,11 +899,8 @@ class BaseAnthropicMessagesStreamingIterator:
async def _fire_detached_failure_hook(self, exc: Exception) -> None:
from litellm._logging import verbose_proxy_logger
on_detached_failure: Final = getattr(self.litellm_logging_obj, "on_detached_stream_failure", None)
if on_detached_failure is None:
return
try:
await on_detached_failure(exc)
await self.litellm_logging_obj.report_detached_stream_failure(exc)
except Exception as hook_failure: # noqa: BLE001 # a failing proxy hook must not crash the detached pump
verbose_proxy_logger.warning(
"async_sse_wrapper detached failure hook raised: %s(%s)", type(hook_failure).__name__, hook_failure

View file

@ -2,7 +2,7 @@
## Translates OpenAI call to Anthropic `/v1/messages` format
import asyncio
from collections import deque
from collections.abc import AsyncGenerator, AsyncIterable, Awaitable, Callable, Iterable, Iterator, Mapping, Sequence
from collections.abc import AsyncGenerator, AsyncIterable, Iterable, Iterator, Mapping, Sequence
from typing import TYPE_CHECKING, Final, Protocol, runtime_checkable
import anyio
@ -486,14 +486,12 @@ class AnthropicResponsesStreamWrapper:
if failure is None:
return
self._unreported_failure = None
on_failure: Final[Callable[[Exception], Awaitable[None]] | None] = getattr(
self.litellm_logging_obj, "_on_detached_stream_failure", None
)
if on_failure is None:
logging_obj: Final = self.litellm_logging_obj
if logging_obj is None:
return
try:
with anyio.CancelScope(shield=True):
await on_failure(failure)
await logging_obj.report_detached_stream_failure(failure)
except Exception as hook_failure: # noqa: BLE001 # a failing proxy hook must not hide the error event from the client
verbose_logger.warning(
"AnthropicResponsesStreamWrapper: stream failure hook raised: %s(%s)",

View file

@ -5,7 +5,6 @@ import json
import logging
import math
import time
import weakref
from collections.abc import (
AsyncGenerator,
AsyncIterator,
@ -97,6 +96,7 @@ from litellm.litellm_core_utils.core_helpers import (
redact_nested_match_and_regex_keys,
require_attributes,
stream_cost_injection_requested,
without,
)
from litellm.litellm_core_utils.dd_tracing import NullTracer, tracer
from litellm.litellm_core_utils.get_supported_openai_params import (
@ -2728,47 +2728,35 @@ def _content_text(fields: Mapping[str, object]) -> str:
return str(fields.get("content", ""))
async def _no_stream_failure_to_release() -> None:
return None
def _hold_stream_failures(logging_obj: LiteLLMLoggingObj | None) -> None:
if logging_obj is not None:
logging_obj.hold_stream_failures()
@dataclass(slots=True)
class _ClientStreamFailureReport:
logging_obj: LiteLLMLoggingObj
report: Callable[[Exception], Awaitable[None]]
held: Exception | None = None
@classmethod
def hold_while_streaming(cls, logging_obj: LiteLLMLoggingObj | None) -> Callable[[], Awaitable[None]]:
if logging_obj is None or logging_obj.on_detached_stream_failure is None:
return _no_stream_failure_to_release
return cls(logging_obj, logging_obj.on_detached_stream_failure)._start_holding()
def _start_holding(self) -> Callable[[], Awaitable[None]]:
self.logging_obj.on_detached_stream_failure = self.hold
return self.release
async def hold(self, failure: Exception) -> None:
if self.held is None:
self.held = failure
async def release(self) -> None:
self.logging_obj.on_detached_stream_failure = self.report
if self.held is None:
return
await asyncio.shield(
spawn_logging_task(
asyncio.get_running_loop(), self._report_held(self.held), task_name="stream failure report"
)
async def _release_held_stream_failure(logging_obj: LiteLLMLoggingObj | None) -> None:
"""Report the detached-stream failure held while the client stream ran, if the stream did not
report one itself."""
if logging_obj is None:
return
held: Final = logging_obj.release_held_stream_failure()
if held is None:
return
await asyncio.shield(
spawn_logging_task(
asyncio.get_running_loop(),
_report_held_stream_failure(logging_obj, held),
task_name="stream failure report",
)
)
async def _report_held(self, failure: Exception) -> None:
try:
await self.report(failure)
except Exception as hook_failure: # noqa: BLE001 # a failing failure hook must not skip the stream cleanup
verbose_proxy_logger.warning(
"Stream failure hook raised: %s(%s)", type(hook_failure).__name__, sanitize_for_log(hook_failure)
)
async def _report_held_stream_failure(logging_obj: LiteLLMLoggingObj, failure: Exception) -> None:
try:
await logging_obj.report_detached_stream_failure(failure)
except Exception as hook_failure: # noqa: BLE001 # a failing failure hook must not skip the stream cleanup
verbose_proxy_logger.warning(
"Stream failure hook raised: %s(%s)", type(hook_failure).__name__, sanitize_for_log(hook_failure)
)
def _metadata_logged_on(logged_on: LiteLLMLoggingObj) -> Mapping[str, object]:
@ -4480,22 +4468,19 @@ class ProxyBaseLLMRequestProcessing:
a replacement error the hook raises has no client left to reach.
"""
# The closure lives on the logging object until the logging object dies, so it must not
# hold the logging object, or a request stays alive until a full GC pass (#51277)
logging_obj_ref: Final = weakref.ref(logging_obj)
request_data: Final = {
key: value for key, value in self._request_fields().items() if key != "litellm_logging_obj"
}
# hold the logging object, or a request stays alive until a full GC pass (#51277). The
# caller passes the logging object in when it fires.
request_data: Final = without(self._request_fields(), "litellm_logging_obj")
route_type: Final = self.route_type
async def _on_detached_stream_failure(exc: Exception) -> None:
live_logging_obj: Final = logging_obj_ref()
if live_logging_obj is None or not live_logging_obj.claim_stream_failure_report():
async def _on_detached_stream_failure(failed_logging_obj: LiteLLMLoggingObj, exc: Exception) -> None:
if not failed_logging_obj.claim_stream_failure_report():
return
try:
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
original_exception=exc,
request_data={**request_data, "litellm_logging_obj": live_logging_obj},
request_data={**request_data, "litellm_logging_obj": failed_logging_obj},
dispatched_call_type=route_type,
)
except HTTPException:
@ -5536,7 +5521,7 @@ class ProxyBaseLLMRequestProcessing:
request_data=request_data,
)
timing_sink: Final = stream_logging_obj if isinstance(stream_logging_obj, LiteLLMLoggingObj) else None
release_stream_failure: Final = _ClientStreamFailureReport.hold_while_streaming(timing_sink)
_hold_stream_failures(timing_sink)
try:
str_so_far = ""
async for chunk in guarded_stream:
@ -5648,7 +5633,7 @@ class ProxyBaseLLMRequestProcessing:
yield seal + error_frame if seal else error_frame
finally:
try:
await release_stream_failure()
await _release_held_stream_failure(timing_sink)
finally:
await ProxyBaseLLMRequestProcessing.finalize_streaming_generator_cleanup(
request=request,

View file

@ -282,7 +282,7 @@ def _make_logging_obj(
)
async def _proxy_boundary_hook(exc: Exception) -> None:
async def _proxy_boundary_hook(_logging_obj: object, exc: Exception) -> None:
return None

View file

@ -581,7 +581,7 @@ class _DetachedFailureRecorder:
def __init__(self):
self.exceptions = []
async def __call__(self, exc: Exception) -> None:
async def __call__(self, _logging_obj: object, exc: Exception) -> None:
self.exceptions.append(exc)

View file

@ -707,11 +707,13 @@ class _FailureHookRecorder:
def __init__(self) -> None:
self.received: tuple[Exception, ...] = ()
async def __call__(self, failure: Exception) -> None:
async def __call__(self, _logging_obj: object, failure: Exception) -> None:
self.received = (*self.received, failure)
def _logging_obj_with_failure_hook(on_failure: Callable[[Exception], Awaitable[None]]) -> LitellmLogging:
def _logging_obj_with_failure_hook(
on_failure: Callable[[LitellmLogging, Exception], Awaitable[None]],
) -> LitellmLogging:
logging_obj = LitellmLogging(
model="m",
messages=[{"role": "user", "content": "hi"}],
@ -740,7 +742,7 @@ class _DeliveryCountingConsumer:
self.delivered: tuple[dict[str, object], ...] = ()
self.delivered_when_hook_fired: tuple[int, ...] = ()
async def hook(self, failure: Exception) -> None:
async def hook(self, _logging_obj: object, failure: Exception) -> None:
self.delivered_when_hook_fired = (*self.delivered_when_hook_fired, len(self.delivered))
async def drain(self, wrapper: AnthropicResponsesStreamWrapper) -> None:
@ -806,7 +808,7 @@ class TestUpstreamFailureReachesTheProxyFailureHook:
assert "Rate limit reached for gpt-5.5, try again in 20s." in failure.message
def test_a_raising_hook_never_hides_the_error_event_from_the_client(self):
async def _exploding_hook(failure: Exception) -> None:
async def _exploding_hook(_logging_obj: object, failure: Exception) -> None:
raise RuntimeError("spend writer is down")
logging_obj = _logging_obj_with_failure_hook(_exploding_hook)
@ -872,7 +874,7 @@ class TestUpstreamFailureReachesTheProxyFailureHook:
hook_released = anyio.Event()
finished: list[Exception] = []
async def _held_hook(failure: Exception) -> None:
async def _held_hook(_logging_obj: object, failure: Exception) -> None:
hook_started.set()
await hook_released.wait()
finished.append(failure)

View file

@ -13525,7 +13525,7 @@ class TestDetachedStreamFailureHook:
)
failure = RuntimeError("upstream died after the client left")
await logging_obj.on_detached_stream_failure(failure)
await logging_obj.on_detached_stream_failure(logging_obj, failure)
assert recorder.calls == [
{
@ -13550,7 +13550,7 @@ class TestDetachedStreamFailureHook:
)
failure = RuntimeError("upstream died after the client left")
await logging_obj.on_detached_stream_failure(failure)
await logging_obj.on_detached_stream_failure(logging_obj, failure)
assert [call["original_exception"] for call in recorder.calls] == [failure]
@ -13593,7 +13593,7 @@ class TestDetachedStreamFailureHook:
def iterator_hook_for(logging_obj):
async def iterator_hook(**_kwargs):
yield b"event: message_start\n\n"
await logging_obj.on_detached_stream_failure(upstream_failure)
await logging_obj.report_detached_stream_failure(upstream_failure)
yield b"event: error\n\n"
yield b"event: message_stop\n\n"
if after_report is not None:
@ -13696,7 +13696,7 @@ class TestDetachedStreamFailureHook:
logging_obj, stream = self._armed_stream(recorder, lambda _logging_obj: iterator_hook)
frames = [frame async for frame in stream]
await logging_obj.on_detached_stream_failure(upstream_failure)
await logging_obj.report_detached_stream_failure(upstream_failure)
assert frames == ["data: chunk\n\n"]
assert [call["original_exception"] for call in recorder.calls] == [upstream_failure]
@ -13733,7 +13733,7 @@ class TestDetachedStreamFailureHook:
logging_obj, stream = self._armed_stream(recorder, lambda _logging_obj: iterator_hook)
frames = [frame async for frame in stream]
await logging_obj.on_detached_stream_failure(upstream_failure)
await logging_obj.report_detached_stream_failure(upstream_failure)
assert frames == ["data: chunk\n\n", "data: error\n\n"]
assert [call["original_exception"] for call in recorder.calls] == [hook_failure]