mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
refactor(proxy): keep the detached stream failure hook state on the logging object (#51962)
Follow-up to #51318. The detached stream failure hook lives on the logging object, so it must not hold that object (#51277). Instead of a weakref, Logging.report_detached_stream_failure passes itself to the hook, so the Anthropic stream iterators call it rather than reading the hook attribute. The client stream's hold-until-it-ends latch moves onto Logging too (hold_stream_failures, release_held_stream_failure), replacing _ClientStreamFailureReport and its swapping of the hook attribute, and the hook's request data is copied with the without helper
This commit is contained in:
parent
d034fb681c
commit
2f448a9ee8
8 changed files with 82 additions and 74 deletions
|
|
@ -990,11 +990,11 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
self.llm_caching_handler = value
|
||||
|
||||
@property
|
||||
def _on_detached_stream_failure(self) -> Callable[[Exception], Awaitable[None]] | None:
|
||||
def _on_detached_stream_failure(self) -> Callable[["Logging", Exception], Awaitable[None]] | None:
|
||||
return self.on_detached_stream_failure
|
||||
|
||||
@_on_detached_stream_failure.setter
|
||||
def _on_detached_stream_failure(self, value: Callable[[Exception], Awaitable[None]] | None) -> None:
|
||||
def _on_detached_stream_failure(self, value: Callable[["Logging", Exception], Awaitable[None]] | None) -> None:
|
||||
self.on_detached_stream_failure = value
|
||||
|
||||
@property
|
||||
|
|
@ -1175,8 +1175,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
self._async_success_scheduled: bool = False
|
||||
self._async_success_enqueued: bool = False
|
||||
self._sync_success_scheduled: bool = False
|
||||
self.on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = None
|
||||
self.on_detached_stream_failure: Callable[[Logging, Exception], Awaitable[None]] | None = None
|
||||
self._stream_failure_claimed: bool = False
|
||||
self._holding_stream_failure: bool = False
|
||||
self._held_stream_failure: Exception | None = None
|
||||
self._stash_owed_on_success: bool = False
|
||||
self._stash_billed_by_continuation: bool = False
|
||||
self._failed_attempts_owed_on_success: tuple[ServedRound, ...] = ()
|
||||
|
|
@ -3172,6 +3174,30 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
if self.cost_breakdown is not None:
|
||||
self.cost_breakdown = _with_failed_attempts_cost(self.cost_breakdown, own_cost, failed_attempts_cost)
|
||||
|
||||
async def report_detached_stream_failure(self, exc: Exception) -> None:
|
||||
"""Fires the proxy's detached-stream failure hook, if one is armed, with this logging object.
|
||||
While the proxy's client stream holds failures, the first one is kept for it instead."""
|
||||
hook: Final = self.on_detached_stream_failure
|
||||
if hook is None:
|
||||
return
|
||||
if self._holding_stream_failure:
|
||||
if self._held_stream_failure is None:
|
||||
self._held_stream_failure = exc
|
||||
return
|
||||
await hook(self, exc)
|
||||
|
||||
def hold_stream_failures(self) -> None:
|
||||
"""Defer detached-stream failure reports until the client stream ends, so a failure the
|
||||
client stream reports itself claims the report first."""
|
||||
self._holding_stream_failure = True
|
||||
|
||||
def release_held_stream_failure(self) -> Exception | None:
|
||||
"""Stop holding, and return the failure held meanwhile for the caller to report."""
|
||||
self._holding_stream_failure = False
|
||||
held: Final = self._held_stream_failure
|
||||
self._held_stream_failure = None
|
||||
return held
|
||||
|
||||
def claim_stream_failure_report(self) -> bool:
|
||||
if self._stream_failure_claimed:
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -899,11 +899,8 @@ class BaseAnthropicMessagesStreamingIterator:
|
|||
async def _fire_detached_failure_hook(self, exc: Exception) -> None:
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
||||
on_detached_failure: Final = getattr(self.litellm_logging_obj, "on_detached_stream_failure", None)
|
||||
if on_detached_failure is None:
|
||||
return
|
||||
try:
|
||||
await on_detached_failure(exc)
|
||||
await self.litellm_logging_obj.report_detached_stream_failure(exc)
|
||||
except Exception as hook_failure: # noqa: BLE001 # a failing proxy hook must not crash the detached pump
|
||||
verbose_proxy_logger.warning(
|
||||
"async_sse_wrapper detached failure hook raised: %s(%s)", type(hook_failure).__name__, hook_failure
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
## Translates OpenAI call to Anthropic `/v1/messages` format
|
||||
import asyncio
|
||||
from collections import deque
|
||||
from collections.abc import AsyncGenerator, AsyncIterable, Awaitable, Callable, Iterable, Iterator, Mapping, Sequence
|
||||
from collections.abc import AsyncGenerator, AsyncIterable, Iterable, Iterator, Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Final, Protocol, runtime_checkable
|
||||
|
||||
import anyio
|
||||
|
|
@ -486,14 +486,12 @@ class AnthropicResponsesStreamWrapper:
|
|||
if failure is None:
|
||||
return
|
||||
self._unreported_failure = None
|
||||
on_failure: Final[Callable[[Exception], Awaitable[None]] | None] = getattr(
|
||||
self.litellm_logging_obj, "_on_detached_stream_failure", None
|
||||
)
|
||||
if on_failure is None:
|
||||
logging_obj: Final = self.litellm_logging_obj
|
||||
if logging_obj is None:
|
||||
return
|
||||
try:
|
||||
with anyio.CancelScope(shield=True):
|
||||
await on_failure(failure)
|
||||
await logging_obj.report_detached_stream_failure(failure)
|
||||
except Exception as hook_failure: # noqa: BLE001 # a failing proxy hook must not hide the error event from the client
|
||||
verbose_logger.warning(
|
||||
"AnthropicResponsesStreamWrapper: stream failure hook raised: %s(%s)",
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ import json
|
|||
import logging
|
||||
import math
|
||||
import time
|
||||
import weakref
|
||||
from collections.abc import (
|
||||
AsyncGenerator,
|
||||
AsyncIterator,
|
||||
|
|
@ -97,6 +96,7 @@ from litellm.litellm_core_utils.core_helpers import (
|
|||
redact_nested_match_and_regex_keys,
|
||||
require_attributes,
|
||||
stream_cost_injection_requested,
|
||||
without,
|
||||
)
|
||||
from litellm.litellm_core_utils.dd_tracing import NullTracer, tracer
|
||||
from litellm.litellm_core_utils.get_supported_openai_params import (
|
||||
|
|
@ -2728,47 +2728,35 @@ def _content_text(fields: Mapping[str, object]) -> str:
|
|||
return str(fields.get("content", ""))
|
||||
|
||||
|
||||
async def _no_stream_failure_to_release() -> None:
|
||||
return None
|
||||
def _hold_stream_failures(logging_obj: LiteLLMLoggingObj | None) -> None:
|
||||
if logging_obj is not None:
|
||||
logging_obj.hold_stream_failures()
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _ClientStreamFailureReport:
|
||||
logging_obj: LiteLLMLoggingObj
|
||||
report: Callable[[Exception], Awaitable[None]]
|
||||
held: Exception | None = None
|
||||
|
||||
@classmethod
|
||||
def hold_while_streaming(cls, logging_obj: LiteLLMLoggingObj | None) -> Callable[[], Awaitable[None]]:
|
||||
if logging_obj is None or logging_obj.on_detached_stream_failure is None:
|
||||
return _no_stream_failure_to_release
|
||||
return cls(logging_obj, logging_obj.on_detached_stream_failure)._start_holding()
|
||||
|
||||
def _start_holding(self) -> Callable[[], Awaitable[None]]:
|
||||
self.logging_obj.on_detached_stream_failure = self.hold
|
||||
return self.release
|
||||
|
||||
async def hold(self, failure: Exception) -> None:
|
||||
if self.held is None:
|
||||
self.held = failure
|
||||
|
||||
async def release(self) -> None:
|
||||
self.logging_obj.on_detached_stream_failure = self.report
|
||||
if self.held is None:
|
||||
return
|
||||
await asyncio.shield(
|
||||
spawn_logging_task(
|
||||
asyncio.get_running_loop(), self._report_held(self.held), task_name="stream failure report"
|
||||
)
|
||||
async def _release_held_stream_failure(logging_obj: LiteLLMLoggingObj | None) -> None:
|
||||
"""Report the detached-stream failure held while the client stream ran, if the stream did not
|
||||
report one itself."""
|
||||
if logging_obj is None:
|
||||
return
|
||||
held: Final = logging_obj.release_held_stream_failure()
|
||||
if held is None:
|
||||
return
|
||||
await asyncio.shield(
|
||||
spawn_logging_task(
|
||||
asyncio.get_running_loop(),
|
||||
_report_held_stream_failure(logging_obj, held),
|
||||
task_name="stream failure report",
|
||||
)
|
||||
)
|
||||
|
||||
async def _report_held(self, failure: Exception) -> None:
|
||||
try:
|
||||
await self.report(failure)
|
||||
except Exception as hook_failure: # noqa: BLE001 # a failing failure hook must not skip the stream cleanup
|
||||
verbose_proxy_logger.warning(
|
||||
"Stream failure hook raised: %s(%s)", type(hook_failure).__name__, sanitize_for_log(hook_failure)
|
||||
)
|
||||
|
||||
async def _report_held_stream_failure(logging_obj: LiteLLMLoggingObj, failure: Exception) -> None:
|
||||
try:
|
||||
await logging_obj.report_detached_stream_failure(failure)
|
||||
except Exception as hook_failure: # noqa: BLE001 # a failing failure hook must not skip the stream cleanup
|
||||
verbose_proxy_logger.warning(
|
||||
"Stream failure hook raised: %s(%s)", type(hook_failure).__name__, sanitize_for_log(hook_failure)
|
||||
)
|
||||
|
||||
|
||||
def _metadata_logged_on(logged_on: LiteLLMLoggingObj) -> Mapping[str, object]:
|
||||
|
|
@ -4480,22 +4468,19 @@ class ProxyBaseLLMRequestProcessing:
|
|||
a replacement error the hook raises has no client left to reach.
|
||||
"""
|
||||
# The closure lives on the logging object until the logging object dies, so it must not
|
||||
# hold the logging object, or a request stays alive until a full GC pass (#51277)
|
||||
logging_obj_ref: Final = weakref.ref(logging_obj)
|
||||
request_data: Final = {
|
||||
key: value for key, value in self._request_fields().items() if key != "litellm_logging_obj"
|
||||
}
|
||||
# hold the logging object, or a request stays alive until a full GC pass (#51277). The
|
||||
# caller passes the logging object in when it fires.
|
||||
request_data: Final = without(self._request_fields(), "litellm_logging_obj")
|
||||
route_type: Final = self.route_type
|
||||
|
||||
async def _on_detached_stream_failure(exc: Exception) -> None:
|
||||
live_logging_obj: Final = logging_obj_ref()
|
||||
if live_logging_obj is None or not live_logging_obj.claim_stream_failure_report():
|
||||
async def _on_detached_stream_failure(failed_logging_obj: LiteLLMLoggingObj, exc: Exception) -> None:
|
||||
if not failed_logging_obj.claim_stream_failure_report():
|
||||
return
|
||||
try:
|
||||
await proxy_logging_obj.post_call_failure_hook(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
original_exception=exc,
|
||||
request_data={**request_data, "litellm_logging_obj": live_logging_obj},
|
||||
request_data={**request_data, "litellm_logging_obj": failed_logging_obj},
|
||||
dispatched_call_type=route_type,
|
||||
)
|
||||
except HTTPException:
|
||||
|
|
@ -5536,7 +5521,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
request_data=request_data,
|
||||
)
|
||||
timing_sink: Final = stream_logging_obj if isinstance(stream_logging_obj, LiteLLMLoggingObj) else None
|
||||
release_stream_failure: Final = _ClientStreamFailureReport.hold_while_streaming(timing_sink)
|
||||
_hold_stream_failures(timing_sink)
|
||||
try:
|
||||
str_so_far = ""
|
||||
async for chunk in guarded_stream:
|
||||
|
|
@ -5648,7 +5633,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
yield seal + error_frame if seal else error_frame
|
||||
finally:
|
||||
try:
|
||||
await release_stream_failure()
|
||||
await _release_held_stream_failure(timing_sink)
|
||||
finally:
|
||||
await ProxyBaseLLMRequestProcessing.finalize_streaming_generator_cleanup(
|
||||
request=request,
|
||||
|
|
|
|||
|
|
@ -282,7 +282,7 @@ def _make_logging_obj(
|
|||
)
|
||||
|
||||
|
||||
async def _proxy_boundary_hook(exc: Exception) -> None:
|
||||
async def _proxy_boundary_hook(_logging_obj: object, exc: Exception) -> None:
|
||||
return None
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -581,7 +581,7 @@ class _DetachedFailureRecorder:
|
|||
def __init__(self):
|
||||
self.exceptions = []
|
||||
|
||||
async def __call__(self, exc: Exception) -> None:
|
||||
async def __call__(self, _logging_obj: object, exc: Exception) -> None:
|
||||
self.exceptions.append(exc)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -707,11 +707,13 @@ class _FailureHookRecorder:
|
|||
def __init__(self) -> None:
|
||||
self.received: tuple[Exception, ...] = ()
|
||||
|
||||
async def __call__(self, failure: Exception) -> None:
|
||||
async def __call__(self, _logging_obj: object, failure: Exception) -> None:
|
||||
self.received = (*self.received, failure)
|
||||
|
||||
|
||||
def _logging_obj_with_failure_hook(on_failure: Callable[[Exception], Awaitable[None]]) -> LitellmLogging:
|
||||
def _logging_obj_with_failure_hook(
|
||||
on_failure: Callable[[LitellmLogging, Exception], Awaitable[None]],
|
||||
) -> LitellmLogging:
|
||||
logging_obj = LitellmLogging(
|
||||
model="m",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
|
|
@ -740,7 +742,7 @@ class _DeliveryCountingConsumer:
|
|||
self.delivered: tuple[dict[str, object], ...] = ()
|
||||
self.delivered_when_hook_fired: tuple[int, ...] = ()
|
||||
|
||||
async def hook(self, failure: Exception) -> None:
|
||||
async def hook(self, _logging_obj: object, failure: Exception) -> None:
|
||||
self.delivered_when_hook_fired = (*self.delivered_when_hook_fired, len(self.delivered))
|
||||
|
||||
async def drain(self, wrapper: AnthropicResponsesStreamWrapper) -> None:
|
||||
|
|
@ -806,7 +808,7 @@ class TestUpstreamFailureReachesTheProxyFailureHook:
|
|||
assert "Rate limit reached for gpt-5.5, try again in 20s." in failure.message
|
||||
|
||||
def test_a_raising_hook_never_hides_the_error_event_from_the_client(self):
|
||||
async def _exploding_hook(failure: Exception) -> None:
|
||||
async def _exploding_hook(_logging_obj: object, failure: Exception) -> None:
|
||||
raise RuntimeError("spend writer is down")
|
||||
|
||||
logging_obj = _logging_obj_with_failure_hook(_exploding_hook)
|
||||
|
|
@ -872,7 +874,7 @@ class TestUpstreamFailureReachesTheProxyFailureHook:
|
|||
hook_released = anyio.Event()
|
||||
finished: list[Exception] = []
|
||||
|
||||
async def _held_hook(failure: Exception) -> None:
|
||||
async def _held_hook(_logging_obj: object, failure: Exception) -> None:
|
||||
hook_started.set()
|
||||
await hook_released.wait()
|
||||
finished.append(failure)
|
||||
|
|
|
|||
|
|
@ -13525,7 +13525,7 @@ class TestDetachedStreamFailureHook:
|
|||
)
|
||||
|
||||
failure = RuntimeError("upstream died after the client left")
|
||||
await logging_obj.on_detached_stream_failure(failure)
|
||||
await logging_obj.on_detached_stream_failure(logging_obj, failure)
|
||||
|
||||
assert recorder.calls == [
|
||||
{
|
||||
|
|
@ -13550,7 +13550,7 @@ class TestDetachedStreamFailureHook:
|
|||
)
|
||||
failure = RuntimeError("upstream died after the client left")
|
||||
|
||||
await logging_obj.on_detached_stream_failure(failure)
|
||||
await logging_obj.on_detached_stream_failure(logging_obj, failure)
|
||||
|
||||
assert [call["original_exception"] for call in recorder.calls] == [failure]
|
||||
|
||||
|
|
@ -13593,7 +13593,7 @@ class TestDetachedStreamFailureHook:
|
|||
def iterator_hook_for(logging_obj):
|
||||
async def iterator_hook(**_kwargs):
|
||||
yield b"event: message_start\n\n"
|
||||
await logging_obj.on_detached_stream_failure(upstream_failure)
|
||||
await logging_obj.report_detached_stream_failure(upstream_failure)
|
||||
yield b"event: error\n\n"
|
||||
yield b"event: message_stop\n\n"
|
||||
if after_report is not None:
|
||||
|
|
@ -13696,7 +13696,7 @@ class TestDetachedStreamFailureHook:
|
|||
logging_obj, stream = self._armed_stream(recorder, lambda _logging_obj: iterator_hook)
|
||||
|
||||
frames = [frame async for frame in stream]
|
||||
await logging_obj.on_detached_stream_failure(upstream_failure)
|
||||
await logging_obj.report_detached_stream_failure(upstream_failure)
|
||||
|
||||
assert frames == ["data: chunk\n\n"]
|
||||
assert [call["original_exception"] for call in recorder.calls] == [upstream_failure]
|
||||
|
|
@ -13733,7 +13733,7 @@ class TestDetachedStreamFailureHook:
|
|||
logging_obj, stream = self._armed_stream(recorder, lambda _logging_obj: iterator_hook)
|
||||
|
||||
frames = [frame async for frame in stream]
|
||||
await logging_obj.on_detached_stream_failure(upstream_failure)
|
||||
await logging_obj.report_detached_stream_failure(upstream_failure)
|
||||
|
||||
assert frames == ["data: chunk\n\n", "data: error\n\n"]
|
||||
assert [call["original_exception"] for call in recorder.calls] == [hook_failure]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue