mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(interactions): retain async stream success logging task so it survives GC and shutdown drain (#52300)
Route the async Interactions API stream success logging through spawn_logging_task instead of a bare asyncio.create_task, so the pending task is strongly referenced, failures are logged, and proxy shutdown drains it. Fixes #50894 Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com>
This commit is contained in:
parent
bf7bcfca78
commit
bcb1f486a6
2 changed files with 66 additions and 2 deletions
|
|
@ -23,6 +23,7 @@ from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_b
|
|||
from litellm.litellm_core_utils.thread_pool_executor import executor
|
||||
from litellm.llms.base_llm.interactions.transformation import BaseInteractionsAPIConfig
|
||||
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
|
||||
from litellm.responses.streaming_iterator import spawn_logging_task
|
||||
from litellm.types.interactions import (
|
||||
InteractionsAPIStreamingResponse,
|
||||
)
|
||||
|
|
@ -185,14 +186,16 @@ class InteractionsAPIStreamingIterator(BaseInteractionsAPIStreamingIterator):
|
|||
|
||||
logging_response: Final = copy.deepcopy(self.completed_response)
|
||||
|
||||
asyncio.create_task(
|
||||
spawn_logging_task(
|
||||
asyncio.get_running_loop(),
|
||||
self.logging_obj.dispatch_success_handlers(
|
||||
logging_response,
|
||||
start_time=self.start_time,
|
||||
end_time=datetime.now(),
|
||||
cache_hit=None,
|
||||
prefer_async_handlers=True,
|
||||
)
|
||||
),
|
||||
task_name="Interactions API stream success logging",
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -5,8 +5,10 @@ async_success_handler (cross-thread pydantic mutation segfaults pydantic-core).
|
|||
"""
|
||||
|
||||
import asyncio
|
||||
import gc
|
||||
import json
|
||||
import time
|
||||
import weakref
|
||||
from datetime import datetime
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -23,6 +25,7 @@ from litellm.interactions.streaming_iterator import (
|
|||
from litellm.litellm_core_utils import thread_pool_executor as thread_pool_executor_module
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
|
||||
from litellm.llms.gemini.interactions.transformation import GoogleAIStudioInteractionsConfig
|
||||
from litellm.responses.streaming_iterator import drain_logging_tasks
|
||||
from litellm.types.interactions import InteractionsAPIStreamingResponse
|
||||
|
||||
|
||||
|
|
@ -191,3 +194,61 @@ async def test_stream_keeps_unicode_line_separators_inside_the_delta_text(
|
|||
first: Final = await _first_event(iterator)
|
||||
|
||||
assert first.delta == delta
|
||||
|
||||
|
||||
class _ParkedSuccessLogger(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.parked: asyncio.Event = asyncio.Event()
|
||||
self.gate: weakref.ref[asyncio.Future[None]] | None = None
|
||||
self.logged: tuple[object, ...] = ()
|
||||
|
||||
async def async_log_success_event(
|
||||
self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime
|
||||
) -> None:
|
||||
gate: Final[asyncio.Future[None]] = asyncio.get_running_loop().create_future()
|
||||
self.gate = weakref.ref(gate)
|
||||
self.parked.set()
|
||||
await gate
|
||||
self.logged = (*self.logged, response_obj)
|
||||
|
||||
def open_gate(self) -> asyncio.Future[None] | None:
|
||||
gate: Final = self.gate() if self.gate is not None else None
|
||||
if gate is not None and not gate.done():
|
||||
gate.set_result(None)
|
||||
return gate
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pending_stream_success_log_survives_gc_and_is_awaited_by_shutdown_drain() -> None:
|
||||
recorder: Final = _ParkedSuccessLogger()
|
||||
logging_obj: Final = LitellmLogging(
|
||||
model="gemini/gemini-3.1-pro-preview",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
call_type="ainteraction",
|
||||
start_time=datetime(2026, 10, 9, 12, 0, 0),
|
||||
litellm_call_id="interactions-retained-log",
|
||||
function_id="interactions-retained-log",
|
||||
dynamic_async_success_callbacks=[recorder],
|
||||
)
|
||||
completed: Final = {"event_type": "interaction.completed", "id": "interaction-1", "status": "completed"}
|
||||
iterator: Final = InteractionsAPIStreamingIterator(
|
||||
response=httpx.Response(200, content=f"data: {json.dumps(completed)}\n\n".encode()),
|
||||
model="gemini/gemini-3.1-pro-preview",
|
||||
interactions_api_config=GoogleAIStudioInteractionsConfig(),
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
try:
|
||||
streamed: Final = [event async for event in iterator]
|
||||
await asyncio.wait_for(recorder.parked.wait(), timeout=5)
|
||||
_ = gc.collect()
|
||||
|
||||
assert recorder.open_gate() is not None
|
||||
await drain_logging_tasks()
|
||||
assert len(recorder.logged) == 1
|
||||
assert [event.event_type for event in streamed] == ["interaction.completed"]
|
||||
finally:
|
||||
_ = recorder.open_gate()
|
||||
await drain_logging_tasks()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue