mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(proxy): hold the logging object weakly in the detached stream failure hook (#51318)
* fix(proxy): hold the logging object weakly in the detached stream failure hook The closure armed by _arm_detached_stream_failure_hook is stored on the logging object and captured the logging object, the request data that contains it, and the processing object that holds that data. Every streamed /v1/messages request therefore ended in a reference cycle and stayed in memory until a full cyclic GC pass instead of being freed by reference counting when the stream finished. Capture the logging object through a weakref, a copy of the request data without the litellm_logging_obj key, and the route type as a value. The hook adds the live logging object back to the request data when it fires. All three callers reach the hook through the logging object, so it is always alive at that point. Add a regression test that streams a request to completion, drops every reference, and checks with the cyclic collector disabled that the logging object is gone. Fixes #51277 * test(proxy): cover finished streamed /v1/messages calls being freed without a gc pass Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): annotate the finished-stream gc test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): split the freed-calls assert message under the line limit Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: mateo <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
b168ca7ae4
commit
e6b673e6e0
3 changed files with 59 additions and 15 deletions
|
|
@ -5,6 +5,7 @@ import json
|
|||
import logging
|
||||
import math
|
||||
import time
|
||||
import weakref
|
||||
from collections.abc import (
|
||||
AsyncGenerator,
|
||||
AsyncIterator,
|
||||
|
|
@ -4461,6 +4462,9 @@ class ProxyBaseLLMRequestProcessing:
|
|||
),
|
||||
)
|
||||
|
||||
def _request_fields(self) -> Mapping[str, object]:
|
||||
return self.data
|
||||
|
||||
def _arm_detached_stream_failure_hook(
|
||||
self,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
|
|
@ -4475,17 +4479,24 @@ class ProxyBaseLLMRequestProcessing:
|
|||
so the failed spend is still written and the budget reservation released;
|
||||
a replacement error the hook raises has no client left to reach.
|
||||
"""
|
||||
request_data: Final = self.data
|
||||
# The closure lives on the logging object until the logging object dies, so it must not
|
||||
# hold the logging object, or a request stays alive until a full GC pass (#51277)
|
||||
logging_obj_ref: Final = weakref.ref(logging_obj)
|
||||
request_data: Final = {
|
||||
key: value for key, value in self._request_fields().items() if key != "litellm_logging_obj"
|
||||
}
|
||||
route_type: Final = self.route_type
|
||||
|
||||
async def _on_detached_stream_failure(exc: Exception) -> None:
|
||||
if not logging_obj.claim_stream_failure_report():
|
||||
live_logging_obj: Final = logging_obj_ref()
|
||||
if live_logging_obj is None or not live_logging_obj.claim_stream_failure_report():
|
||||
return
|
||||
try:
|
||||
await proxy_logging_obj.post_call_failure_hook(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
original_exception=exc,
|
||||
request_data=request_data,
|
||||
dispatched_call_type=self.route_type,
|
||||
request_data={**request_data, "litellm_logging_obj": live_logging_obj},
|
||||
dispatched_call_type=route_type,
|
||||
)
|
||||
except HTTPException:
|
||||
return
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import json
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
|
|
@ -5,7 +6,7 @@ from typing import Final
|
|||
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.anthropic_sse import message_json
|
||||
from integration._support.anthropic_sse import message_json, message_stream, stream_reply
|
||||
from integration._support.client import (
|
||||
JSON_OBJECT,
|
||||
Gateway,
|
||||
|
|
@ -90,33 +91,44 @@ def no_cleanup_pass(uncached_gateway: Gateway) -> Iterator[None]:
|
|||
|
||||
def _answer(request: Request) -> Reply:
|
||||
assert request.method == "POST" and request.target == "/v1/messages"
|
||||
return Reply(body=message_json("msg_freed", _MODEL, "Hi!"))
|
||||
streamed: Final = JSON_OBJECT.validate_python(json.loads(request.body)).get("stream") is True
|
||||
return (
|
||||
stream_reply(message_stream("msg_freed", _MODEL, "Hi!"))
|
||||
if streamed
|
||||
else Reply(body=message_json("msg_freed", _MODEL, "Hi!"))
|
||||
)
|
||||
|
||||
|
||||
def _call(gateway: Gateway, path: str, model: str) -> None:
|
||||
def _call(gateway: Gateway, path: str, stream: bool) -> None:
|
||||
messages: Final[list[JsonValue]] = [{"role": "user", "content": f"Say hi {uuid.uuid4().hex}"}]
|
||||
body: Final[dict[str, JsonValue]] = (
|
||||
{"model": model, "max_tokens": 16, "messages": messages}
|
||||
{"model": _CONFIG_MODEL, "max_tokens": 16, "messages": messages, "stream": stream}
|
||||
if path == "/v1/messages"
|
||||
else {"model": model, "messages": messages}
|
||||
else {"model": _CONFIG_MODEL, "messages": messages, "stream": stream}
|
||||
)
|
||||
response: Final = gateway.request("POST", path, body)
|
||||
assert response.status_code == 200, response.text
|
||||
assert "Hi!" in response.text, response.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ("/v1/messages", "/v1/chat/completions"))
|
||||
@pytest.mark.parametrize(
|
||||
("path", "stream"),
|
||||
(("/v1/messages", False), ("/v1/messages", True), ("/v1/chat/completions", False)),
|
||||
ids=("messages", "streamed-messages", "chat-completions"),
|
||||
)
|
||||
def test_finished_calls_are_freed_without_a_cleanup_pass(
|
||||
uncached_gateway: Gateway, no_cleanup_pass: None, path: str
|
||||
uncached_gateway: Gateway, no_cleanup_pass: None, path: str, stream: bool
|
||||
) -> None:
|
||||
gateway: Final = uncached_gateway
|
||||
_call(gateway, path, _CONFIG_MODEL)
|
||||
_call(gateway, path, stream)
|
||||
before: Final = _log_records(gateway)
|
||||
for _ in range(_CALLS):
|
||||
_call(gateway, path, _CONFIG_MODEL)
|
||||
_call(gateway, path, stream)
|
||||
after: Final = eventually(
|
||||
lambda: _log_records(gateway), lambda count: count <= before + _SLACK, seconds=15, return_last_on_timeout=True
|
||||
)
|
||||
assert after <= before + _SLACK, (
|
||||
f"{after - before} of {_CALLS} finished {path} calls still hold their log record with Python's cycle "
|
||||
"collector held off, so each one waits for a full cleanup pass before its memory comes back"
|
||||
f"{after - before} of {_CALLS} finished {'streamed ' if stream else ''}{path} calls still hold their log "
|
||||
"record with Python's cycle collector held off, so each one waits for a full cleanup pass "
|
||||
"before its memory comes back"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2,9 +2,11 @@ import asyncio
|
|||
import contextvars
|
||||
import copy
|
||||
import datetime
|
||||
import gc
|
||||
import gzip
|
||||
import json
|
||||
import time
|
||||
import weakref
|
||||
from base64 import urlsafe_b64encode
|
||||
from collections.abc import AsyncIterable, Awaitable, Coroutine, Mapping
|
||||
from types import MappingProxyType, SimpleNamespace
|
||||
|
|
@ -13699,6 +13701,25 @@ class TestDetachedStreamFailureHook:
|
|||
assert frames == ["data: chunk\n\n"]
|
||||
assert [call["original_exception"] for call in recorder.calls] == [upstream_failure]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_finished_stream_frees_the_logging_object_without_a_gc_pass(self) -> None:
|
||||
recorder: Final = _FailureHookRecorder()
|
||||
|
||||
async def iterator_hook(**_kwargs: object) -> AsyncGenerator[bytes, None]:
|
||||
yield b"event: message_start\n\n"
|
||||
|
||||
logging_obj, stream = self._armed_stream(recorder, lambda _logging_obj: iterator_hook)
|
||||
logging_ref: Final = weakref.ref(logging_obj)
|
||||
frames: Final = [frame async for frame in stream]
|
||||
|
||||
gc.disable()
|
||||
try:
|
||||
del logging_obj, stream
|
||||
assert frames == ["data: chunk\n\n"]
|
||||
assert logging_ref() is None
|
||||
finally:
|
||||
gc.enable()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failure_reported_by_an_iterator_hook_is_not_reported_again_when_the_stream_closes(self):
|
||||
recorder = _FailureHookRecorder()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue