fix(proxy): hold the logging object weakly in the detached stream failure hook (#51318)

* fix(proxy): hold the logging object weakly in the detached stream failure hook

The closure armed by _arm_detached_stream_failure_hook is stored on the
logging object and captured the logging object, the request data that
contains it, and the processing object that holds that data. Every
streamed /v1/messages request therefore ended in a reference cycle and
stayed in memory until a full cyclic GC pass instead of being freed by
reference counting when the stream finished.

Capture the logging object through a weakref, a copy of the request data
without the litellm_logging_obj key, and the route type as a value. The
hook adds the live logging object back to the request data when it fires.
All three callers reach the hook through the logging object, so it is
always alive at that point.

Add a regression test that streams a request to completion, drops every
reference, and checks with the cyclic collector disabled that the logging
object is gone.

Fixes #51277

* test(proxy): cover finished streamed /v1/messages calls being freed without a gc pass

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(proxy): annotate the finished-stream gc test

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(proxy): split the freed-calls assert message under the line limit

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: mateo <277851410+mateo-berri@users.noreply.github.com>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Rhythm Arya 2026-11-15 13:13:24 -07:00 • committed by GitHub
parent b168ca7ae4
commit e6b673e6e0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 59 additions and 15 deletions

View file

@ -5,6 +5,7 @@ import json
import logging
import math
import time
import weakref
from collections.abc import (
AsyncGenerator,
AsyncIterator,
@ -4461,6 +4462,9 @@ class ProxyBaseLLMRequestProcessing:
),
)
def _request_fields(self) -> Mapping[str, object]:
return self.data
def _arm_detached_stream_failure_hook(
self,
logging_obj: LiteLLMLoggingObj,
@ -4475,17 +4479,24 @@ class ProxyBaseLLMRequestProcessing:
so the failed spend is still written and the budget reservation released;
a replacement error the hook raises has no client left to reach.
"""
request_data: Final = self.data
# The closure lives on the logging object until the logging object dies, so it must not
# hold the logging object, or a request stays alive until a full GC pass (#51277)
logging_obj_ref: Final = weakref.ref(logging_obj)
request_data: Final = {
key: value for key, value in self._request_fields().items() if key != "litellm_logging_obj"
}
route_type: Final = self.route_type
async def _on_detached_stream_failure(exc: Exception) -> None:
if not logging_obj.claim_stream_failure_report():
live_logging_obj: Final = logging_obj_ref()
if live_logging_obj is None or not live_logging_obj.claim_stream_failure_report():
return
try:
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
original_exception=exc,
request_data=request_data,
dispatched_call_type=self.route_type,
request_data={**request_data, "litellm_logging_obj": live_logging_obj},
dispatched_call_type=route_type,
)
except HTTPException:
return

View file

@ -1,3 +1,4 @@
import json
import uuid
from collections.abc import Iterator
from pathlib import Path
@ -5,7 +6,7 @@ from typing import Final
import pytest
import yaml
from integration._support.anthropic_sse import message_json
from integration._support.anthropic_sse import message_json, message_stream, stream_reply
from integration._support.client import (
JSON_OBJECT,
Gateway,
@ -90,33 +91,44 @@ def no_cleanup_pass(uncached_gateway: Gateway) -> Iterator[None]:
def _answer(request: Request) -> Reply:
assert request.method == "POST" and request.target == "/v1/messages"
return Reply(body=message_json("msg_freed", _MODEL, "Hi!"))
streamed: Final = JSON_OBJECT.validate_python(json.loads(request.body)).get("stream") is True
return (
stream_reply(message_stream("msg_freed", _MODEL, "Hi!"))
if streamed
else Reply(body=message_json("msg_freed", _MODEL, "Hi!"))
)
def _call(gateway: Gateway, path: str, model: str) -> None:
def _call(gateway: Gateway, path: str, stream: bool) -> None:
messages: Final[list[JsonValue]] = [{"role": "user", "content": f"Say hi {uuid.uuid4().hex}"}]
body: Final[dict[str, JsonValue]] = (
{"model": model, "max_tokens": 16, "messages": messages}
{"model": _CONFIG_MODEL, "max_tokens": 16, "messages": messages, "stream": stream}
if path == "/v1/messages"
else {"model": model, "messages": messages}
else {"model": _CONFIG_MODEL, "messages": messages, "stream": stream}
)
response: Final = gateway.request("POST", path, body)
assert response.status_code == 200, response.text
assert "Hi!" in response.text, response.text
@pytest.mark.parametrize("path", ("/v1/messages", "/v1/chat/completions"))
@pytest.mark.parametrize(
("path", "stream"),
(("/v1/messages", False), ("/v1/messages", True), ("/v1/chat/completions", False)),
ids=("messages", "streamed-messages", "chat-completions"),
)
def test_finished_calls_are_freed_without_a_cleanup_pass(
uncached_gateway: Gateway, no_cleanup_pass: None, path: str
uncached_gateway: Gateway, no_cleanup_pass: None, path: str, stream: bool
) -> None:
gateway: Final = uncached_gateway
_call(gateway, path, _CONFIG_MODEL)
_call(gateway, path, stream)
before: Final = _log_records(gateway)
for _ in range(_CALLS):
_call(gateway, path, _CONFIG_MODEL)
_call(gateway, path, stream)
after: Final = eventually(
lambda: _log_records(gateway), lambda count: count <= before + _SLACK, seconds=15, return_last_on_timeout=True
)
assert after <= before + _SLACK, (
f"{after - before} of {_CALLS} finished {path} calls still hold their log record with Python's cycle "
"collector held off, so each one waits for a full cleanup pass before its memory comes back"
f"{after - before} of {_CALLS} finished {'streamed ' if stream else ''}{path} calls still hold their log "
"record with Python's cycle collector held off, so each one waits for a full cleanup pass "
"before its memory comes back"
)

View file

@ -2,9 +2,11 @@ import asyncio
import contextvars
import copy
import datetime
import gc
import gzip
import json
import time
import weakref
from base64 import urlsafe_b64encode
from collections.abc import AsyncIterable, Awaitable, Coroutine, Mapping
from types import MappingProxyType, SimpleNamespace
@ -13699,6 +13701,25 @@ class TestDetachedStreamFailureHook:
assert frames == ["data: chunk\n\n"]
assert [call["original_exception"] for call in recorder.calls] == [upstream_failure]
@pytest.mark.asyncio
async def test_finished_stream_frees_the_logging_object_without_a_gc_pass(self) -> None:
recorder: Final = _FailureHookRecorder()
async def iterator_hook(**_kwargs: object) -> AsyncGenerator[bytes, None]:
yield b"event: message_start\n\n"
logging_obj, stream = self._armed_stream(recorder, lambda _logging_obj: iterator_hook)
logging_ref: Final = weakref.ref(logging_obj)
frames: Final = [frame async for frame in stream]
gc.disable()
try:
del logging_obj, stream
assert frames == ["data: chunk\n\n"]
assert logging_ref() is None
finally:
gc.enable()
@pytest.mark.asyncio
async def test_failure_reported_by_an_iterator_hook_is_not_reported_again_when_the_stream_closes(self):
recorder = _FailureHookRecorder()