diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 7aa93c253d4..1fdb51deb2c 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -5,6 +5,7 @@ import json import logging import math import time +import weakref from collections.abc import ( AsyncGenerator, AsyncIterator, @@ -4461,6 +4462,9 @@ class ProxyBaseLLMRequestProcessing: ), ) + def _request_fields(self) -> Mapping[str, object]: + return self.data + def _arm_detached_stream_failure_hook( self, logging_obj: LiteLLMLoggingObj, @@ -4475,17 +4479,24 @@ class ProxyBaseLLMRequestProcessing: so the failed spend is still written and the budget reservation released; a replacement error the hook raises has no client left to reach. """ - request_data: Final = self.data + # The closure lives on the logging object until the logging object dies, so it must not + # hold the logging object, or a request stays alive until a full GC pass (#51277) + logging_obj_ref: Final = weakref.ref(logging_obj) + request_data: Final = { + key: value for key, value in self._request_fields().items() if key != "litellm_logging_obj" + } + route_type: Final = self.route_type async def _on_detached_stream_failure(exc: Exception) -> None: - if not logging_obj.claim_stream_failure_report(): + live_logging_obj: Final = logging_obj_ref() + if live_logging_obj is None or not live_logging_obj.claim_stream_failure_report(): return try: await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=exc, - request_data=request_data, - dispatched_call_type=self.route_type, + request_data={**request_data, "litellm_logging_obj": live_logging_obj}, + dispatched_call_type=route_type, ) except HTTPException: return diff --git a/tests/integration/messages_endpoint/test_finished_requests_freed_wire.py b/tests/integration/messages_endpoint/test_finished_requests_freed_wire.py index 12863d5c0f8..7d6a3d546a9 100644 --- a/tests/integration/messages_endpoint/test_finished_requests_freed_wire.py +++ b/tests/integration/messages_endpoint/test_finished_requests_freed_wire.py @@ -1,3 +1,4 @@ +import json import uuid from collections.abc import Iterator from pathlib import Path @@ -5,7 +6,7 @@ from typing import Final import pytest import yaml -from integration._support.anthropic_sse import message_json +from integration._support.anthropic_sse import message_json, message_stream, stream_reply from integration._support.client import ( JSON_OBJECT, Gateway, @@ -90,33 +91,44 @@ def no_cleanup_pass(uncached_gateway: Gateway) -> Iterator[None]: def _answer(request: Request) -> Reply: assert request.method == "POST" and request.target == "/v1/messages" - return Reply(body=message_json("msg_freed", _MODEL, "Hi!")) + streamed: Final = JSON_OBJECT.validate_python(json.loads(request.body)).get("stream") is True + return ( + stream_reply(message_stream("msg_freed", _MODEL, "Hi!")) + if streamed + else Reply(body=message_json("msg_freed", _MODEL, "Hi!")) + ) -def _call(gateway: Gateway, path: str, model: str) -> None: +def _call(gateway: Gateway, path: str, stream: bool) -> None: messages: Final[list[JsonValue]] = [{"role": "user", "content": f"Say hi {uuid.uuid4().hex}"}] body: Final[dict[str, JsonValue]] = ( - {"model": model, "max_tokens": 16, "messages": messages} + {"model": _CONFIG_MODEL, "max_tokens": 16, "messages": messages, "stream": stream} if path == "/v1/messages" - else {"model": model, "messages": messages} + else {"model": _CONFIG_MODEL, "messages": messages, "stream": stream} ) response: Final = gateway.request("POST", path, body) assert response.status_code == 200, response.text + assert "Hi!" in response.text, response.text -@pytest.mark.parametrize("path", ("/v1/messages", "/v1/chat/completions")) +@pytest.mark.parametrize( + ("path", "stream"), + (("/v1/messages", False), ("/v1/messages", True), ("/v1/chat/completions", False)), + ids=("messages", "streamed-messages", "chat-completions"), +) def test_finished_calls_are_freed_without_a_cleanup_pass( - uncached_gateway: Gateway, no_cleanup_pass: None, path: str + uncached_gateway: Gateway, no_cleanup_pass: None, path: str, stream: bool ) -> None: gateway: Final = uncached_gateway - _call(gateway, path, _CONFIG_MODEL) + _call(gateway, path, stream) before: Final = _log_records(gateway) for _ in range(_CALLS): - _call(gateway, path, _CONFIG_MODEL) + _call(gateway, path, stream) after: Final = eventually( lambda: _log_records(gateway), lambda count: count <= before + _SLACK, seconds=15, return_last_on_timeout=True ) assert after <= before + _SLACK, ( - f"{after - before} of {_CALLS} finished {path} calls still hold their log record with Python's cycle " - "collector held off, so each one waits for a full cleanup pass before its memory comes back" + f"{after - before} of {_CALLS} finished {'streamed ' if stream else ''}{path} calls still hold their log " + "record with Python's cycle collector held off, so each one waits for a full cleanup pass " + "before its memory comes back" ) diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 08acef0e8f8..fcf5f276516 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -2,9 +2,11 @@ import asyncio import contextvars import copy import datetime +import gc import gzip import json import time +import weakref from base64 import urlsafe_b64encode from collections.abc import AsyncIterable, Awaitable, Coroutine, Mapping from types import MappingProxyType, SimpleNamespace @@ -13699,6 +13701,25 @@ class TestDetachedStreamFailureHook: assert frames == ["data: chunk\n\n"] assert [call["original_exception"] for call in recorder.calls] == [upstream_failure] + @pytest.mark.asyncio + async def test_finished_stream_frees_the_logging_object_without_a_gc_pass(self) -> None: + recorder: Final = _FailureHookRecorder() + + async def iterator_hook(**_kwargs: object) -> AsyncGenerator[bytes, None]: + yield b"event: message_start\n\n" + + logging_obj, stream = self._armed_stream(recorder, lambda _logging_obj: iterator_hook) + logging_ref: Final = weakref.ref(logging_obj) + frames: Final = [frame async for frame in stream] + + gc.disable() + try: + del logging_obj, stream + assert frames == ["data: chunk\n\n"] + assert logging_ref() is None + finally: + gc.enable() + @pytest.mark.asyncio async def test_failure_reported_by_an_iterator_hook_is_not_reported_again_when_the_stream_closes(self): recorder = _FailureHookRecorder()