From ed795b10da602ac96c93c60116165ca5fa53fc4c Mon Sep 17 00:00:00 2001 From: 1divyanshdev Date: Wed, 28 Oct 2026 14:33:54 +0530 Subject: [PATCH] fix(wrapper): release failed-call exception references --- .github/workflows/test-merge-smoke.yml | 6 +- litellm/utils.py | 21 +++ tests/unit/test_utils.py | 178 +++++++++++++++++++++++++ 3 files changed, 203 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-merge-smoke.yml b/.github/workflows/test-merge-smoke.yml index f3f09523491..173de86d344 100644 --- a/.github/workflows/test-merge-smoke.yml +++ b/.github/workflows/test-merge-smoke.yml @@ -76,11 +76,13 @@ jobs: - name: Import and CLI checks run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py cli - - name: Routing and UUID compatibility + - name: Routing, UUID and wrapper failure compatibility run: >- $IN_NETNS .venv/bin/python -m pytest tests/unit/router_strategy/test_complexity_router.py::TestCustomDimensions - tests/unit/test_uuid_helper.py -q + tests/unit/test_uuid_helper.py + tests/unit/test_utils.py::test_failed_sync_completion_releases_prompt_without_cyclic_gc + tests/unit/test_utils.py::test_failed_acompletion_releases_prompt_without_cyclic_gc -q - name: Proxy startup check run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py proxy-startup --diagnostics-dir "${RUNNER_TEMP}/smoke-diagnostics" diff --git a/litellm/utils.py b/litellm/utils.py index 6f0b1ca2e3e..82bc1bbf6c1 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2165,6 +2165,14 @@ def client(original_function): logging_obj.failure_handler( e, traceback_exception, start_time, end_time ) # DO NOT MAKE THREADED - router retry fallback relies on this! + # All failure callbacks have now been dispatched. Release the stored + # exception and the logging object reference so the re-raised exception's + # traceback frames cannot pin this request's Logging object (and through + # it the request payload) in a reference cycle only a GC pass can help. + if logging_obj is not None: + logging_obj.model_call_details.pop("exception", None) + kwargs.pop("litellm_logging_obj", None) + del logging_obj raise e @wraps(original_function) @@ -2190,6 +2198,7 @@ def client(original_function): kwargs.pop("_is_litellm_internal_call", None) # discard if injected _is_litellm_internal_call: Final = is_internal_call.get() _deployment_call_end_time: datetime.datetime | None = None + uncached_call_kwargs: dict | None = None try: if logging_obj is None: @@ -2300,6 +2309,8 @@ def client(original_function): and _caching_handler_response.embedding_uncached_input is not None else kwargs ) + if call_kwargs is not kwargs: + uncached_call_kwargs = call_kwargs try: result = await original_function(*args, **call_kwargs) except Exception as deployment_error: @@ -2486,6 +2497,16 @@ def client(original_function): timeout: Final = _get_wrapper_timeout(kwargs=kwargs) setattr(e, "timeout", timeout) + # Same cleanup as the sync wrapper: every failure callback has been + # dispatched above, so release the stored exception and our kwargs + # references before re-raising. `logging_obj` itself cannot be deleted + # here because the finally block below still needs it to restore + # correlation contextvars. + if logging_obj is not None and not _is_litellm_internal_call: + logging_obj.model_call_details.pop("exception", None) + kwargs.pop("litellm_logging_obj", None) + if uncached_call_kwargs is not None: + uncached_call_kwargs.pop("litellm_logging_obj", None) raise e finally: diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 38287b25169..601756df941 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -4,15 +4,19 @@ import base64 import copy import contextlib import contextvars +import gc import io import json import logging import os import queue import threading +import traceback +import weakref from collections.abc import Callable, Iterator, Mapping from concurrent.futures import Future, ThreadPoolExecutor from datetime import datetime, timedelta, timezone +from http.server import BaseHTTPRequestHandler, HTTPServer from pathlib import Path, PurePath from typing import Final, SupportsIndex, cast from unittest import mock @@ -10880,3 +10884,177 @@ def test_get_optional_params_forwards_a_tool_config_cache_point_to_a_downstream_ ) assert optional_params["extra_body"] == {"kept": True, "cache_control_injection_points": [point]} + + +class _Always400OpenAIChatHandler(BaseHTTPRequestHandler): + """Local OpenAI-compatible deployment that rejects every chat completion with a 400.""" + + def do_POST(self) -> None: + self.rfile.read(int(self.headers.get("content-length", 0))) + error_body = json.dumps( + { + "error": { + "message": "This model's maximum context length is 32768 tokens.", + "type": "invalid_request_error", + "code": "context_length_exceeded", + } + } + ).encode() + self.send_response(400) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(error_body))) + self.end_headers() + self.wfile.write(error_body) + + def log_message(self, format, *args): + pass + + +@contextlib.contextmanager +def _local_always_400_chat_completions_server() -> Iterator[str]: + server = HTTPServer(("127.0.0.1", 0), _Always400OpenAIChatHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_address[1]}/v1" + finally: + server.shutdown() + server.server_close() + + +class _WeakRefPrompt(str): + """str subclass, so a weakref can observe when litellm releases the request prompt.""" + + +_PROMPT_SIZE = 400_000 + + +class _CaptureFailureEventLogger(CustomLogger): + """Failure callback observer. Records only derived payload facts, never the + kwargs dict itself, so it cannot keep the logging cycle alive.""" + + def __init__(self) -> None: + super().__init__() + self.failure_event_facts = [] + + def _record_failure_event(self, kwargs: dict) -> None: + standard_payload = kwargs.get("standard_logging_object") or {} + error_information = standard_payload.get("error_information") or {} + self.failure_event_facts.append( + { + "exception_present": "exception" in kwargs, + "error_code": error_information.get("error_code"), + } + ) + + def log_failure_event(self, kwargs, response_obj, start_time, end_time) -> None: + self._record_failure_event(kwargs) + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time) -> None: + self._record_failure_event(kwargs) + + +def test_failed_sync_completion_releases_prompt_without_cyclic_gc(monkeypatch: pytest.MonkeyPatch): + """Regression for issue #48649: after a failed sync completion() the cycle + exception -> traceback -> wrapper/impl frames -> Logging -> + model_call_details["exception"] used to pin the request messages until a + full generation-2 collection. With the cyclic GC disabled, the prompt must + still be freed by plain refcounting once the failed call returns, the + failure callback must see the complete payload before the cleanup, and the + exception must reach the caller with its traceback intact.""" + observer = _CaptureFailureEventLogger() + monkeypatch.setattr(litellm, "failure_callback", [observer]) + gc_was_enabled = gc.isenabled() + gc.disable() + try: + with _local_always_400_chat_completions_server() as api_base: + prompt = _WeakRefPrompt("x" * _PROMPT_SIZE) + messages = [{"role": "user", "content": prompt}] + facts = {} + try: + litellm.completion( + model="openai/default", + api_base=api_base, + api_key="not-used", + messages=messages, + num_retries=0, + max_retries=0, + ) + except litellm.exceptions.ContextWindowExceededError as exc: + facts["error_message"] = str(exc) + facts["error_type"] = type(exc).__name__ + facts["has_traceback_frames"] = bool(traceback.format_tb(exc.__traceback__)) + facts["context_chained"] = exc.__cause__ is not None or exc.__context__ is not None + prompt_ref = weakref.ref(prompt) + del prompt, messages + + assert prompt_ref() is None, "failed sync request pinned the prompt in the logging cycle (issue #48649)" + assert facts["error_type"] == "ContextWindowExceededError" + assert "maximum context length" in facts["error_message"] + assert facts["has_traceback_frames"] is True + assert facts["context_chained"] is True + assert observer.failure_event_facts, "sync failure handler never dispatched the failure callback" + assert all(event["exception_present"] for event in observer.failure_event_facts) + assert all(event["error_code"] == "400" for event in observer.failure_event_facts) + finally: + if gc_was_enabled: + gc.enable() + + +@pytest.mark.asyncio +async def test_failed_acompletion_releases_prompt_without_cyclic_gc(monkeypatch: pytest.MonkeyPatch): + """Async twin of the issue #48649 regression: a failed acompletion() must not + pin the request prompt through the exception/logging reference cycle either.""" + observer = _CaptureFailureEventLogger() + monkeypatch.setattr(litellm, "_async_failure_callback", [observer]) + gc_was_enabled = gc.isenabled() + gc.disable() + try: + with _local_always_400_chat_completions_server() as api_base: + prompt = _WeakRefPrompt("x" * _PROMPT_SIZE) + messages = [{"role": "user", "content": prompt}] + facts = {} + try: + await litellm.acompletion( + model="openai/default", + api_base=api_base, + api_key="not-used", + messages=messages, + num_retries=0, + max_retries=0, + ) + except litellm.exceptions.ContextWindowExceededError as exc: + facts["error_message"] = str(exc) + facts["error_type"] = type(exc).__name__ + facts["has_traceback_frames"] = bool(traceback.format_tb(exc.__traceback__)) + facts["context_chained"] = exc.__cause__ is not None or exc.__context__ is not None + prompt_ref = weakref.ref(prompt) + del prompt, messages + + assert prompt_ref() is None, "failed async request pinned the prompt in the logging cycle (issue #48649)" + assert facts["error_type"] == "ContextWindowExceededError" + assert "maximum context length" in facts["error_message"] + assert facts["has_traceback_frames"] is True + assert facts["context_chained"] is True + assert observer.failure_event_facts, "async failure handler never dispatched the failure callback" + assert all(event["exception_present"] for event in observer.failure_event_facts) + assert all(event["error_code"] == "400" for event in observer.failure_event_facts) + finally: + if gc_was_enabled: + gc.enable() + + +def test_successful_completion_keeps_kwargs_and_callbacks_untouched(monkeypatch: pytest.MonkeyPatch): + """The success path must keep working unchanged: a mock-based completion + returns its payload, and no failure callback is dispatched for it.""" + observer = _CaptureFailureEventLogger() + monkeypatch.setattr(litellm, "failure_callback", [observer]) + response = litellm.completion( + model="openai/default", + api_key="not-used", + messages=[{"role": "user", "content": _WeakRefPrompt("short")}], + mock_response="all fine", + ) + + assert response.choices[0].message.content == "all fine" + assert observer.failure_event_facts == []