fix(otel): keep only the request text per span so a failed request's span can be freed

litellm/integrations/otel/logger.py

@@ -3,7 +3,7 @@
 import sys
 import weakref
 from collections import OrderedDict
-from collections.abc import Callable, Iterator, Mapping, Sequence
+from collections.abc import Callable, Collection, Iterator, Mapping, Sequence
 from contextlib import contextmanager, nullcontext
 from dataclasses import replace
 from datetime import datetime
@@ -107,22 +107,22 @@ def _span_error_from_exception(
     *,
     status_code: int | None = None,
     traceback_str: str | None = None,
-    request_data: Mapping[str, object] | None = None,
+    scrub_texts: Collection[str] | None = None,
 ) -> SpanError:
     """A ``SpanError`` for a proxy-level failure that never produced a
     ``StandardLoggingPayload`` (auth / validation / malformed-body rejections),
     mirroring ``_parse_error``'s field mapping so it stamps the same v2 keys a
     failed LLM call does. ``status_code`` pins ``error.code`` to the real response
-    status, matching v1's SERVER-span behavior. ``request_data`` scrubs the
-    request's own text out of the error when its message logging is redacted."""
+    status, matching v1's SERVER-span behavior. ``scrub_texts`` is the request's
+    own text to scrub out of the error, set when its message logging is redacted."""
     from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
-    from litellm.litellm_core_utils.redact_messages import scrubbed_error_information_for_request
+    from litellm.litellm_core_utils.redact_messages import scrubbed_error_information
 
     raw_info: Final = StandardLoggingPayloadSetup.get_error_information(
         original_exception=exception,
         traceback_str=traceback_str,
     )
-    info: Final = raw_info if request_data is None else scrubbed_error_information_for_request(raw_info, request_data)
+    info: Final = raw_info if scrub_texts is None else scrubbed_error_information(raw_info, scrub_texts)
     return SpanError(
         error_type=info.get("error_class") or info.get("error_code") or None,
         message=info.get("error_message") or None,
@@ -132,7 +132,8 @@ def _span_error_from_exception(
     )
 
 
-_failed_request_by_span: Final[weakref.WeakKeyDictionary[Span, Mapping[str, object]]] = weakref.WeakKeyDictionary()
+# The request text each failure hook scrubbed out of its span's error, or None when the request isn't redacted.
+_scrub_texts_by_span: Final[weakref.WeakKeyDictionary[Span, frozenset[str] | None]] = weakref.WeakKeyDictionary()
 
 
 # Any callback whose class belongs to one of these modules is "the OTel
@@ -858,9 +859,7 @@ class OpenTelemetryV2(CustomLogger):
         already_stamped: Final = Error.TYPE in (getattr(span, "attributes", None) or ())
         stamp_error(
             span,
-            _span_error_from_exception(
-                exception, status_code=status_code, request_data=_failed_request_by_span.get(span)
-            ),
+            _span_error_from_exception(exception, status_code=status_code, scrub_texts=_scrub_texts_by_span.get(span)),
             record_event=not already_stamped,
         )
 
@@ -886,10 +885,13 @@ class OpenTelemetryV2(CustomLogger):
         span: Final = mcp_message_transport_span() or request_root_span() or user_api_key_dict.parent_otel_span
         if span is None or not is_recordable_span(span):
             return
-        _failed_request_by_span[span] = request_data
+        from litellm.litellm_core_utils.redact_messages import failure_request_texts, should_redact_failed_request
+
+        scrub_texts: Final = failure_request_texts(request_data) if should_redact_failed_request(request_data) else None
+        _scrub_texts_by_span[span] = scrub_texts
         stamp_error(
             span,
-            _span_error_from_exception(original_exception, traceback_str=traceback_str, request_data=request_data),
+            _span_error_from_exception(original_exception, traceback_str=traceback_str, scrub_texts=scrub_texts),
         )
         return
 

tests/unit/integrations/otel/test_otel_v2_logger.py

@@ -10,8 +10,10 @@ import ast
 import asyncio
 import contextlib
 import contextvars
+import gc
 import os
 import re
+import weakref
 from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
@@ -1617,6 +1619,32 @@ def test_restamp_shows_the_handlers_own_error_with_the_prompt_scrubbed(monkeypat
     assert "router.py" not in span.attributes["litellm.provider.error.stack_trace"]
 
 
+def test_failure_hook_lets_the_failed_requests_span_be_freed(monkeypatch):
+    """The proxy keeps the request's root span in the request's own metadata. What the failure hook
+    keeps for the handler's restamp must not hold that span alive once the request is done."""
+    import litellm
+    from litellm.proxy._types import UserAPIKeyAuth
+
+    monkeypatch.setattr(litellm, "turn_off_message_logging", True)
+    logger, _ = _logger()
+    server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME)
+    server_ref: Final = weakref.ref(server)
+    asyncio.run(
+        logger.async_post_call_failure_hook(
+            request_data={
+                "messages": [{"role": "user", "content": "my ssn is 123-45-6789"}],
+                "metadata": {"litellm_parent_otel_span": server},
+            },
+            original_exception=_proxy_exc("blocked", 400),
+            user_api_key_dict=UserAPIKeyAuth(parent_otel_span=server),
+        )
+    )
+    del server
+    gc.collect()
+
+    assert server_ref() is None
+
+
 def test_record_error_attributes_on_span_does_not_duplicate_an_already_stamped_error():
     """A failure that already went through ``async_post_call_failure_hook`` reaches
     the exception handler too; the second stamp must keep one exception event while