From aeb2aad57200707a9929ac2bdf85dbecb09211f5 Mon Sep 17 00:00:00 2001 From: yucheng Date: Fri, 30 Oct 2026 17:58:29 -0700 Subject: [PATCH] fix(otel): scrub the exception handler's own error on the SERVER span restamp The restamp kept the failure hook's message and stack trace whenever the hook had scrubbed the span, so the span showed the hook's exception (a guardrail's dict detail, the router traceback) instead of the handler's. The hook now remembers the request it stamped the span for, and the restamp scrubs that request's text out of the handler's own error. --- litellm/integrations/otel/logger.py | 28 +++++--------- .../integrations/otel/test_otel_v2_logger.py | 38 +++++++++++++++++++ 2 files changed, 48 insertions(+), 18 deletions(-) diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 6297abaf4ac..6496987c1bd 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -1,6 +1,7 @@ """``CustomLogger`` adapter on the OpenTelemetry span engine.""" import sys +import weakref from collections import OrderedDict from collections.abc import Callable, Iterator, Mapping, Sequence from contextlib import contextmanager, nullcontext @@ -27,7 +28,6 @@ from typing_extensions import TypedDict, Unpack import litellm from litellm._logging import verbose_logger -from litellm.constants import REDACTED_BY_LITELLM from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.otel.emitter import SpanEmitter, stamp_error from litellm.integrations.otel.mappers import resolve_mappers @@ -50,7 +50,7 @@ from litellm.integrations.otel.model.payloads import ( is_mcp_list_tools, is_mcp_tool_call, ) -from litellm.integrations.otel.model.semconv import Error, LiteLLMError +from litellm.integrations.otel.model.semconv import Error from litellm.integrations.otel.model.spans import SpanRole, span_role_for_service from litellm.integrations.otel.model.utils import to_ns from litellm.integrations.otel.plumbing.context import ( @@ -132,18 +132,7 @@ def _span_error_from_exception( ) -def _keeping_scrubbed_text(error: SpanError, attributes: Mapping[str, object]) -> SpanError: - """``error`` with the span's already stamped message and stack trace when the failure hook scrubbed - the request's text out of them, so a restamp from the raw exception does not put it back.""" - message: Final = attributes.get(Error.MESSAGE) - stack_trace: Final = attributes.get(LiteLLMError.STACK_TRACE) - if not any(isinstance(text, str) and REDACTED_BY_LITELLM in text for text in (message, stack_trace)): - return error - return replace( - error, - message=message if isinstance(message, str) else None, - stack_trace=stack_trace if isinstance(stack_trace, str) else None, - ) +_failed_request_by_span: Final[weakref.WeakKeyDictionary[Span, Mapping[str, object]]] = weakref.WeakKeyDictionary() # Any callback whose class belongs to one of these modules is "the OTel @@ -862,14 +851,16 @@ class OpenTelemetryV2(CustomLogger): this span already — ``async_post_call_failure_hook`` and the ``auth`` phase span record their own, and a second event would duplicate it — while the attributes are always restamped so ``error.code`` stays pinned to the real - response status.""" + response status. The restamp scrubs the text of the request the failure hook + stamped this span for, as the hook did.""" if span is None or not is_recordable_span(span): return - attributes: Final[Mapping[str, object]] = getattr(span, "attributes", None) or {} - already_stamped: Final = Error.TYPE in attributes + already_stamped: Final = Error.TYPE in (getattr(span, "attributes", None) or ()) stamp_error( span, - _keeping_scrubbed_text(_span_error_from_exception(exception, status_code=status_code), attributes), + _span_error_from_exception( + exception, status_code=status_code, request_data=_failed_request_by_span.get(span) + ), record_event=not already_stamped, ) @@ -895,6 +886,7 @@ class OpenTelemetryV2(CustomLogger): span: Final = mcp_message_transport_span() or request_root_span() or user_api_key_dict.parent_otel_span if span is None or not is_recordable_span(span): return + _failed_request_by_span[span] = request_data stamp_error( span, _span_error_from_exception(original_exception, traceback_str=traceback_str, request_data=request_data), diff --git a/tests/unit/integrations/otel/test_otel_v2_logger.py b/tests/unit/integrations/otel/test_otel_v2_logger.py index b2fd9e8638c..6817c80401a 100644 --- a/tests/unit/integrations/otel/test_otel_v2_logger.py +++ b/tests/unit/integrations/otel/test_otel_v2_logger.py @@ -1579,6 +1579,44 @@ def test_failure_hook_scrubs_the_prompt_out_of_the_server_span_and_the_restamp_k assert span.attributes["error.type"] == "ProxyException" +def _raised(exc): + try: + raise exc + except Exception as raised: + return raised + + +@pytest.mark.parametrize("redact", [True, False], ids=["redaction_on", "redaction_off"]) +def test_restamp_shows_the_handlers_own_error_with_the_prompt_scrubbed(monkeypatch, redact): + """The proxy's failure hook and its exception handler stamp the same SERVER span from different + exceptions. The handler's restamp keeps its own wording and stack trace, scrubbed like the hook's.""" + from fastapi import HTTPException + + import litellm + from litellm.proxy._types import UserAPIKeyAuth + + secret = "my ssn is 123-45-6789" + monkeypatch.setattr(litellm, "turn_off_message_logging", redact) + logger, exporter = _logger() + server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME) + set_request_root_span(server) + asyncio.run( + logger.async_post_call_failure_hook( + request_data={"messages": [{"role": "user", "content": secret}]}, + original_exception=HTTPException(status_code=400, detail={"error": f"blocked: {secret}", "mode": "pre"}), + user_api_key_dict=UserAPIKeyAuth(), + traceback_str=f' File "/srv/litellm/router.py", line 7, in route\nHTTPException: blocked: {secret}\n', + ) + ) + logger.record_error_attributes_on_span(server, _raised(_proxy_exc(f"blocked: {secret}", 500)), 500) + server.end() + (span,) = exporter.get_finished_spans() + + assert span.attributes["error.message"] == f"blocked: {'redacted-by-litellm' if redact else secret}" + assert "_raised" in span.attributes["litellm.provider.error.stack_trace"] + assert "router.py" not in span.attributes["litellm.provider.error.stack_trace"] + + def test_record_error_attributes_on_span_does_not_duplicate_an_already_stamped_error(): """A failure that already went through ``async_post_call_failure_hook`` reaches the exception handler too; the second stamp must keep one exception event while