diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 6297abaf4ac..6496987c1bd 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -1,6 +1,7 @@ """``CustomLogger`` adapter on the OpenTelemetry span engine.""" import sys +import weakref from collections import OrderedDict from collections.abc import Callable, Iterator, Mapping, Sequence from contextlib import contextmanager, nullcontext @@ -27,7 +28,6 @@ from typing_extensions import TypedDict, Unpack import litellm from litellm._logging import verbose_logger -from litellm.constants import REDACTED_BY_LITELLM from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.otel.emitter import SpanEmitter, stamp_error from litellm.integrations.otel.mappers import resolve_mappers @@ -50,7 +50,7 @@ from litellm.integrations.otel.model.payloads import ( is_mcp_list_tools, is_mcp_tool_call, ) -from litellm.integrations.otel.model.semconv import Error, LiteLLMError +from litellm.integrations.otel.model.semconv import Error from litellm.integrations.otel.model.spans import SpanRole, span_role_for_service from litellm.integrations.otel.model.utils import to_ns from litellm.integrations.otel.plumbing.context import ( @@ -132,18 +132,7 @@ def _span_error_from_exception( ) -def _keeping_scrubbed_text(error: SpanError, attributes: Mapping[str, object]) -> SpanError: - """``error`` with the span's already stamped message and stack trace when the failure hook scrubbed - the request's text out of them, so a restamp from the raw exception does not put it back.""" - message: Final = attributes.get(Error.MESSAGE) - stack_trace: Final = attributes.get(LiteLLMError.STACK_TRACE) - if not any(isinstance(text, str) and REDACTED_BY_LITELLM in text for text in (message, stack_trace)): - return error - return replace( - error, - message=message if isinstance(message, str) else None, - stack_trace=stack_trace if isinstance(stack_trace, str) else None, - ) +_failed_request_by_span: Final[weakref.WeakKeyDictionary[Span, Mapping[str, object]]] = weakref.WeakKeyDictionary() # Any callback whose class belongs to one of these modules is "the OTel @@ -862,14 +851,16 @@ class OpenTelemetryV2(CustomLogger): this span already — ``async_post_call_failure_hook`` and the ``auth`` phase span record their own, and a second event would duplicate it — while the attributes are always restamped so ``error.code`` stays pinned to the real - response status.""" + response status. The restamp scrubs the text of the request the failure hook + stamped this span for, as the hook did.""" if span is None or not is_recordable_span(span): return - attributes: Final[Mapping[str, object]] = getattr(span, "attributes", None) or {} - already_stamped: Final = Error.TYPE in attributes + already_stamped: Final = Error.TYPE in (getattr(span, "attributes", None) or ()) stamp_error( span, - _keeping_scrubbed_text(_span_error_from_exception(exception, status_code=status_code), attributes), + _span_error_from_exception( + exception, status_code=status_code, request_data=_failed_request_by_span.get(span) + ), record_event=not already_stamped, ) @@ -895,6 +886,7 @@ class OpenTelemetryV2(CustomLogger): span: Final = mcp_message_transport_span() or request_root_span() or user_api_key_dict.parent_otel_span if span is None or not is_recordable_span(span): return + _failed_request_by_span[span] = request_data stamp_error( span, _span_error_from_exception(original_exception, traceback_str=traceback_str, request_data=request_data), diff --git a/tests/unit/integrations/otel/test_otel_v2_logger.py b/tests/unit/integrations/otel/test_otel_v2_logger.py index b2fd9e8638c..6817c80401a 100644 --- a/tests/unit/integrations/otel/test_otel_v2_logger.py +++ b/tests/unit/integrations/otel/test_otel_v2_logger.py @@ -1579,6 +1579,44 @@ def test_failure_hook_scrubs_the_prompt_out_of_the_server_span_and_the_restamp_k assert span.attributes["error.type"] == "ProxyException" +def _raised(exc): + try: + raise exc + except Exception as raised: + return raised + + +@pytest.mark.parametrize("redact", [True, False], ids=["redaction_on", "redaction_off"]) +def test_restamp_shows_the_handlers_own_error_with_the_prompt_scrubbed(monkeypatch, redact): + """The proxy's failure hook and its exception handler stamp the same SERVER span from different + exceptions. The handler's restamp keeps its own wording and stack trace, scrubbed like the hook's.""" + from fastapi import HTTPException + + import litellm + from litellm.proxy._types import UserAPIKeyAuth + + secret = "my ssn is 123-45-6789" + monkeypatch.setattr(litellm, "turn_off_message_logging", redact) + logger, exporter = _logger() + server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME) + set_request_root_span(server) + asyncio.run( + logger.async_post_call_failure_hook( + request_data={"messages": [{"role": "user", "content": secret}]}, + original_exception=HTTPException(status_code=400, detail={"error": f"blocked: {secret}", "mode": "pre"}), + user_api_key_dict=UserAPIKeyAuth(), + traceback_str=f' File "/srv/litellm/router.py", line 7, in route\nHTTPException: blocked: {secret}\n', + ) + ) + logger.record_error_attributes_on_span(server, _raised(_proxy_exc(f"blocked: {secret}", 500)), 500) + server.end() + (span,) = exporter.get_finished_spans() + + assert span.attributes["error.message"] == f"blocked: {'redacted-by-litellm' if redact else secret}" + assert "_raised" in span.attributes["litellm.provider.error.stack_trace"] + assert "router.py" not in span.attributes["litellm.provider.error.stack_trace"] + + def test_record_error_attributes_on_span_does_not_duplicate_an_already_stamped_error(): """A failure that already went through ``async_post_call_failure_hook`` reaches the exception handler too; the second stamp must keep one exception event while