From b100051babbe89ca4d5f7fd5ea1f024c5d43caac Mon Sep 17 00:00:00 2001 From: yucheng Date: Wed, 4 Nov 2026 19:09:09 -0800 Subject: [PATCH] fix(logging): record the exception's own frames on OTel and MLflow when a redacted failure logged no traceback --- litellm/integrations/mlflow.py | 4 +- litellm/integrations/opentelemetry.py | 4 +- tests/unit/integrations/test_mlflow.py | 40 +++++++++++++++++++ tests/unit/integrations/test_opentelemetry.py | 37 +++++++++++++++++ 4 files changed, 81 insertions(+), 4 deletions(-) diff --git a/litellm/integrations/mlflow.py b/litellm/integrations/mlflow.py index b338567d248..f58a600ae56 100644 --- a/litellm/integrations/mlflow.py +++ b/litellm/integrations/mlflow.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Any, Final from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger -from litellm.litellm_core_utils.redact_messages import scrubbed_exception_text +from litellm.litellm_core_utils.redact_messages import scrubbed_exception_record if TYPE_CHECKING: from litellm.types.utils import StandardLoggingPayload @@ -90,7 +90,7 @@ class MlflowLogger(CustomLogger): # Record exception info as event if exception := kwargs.get("exception"): - scrubbed: Final = scrubbed_exception_text(kwargs) + scrubbed: Final = scrubbed_exception_record(kwargs) span.add_event( SpanEvent.from_exception(exception) if scrubbed is None diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 41ab1e98079..fecd40dfc9b 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -36,7 +36,7 @@ from litellm.integrations.otel.model.semconv import LiteLLM, Metric from litellm.integrations.otel.plumbing.otlp_tls import resolve_otlp_http_tls from litellm.integrations.otel.routing import routing_decision_attributes from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params -from litellm.litellm_core_utils.redact_messages import scrubbed_exception_text +from litellm.litellm_core_utils.redact_messages import scrubbed_exception_record from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.secret_redaction import redact_string from litellm.litellm_core_utils.service_tier_utils import ( @@ -2272,7 +2272,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): # Get the exception object if available exception: Final = kwargs.get("exception") - scrubbed: Final = scrubbed_exception_text(kwargs) + scrubbed: Final = scrubbed_exception_record(kwargs) if exception is not None and scrubbed is not None: span.record_exception( diff --git a/tests/unit/integrations/test_mlflow.py b/tests/unit/integrations/test_mlflow.py index 915c0bfa786..516c24b27cc 100644 --- a/tests/unit/integrations/test_mlflow.py +++ b/tests/unit/integrations/test_mlflow.py @@ -2,6 +2,7 @@ import asyncio import json import threading from datetime import datetime +from typing import Final from unittest.mock import MagicMock, patch # Adds the grandparent directory to sys.path to allow importing project modules @@ -321,3 +322,42 @@ def test_mlflow_failure_event_records_the_error_text_the_payload_logged(scrubbed assert event.attributes["exception.type"] == "ValueError" assert event.attributes["exception.message"] == error_message assert event.attributes["exception.stacktrace"] == ("frames" if scrubbed else "raw") + + +def _raise_provider_error(message: str) -> None: + raise ValueError(message) + + +def test_mlflow_failure_event_records_the_exceptions_own_frames_when_no_traceback_was_logged() -> None: + secret: Final = "my ssn is 123-45-6789" + modules: Final = _mock_mlflow_modules() + modules["mlflow"].get_current_active_span.return_value = None + modules["mlflow.entities"].SpanEvent = _ExceptionSpanEvent + with pytest.raises(ValueError, match="bad input") as raised: + _raise_provider_error(f"bad input: {secret}") + with patch.dict("sys.modules", modules): + from litellm.integrations.mlflow import MlflowLogger + + mlflow_logger: Final = MlflowLogger() + span: Final = MagicMock(parent_id=None, request_id="req-1") + mlflow_logger._client = MagicMock(**{"start_trace.return_value": span}) + + mlflow_logger._handle_failure( + kwargs={ + "litellm_call_id": "x", + "messages": [{"role": "user", "content": "hi"}], + "exception": raised.value, + "standard_logging_object": { + "request_tags": [], + "error_information": {"error_message": "bad input: redacted-by-litellm", "traceback": ""}, + }, + }, + response_obj=None, + start_time=datetime(2025, 1, 1), + end_time=datetime(2025, 1, 1), + ) + + stacktrace: Final = span.add_event.call_args.args[0].attributes["exception.stacktrace"] + assert secret not in stacktrace + assert "in _raise_provider_error" in stacktrace + assert stacktrace.endswith("ValueError: bad input: redacted-by-litellm\n") diff --git a/tests/unit/integrations/test_opentelemetry.py b/tests/unit/integrations/test_opentelemetry.py index c8cc157611b..e3a06acc716 100644 --- a/tests/unit/integrations/test_opentelemetry.py +++ b/tests/unit/integrations/test_opentelemetry.py @@ -7672,3 +7672,40 @@ def test_record_exception_on_span_records_the_scrubbed_error_text_with_the_same_ assert event.attributes["exception.stacktrace"] == "frames" assert event.attributes["exception.type"] == "ValueError" assert event.attributes["exception.escaped"] == "False" + + +def _raise_provider_error(message: str) -> None: + raise ValueError(message) + + +def test_record_exception_on_span_records_the_exceptions_own_frames_when_no_traceback_was_logged() -> None: + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + + from litellm.integrations.opentelemetry import OpenTelemetry + + exporter: Final = InMemorySpanExporter() + provider: Final = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + secret: Final = "my ssn is 123-45-6789" + with pytest.raises(ValueError, match="bad input") as raised: + _raise_provider_error(f"bad input: {secret}") + + with provider.get_tracer(__name__).start_as_current_span("failure") as span: + OpenTelemetry()._record_exception_on_span( + span=span, + kwargs={ + "exception": raised.value, + "standard_logging_object": { + "error_information": {"error_message": "bad input: redacted-by-litellm", "traceback": ""} + }, + }, + ) + + (event,) = exporter.get_finished_spans()[0].events + stacktrace: Final = str(event.attributes["exception.stacktrace"]) + assert secret not in str(dict(event.attributes)) + assert stacktrace.startswith("Traceback (most recent call last):\n") + assert "in _raise_provider_error" in stacktrace + assert stacktrace.endswith("ValueError: bad input: redacted-by-litellm\n")