fix(otel): honor request-level redaction opt-in on the auth phase span

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

litellm/integrations/otel/logger.py

@@ -759,9 +759,9 @@ class OpenTelemetryV2(CustomLogger):
             pass
 
     @contextmanager
-    def start_phase_span(self, name: str) -> "Iterator[Span]":
+    def start_phase_span(self, name: str, *, redact_content: bool = False) -> "Iterator[Span]":
         span: Final = self._emitter.start_span(SpanRole.SERVICE, name)
-        redact: Final = should_redact_message_logging(_GLOBAL_REDACTION_PROBE)
+        redact: Final = redact_content or should_redact_message_logging(_GLOBAL_REDACTION_PROBE)
         with use_span(
             span,
             end_on_exit=True,
@@ -1016,12 +1016,12 @@ def fan_out_provider() -> ApiTracerProvider:
 
 
 @contextmanager
-def phase_span(name: str) -> "Iterator[Span | None]":
+def phase_span(name: str, *, redact_content: bool = False) -> "Iterator[Span | None]":
     logger: Final = _registered_v2_logger()
     if logger is None:
         yield None
         return
-    with logger.start_phase_span(name) as span:
+    with logger.start_phase_span(name, redact_content=redact_content) as span:
         yield span
 
 

litellm/integrations/otel/runtime.py

@@ -17,7 +17,7 @@ if TYPE_CHECKING:
 
 
 @cache
-def _otel_runtime() -> "tuple[Callable[[str], AbstractContextManager[Span | None]], Callable[..., None]] | None":
+def _otel_runtime() -> "tuple[Callable[..., AbstractContextManager[Span | None]], Callable[..., None]] | None":
     """Resolve the SDK-backed hooks once and cache the outcome, absence included.
 
     CPython never caches a failed import, so without this memoization every call
@@ -32,7 +32,7 @@ def _otel_runtime() -> "tuple[Callable[[str], AbstractContextManager[Span | None
 
 
 @contextmanager
-def phase_span(name: str) -> "Iterator[Span | None]":
+def phase_span(name: str, *, redact_content: bool = False) -> "Iterator[Span | None]":
     """Run a request phase inside a live active span so its DB/service calls nest.
 
     Yields ``None`` (a plain no-op) when the OTel SDK is unavailable or V2 is not
@@ -42,7 +42,7 @@ def phase_span(name: str) -> "Iterator[Span | None]":
     if runtime is None:
         yield None
         return
-    with runtime[0](name) as span:
+    with runtime[0](name, redact_content=redact_content) as span:
         yield span
 
 

litellm/litellm_core_utils/redact_messages.py

@@ -192,6 +192,11 @@ def redacted_standard_logging_payload(payload: Mapping[str, object]) -> Mapping[
 
 _REDACTED_ERROR_FIELDS: Final = ("error_message", "traceback")
 
+_MESSAGE_REDACTION_ENABLE_HEADERS: Final = (
+    "litellm-enable-message-redaction",  # old header. maintain backwards compatibility
+    "x-litellm-enable-message-redaction",  # new header
+)
+
 
 def _is_non_empty_str(value: object) -> bool:
     return isinstance(value, str) and bool(value)
@@ -218,16 +223,10 @@ def redact_error_information(
     )
 
 
-def should_redact_failed_request(request_data: Mapping[str, object]) -> bool:
-    """
-    Message-redaction decision for proxy ``async_post_call_failure_hook`` callbacks, which
-    only receive ``request_data`` (the Logging object is popped before hooks run).
-    ``litellm_metadata`` is included only when present so
-    ``get_metadata_variable_name_from_kwargs`` resolves ``metadata`` for chat routes.
-    ``turn_off_message_logging`` resolves like ``initialize_standard_callback_dynamic_params``:
-    the top-level value when present, else the first client-metadata slot carrying it.
-    """
-    dynamic_param: Final = (
+def _request_turn_off_message_logging(request_data: Mapping[str, object]) -> object:
+    """``turn_off_message_logging`` resolves like ``initialize_standard_callback_dynamic_params``:
+    the top-level value when present, else the first client-metadata slot carrying it."""
+    return (
         request_data["turn_off_message_logging"]
         if "turn_off_message_logging" in request_data
         else next(
@@ -239,6 +238,26 @@ def should_redact_failed_request(request_data: Mapping[str, object]) -> bool:
             None,
         )
     )
+
+
+def request_opts_into_message_redaction(headers: Mapping[str, str], request_data: Mapping[str, object]) -> bool:
+    """Opt-in signals only: usable at auth time before the key's
+    ``allow_client_message_redaction_opt_out`` permission is known, so the disable
+    header and the global flag are deliberately not consulted."""
+    return (
+        any(bool(headers.get(header)) for header in _MESSAGE_REDACTION_ENABLE_HEADERS)
+        or _request_turn_off_message_logging(request_data) is True
+    )
+
+
+def should_redact_failed_request(request_data: Mapping[str, object]) -> bool:
+    """
+    Message-redaction decision for proxy ``async_post_call_failure_hook`` callbacks, which
+    only receive ``request_data`` (the Logging object is popped before hooks run).
+    ``litellm_metadata`` is included only when present so
+    ``get_metadata_variable_name_from_kwargs`` resolves ``metadata`` for chat routes.
+    """
+    dynamic_param: Final = _request_turn_off_message_logging(request_data)
     litellm_params: Final = MappingProxyType(
         {
             key: request_data.get(key)
@@ -446,13 +465,8 @@ def should_redact_message_logging(model_call_details: dict) -> bool:
         # User explicitly disabled redaction via header
         return False
 
-    possible_enable_headers: Final = [
-        "litellm-enable-message-redaction",  # old header. maintain backwards compatibility
-        "x-litellm-enable-message-redaction",  # new header
-    ]
-
     is_redaction_enabled_via_header = False
-    for header in possible_enable_headers:
+    for header in _MESSAGE_REDACTION_ENABLE_HEADERS:
         if bool(request_headers.get(header, False)):
             is_redaction_enabled_via_header = True
             break

litellm/proxy/auth/user_api_key_auth.py

@@ -38,6 +38,7 @@ from litellm.integrations.otel.model.config import is_otel_v2_enabled
 from litellm.integrations.otel.runtime import phase_span, seed_request_identity
 from litellm.litellm_core_utils.dd_tracing import tracer
 from litellm.litellm_core_utils.dot_notation_indexing import get_nested_value
+from litellm.litellm_core_utils.redact_messages import request_opts_into_message_redaction
 from litellm.proxy._types import *
 from litellm.proxy.agent_endpoints.auth.agent_caller import agent_caller_from_headers
 from litellm.proxy.auth.auth_checks import (
@@ -3329,7 +3330,13 @@ async def user_api_key_auth(
     # Run the whole auth phase inside a live ``auth`` span so the DB lookups it
     # triggers (key/user/team object reads) nest under it instead of flattening
     # onto the server span. No-op when OTel V2 isn't active.
-    with phase_span(f"auth {route}"), spend_counter_batch_scope(_spend_counter_redis_cache()):
+    with (
+        phase_span(
+            f"auth {route}",
+            redact_content=request_opts_into_message_redaction(request.headers, request_data),
+        ),
+        spend_counter_batch_scope(_spend_counter_redis_cache()),
+    ):
         try:
             user_api_key_auth_obj: Final = await _user_api_key_auth_builder(
                 request=request,

tests/unit/integrations/otel/test_otel_v2_logger.py

@@ -3381,3 +3381,47 @@ def test_start_phase_span_does_not_record_raw_exception_when_gated():
         assert (exception_events[0].attributes or {}).get("exception.message") == "redacted-by-litellm"
     finally:
         litellm.turn_off_message_logging = False
+
+
+def test_start_phase_span_redact_content_kwarg_redacts_exception():
+    """A caller-provided opt-in (e.g. an enable header seen at auth time, where
+    the key's opt-out permission is not known yet) must redact the phase-span
+    exception even when the global flag is off."""
+    import litellm
+
+    litellm.turn_off_message_logging = False
+    try:
+        logger, exporter = _logger()
+        secret = "secret-prompt-marker"
+        with pytest.raises(RuntimeError):
+            with logger.start_phase_span("auth", redact_content=True):
+                raise RuntimeError(f"auth exploded: {secret}")
+        (span,) = exporter.get_finished_spans()
+        assert secret not in str(dict(span.attributes or {}))
+        assert secret not in str([dict(e.attributes or {}) for e in span.events])
+        assert secret not in str(span.status.description or "")
+        exception_events = [e for e in span.events if e.name == "exception"]
+        assert len(exception_events) == 1
+        assert (exception_events[0].attributes or {}).get("exception.message") == "redacted-by-litellm"
+    finally:
+        litellm.turn_off_message_logging = False
+
+
+def test_start_phase_span_without_opt_in_keeps_raw_exception():
+    """The default ``redact_content=False`` must leave use_span's own raw
+    exception recording untouched when the global flag is off."""
+    import litellm
+
+    litellm.turn_off_message_logging = False
+    try:
+        logger, exporter = _logger()
+        secret = "secret-prompt-marker"
+        with pytest.raises(RuntimeError):
+            with logger.start_phase_span("auth"):
+                raise RuntimeError(f"auth exploded: {secret}")
+        (span,) = exporter.get_finished_spans()
+        exception_events = [e for e in span.events if e.name == "exception"]
+        assert len(exception_events) == 1
+        assert secret in str(dict(exception_events[0].attributes or {}))
+    finally:
+        litellm.turn_off_message_logging = False

tests/unit/integrations/otel/test_runtime.py

@@ -9,6 +9,8 @@ import lock. These tests pin the import to a single resolution.
 
 import builtins
 
+import pytest
+
 import litellm.integrations.otel.runtime as runtime
 
 
@@ -62,3 +64,36 @@ def test_wrappers_no_op_when_runtime_absent(monkeypatch):
         assert span is None
 
     assert runtime.seed_request_identity({"token": "sk-x"}, model="gpt-4o") is None
+
+
+def test_phase_span_forwards_redact_content_to_registered_logger(monkeypatch):
+    """``redact_content`` must reach the v2 logger through the runtime shim: an
+    exception raised inside the forwarded span carries only the redaction marker,
+    even with the global flag off."""
+    pytest.importorskip("opentelemetry")
+
+    from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+
+    import litellm
+    import litellm.integrations.otel.logger as otel_logger
+    from litellm.integrations.otel import OpenTelemetryV2Config
+    from litellm.integrations.otel.logger import OpenTelemetryV2
+    from litellm.integrations.otel.plumbing import providers
+
+    cfg = OpenTelemetryV2Config(exporter="in_memory", legacy_compat=True, baggage_team_metadata_keys=[])
+    exporter = InMemorySpanExporter()
+    logger = OpenTelemetryV2(config=cfg, tracer_provider=providers.build_tracer_provider(cfg, exporter=exporter))
+    monkeypatch.setattr(otel_logger, "_registered_v2_logger", lambda: logger)
+
+    litellm.turn_off_message_logging = False
+    try:
+        secret = "secret-prompt-marker"
+        with pytest.raises(RuntimeError):
+            with runtime.phase_span("auth", redact_content=True):
+                raise RuntimeError(f"auth exploded: {secret}")
+        (span,) = exporter.get_finished_spans()
+        assert secret not in str(dict(span.attributes or {}))
+        assert secret not in str([dict(e.attributes or {}) for e in span.events])
+        assert secret not in str(span.status.description or "")
+    finally:
+        litellm.turn_off_message_logging = False

tests/unit/litellm_core_utils/test_redact_messages.py

@@ -1197,3 +1197,36 @@ class TestShouldRedactFailedRequest:
             turn_off_message_logging=False,
         )
         assert should_redact_failed_request(request_data) is False
+
+
+class TestRequestOptsIntoMessageRedaction:
+    @pytest.mark.parametrize("header", ["litellm-enable-message-redaction", "x-litellm-enable-message-redaction"])
+    def test_enable_header(self, header: str) -> None:
+        from litellm.litellm_core_utils.redact_messages import request_opts_into_message_redaction
+
+        assert request_opts_into_message_redaction({header: "true"}, {}) is True
+
+    def test_top_level_dynamic_param(self) -> None:
+        from litellm.litellm_core_utils.redact_messages import request_opts_into_message_redaction
+
+        assert request_opts_into_message_redaction({}, {"turn_off_message_logging": True}) is True
+
+    def test_metadata_slot_dynamic_param(self) -> None:
+        from litellm.litellm_core_utils.redact_messages import request_opts_into_message_redaction
+
+        assert request_opts_into_message_redaction({}, {"metadata": {"turn_off_message_logging": True}}) is True
+
+    def test_empty_inputs(self) -> None:
+        from litellm.litellm_core_utils.redact_messages import request_opts_into_message_redaction
+
+        assert request_opts_into_message_redaction({}, {}) is False
+
+    def test_disable_header_alone_is_not_an_opt_in(self) -> None:
+        from litellm.litellm_core_utils.redact_messages import request_opts_into_message_redaction
+
+        assert request_opts_into_message_redaction({"litellm-disable-message-redaction": "true"}, {}) is False
+
+    def test_dynamic_param_false(self) -> None:
+        from litellm.litellm_core_utils.redact_messages import request_opts_into_message_redaction
+
+        assert request_opts_into_message_redaction({}, {"turn_off_message_logging": False}) is False