scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-06 02:48:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

fix(logging): keep request-level failure redaction on otel restamp and phase spans

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yucheng 2026-09-29 10:31:14 +00:00
parent 63f91e7177
commit 37e483d39d
6 changed files with 220 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
litellm/integrations/opentelemetry.py
View file

@ -379,6 +379,23 @@ class OpenTelemetryConfig:
)
def _server_span_failure_redact(span: "Span") -> bool:
"""Redaction decision for re-stamping a SERVER span the failure hook may
have already marked: an existing error message/stack-trace attribute keeps
its request-aware value, so a restamp with the global-only probe can't leak
an opt-in's raw text or clobber a valid opt-out."""
from litellm.integrations._types.open_inference import ErrorAttributes
from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
attributes: Final = getattr(span, "attributes", None) or {}
if ErrorAttributes.ERROR_MESSAGE in attributes or ErrorAttributes.ERROR_STACK_TRACE in attributes:
return (
attributes.get(ErrorAttributes.ERROR_MESSAGE) == REDACTED_BY_LITELLM
or attributes.get(ErrorAttributes.ERROR_STACK_TRACE) == REDACTED_BY_LITELLM
)
return should_redact_message_logging({}) # mutable-ok: read-only probe for the global flag
class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
def __init__(
self,
@ -3596,12 +3613,9 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
from litellm.litellm_core_utils.litellm_logging import (
StandardLoggingPayloadSetup,
)
from litellm.litellm_core_utils.redact_messages import (
redact_error_information,
should_redact_message_logging,
)
from litellm.litellm_core_utils.redact_messages import redact_error_information
redact: Final = should_redact_message_logging({}) # mutable-ok: read-only probe for the global flag
redact: Final = _server_span_failure_redact(span)
error_information: Final = StandardLoggingPayloadSetup.get_error_information(original_exception=exception)
error_information["error_code"] = str(status_code)
self._record_exception_on_span(

34
litellm/integrations/otel/logger.py
View file

@ -49,7 +49,7 @@ from litellm.integrations.otel.model.payloads import (
is_mcp_list_tools,
is_mcp_tool_call,
)
from litellm.integrations.otel.model.semconv import Error
from litellm.integrations.otel.model.semconv import Error, LiteLLMError
from litellm.integrations.otel.model.spans import SpanRole, span_role_for_service
from litellm.integrations.otel.model.utils import to_ns
from litellm.integrations.otel.plumbing.context import (
@ -100,6 +100,20 @@ _published_v2_provider: ApiTracerProvider | None = None
_GLOBAL_REDACTION_PROBE: Final[dict[str, object]] = {} # mutable-ok: read-only global-redaction probe
def _span_failure_redact(span: "Span") -> bool:
"""Redaction decision for re-stamping a span the failure hook may have
already marked: an existing ``error.message``/stack-trace attribute keeps
its request-aware value, so a later restamp with the global probe can't
leak an opt-in's raw text or clobber a valid opt-out."""
attributes: Final = getattr(span, "attributes", None) or {}
if Error.MESSAGE in attributes or LiteLLMError.STACK_TRACE in attributes:
return (
attributes.get(Error.MESSAGE) == REDACTED_BY_LITELLM
or attributes.get(LiteLLMError.STACK_TRACE) == REDACTED_BY_LITELLM
)
return should_redact_message_logging(_GLOBAL_REDACTION_PROBE)
def _span_error_from_exception(
exception: "Exception | None",
*,
@ -747,18 +761,22 @@ class OpenTelemetryV2(CustomLogger):
@contextmanager
def start_phase_span(self, name: str) -> "Iterator[Span]":
span: Final = self._emitter.start_span(SpanRole.SERVICE, name)
with use_span(span, end_on_exit=True):
redact: Final = should_redact_message_logging(_GLOBAL_REDACTION_PROBE)
with use_span(
span,
end_on_exit=True,
record_exception=not redact,
set_status_on_exception=not redact,
):
try:
yield span
except Exception as exc:
if is_recordable_span(span):
stamp_error(
span,
_span_error_from_exception(
exc, redact_content=should_redact_message_logging(_GLOBAL_REDACTION_PROBE)
),
record_event=False,
set_status=False,
_span_error_from_exception(exc, redact_content=redact),
record_event=redact,
set_status=redact,
)
raise
@ -803,7 +821,7 @@ class OpenTelemetryV2(CustomLogger):
_span_error_from_exception(
exception,
status_code=status_code,
redact_content=should_redact_message_logging(_GLOBAL_REDACTION_PROBE),
redact_content=_span_failure_redact(span),
),
record_event=not already_stamped,
)

21
litellm/litellm_core_utils/redact_messages.py
View file

@ -21,6 +21,9 @@ from litellm.litellm_core_utils.classifier_logging import without_classifier_aud
from litellm.litellm_core_utils.core_helpers import (
get_metadata_variable_name_from_kwargs,
)
from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
iter_client_callback_metadata_dicts,
)
from litellm.litellm_core_utils.served_output_texts import SERVED_OUTPUT_TEXTS_KEY
from litellm.llms.vertex_ai.common_utils import (
redact_vertex_ai_metadata_from_litellm_params,
@ -221,7 +224,23 @@ def should_redact_failed_request(request_data: Mapping[str, object]) -> bool:
only receive ``request_data`` (the Logging object is popped before hooks run).
``litellm_metadata`` is included only when present so
``get_metadata_variable_name_from_kwargs`` resolves ``metadata`` for chat routes.
``turn_off_message_logging`` resolves like ``initialize_standard_callback_dynamic_params``:
the top-level value when present, else the first client-metadata slot carrying it.
"""
dynamic_param: Final = (
request_data["turn_off_message_logging"]
if "turn_off_message_logging" in request_data
else next(
(
slot["turn_off_message_logging"]
for _, slot in iter_client_callback_metadata_dicts(
cast(dict[str, Any], request_data) # cast-ok: the helper only reads mapping keys
)
if "turn_off_message_logging" in slot
),
None,
)
)
litellm_params: Final = MappingProxyType(
{
key: request_data.get(key)
@ -233,7 +252,7 @@ def should_redact_failed_request(request_data: Mapping[str, object]) -> bool:
{ # mutable-ok: the model_call_details shape the decision helper reads
"litellm_params": litellm_params,
"standard_callback_dynamic_params": { # mutable-ok: dynamic-params slot the helper reads
"turn_off_message_logging": request_data.get("turn_off_message_logging")
"turn_off_message_logging": dynamic_param
},
}
)

89
tests/unit/integrations/otel/test_otel_v2_logger.py
View file

@ -3292,3 +3292,92 @@ def test_async_post_call_failure_hook_keeps_error_text_when_not_gated():
server.end()
(span,) = exporter.get_finished_spans()
assert secret in span.attributes["error.message"]
def test_record_error_attributes_on_span_preserves_request_opt_in_redaction():
"""Global flag off but the request opts in via header: the failure hook
stamps redacted values on the SERVER span; the exception-handler restamp
must keep them instead of re-writing raw error text from the global probe."""
import litellm
from litellm.proxy._types import UserAPIKeyAuth
litellm.turn_off_message_logging = False
try:
logger, exporter = _logger()
server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME)
set_request_root_span(server)
secret = "secret-prompt-marker"
exc = _proxy_exc(f"Unsupported content: {secret}", 400)
asyncio.run(
logger.async_post_call_failure_hook(
request_data={"metadata": {"headers": {"x-litellm-enable-message-redaction": "true"}}},
original_exception=exc,
user_api_key_dict=UserAPIKeyAuth(),
traceback_str=f"Traceback ... {secret} ...",
)
)
logger.record_error_attributes_on_span(server, exc, 400)
server.end()
(span,) = exporter.get_finished_spans()
assert secret not in str(dict(span.attributes or {}))
assert secret not in str([dict(e.attributes or {}) for e in span.events])
assert secret not in str(span.status.description or "")
assert span.attributes["error.message"] == "redacted-by-litellm"
assert span.attributes["litellm.provider.error.code"] == "400"
finally:
litellm.turn_off_message_logging = False
def test_record_error_attributes_on_span_preserves_opt_out_raw_message():
"""Global flag on but the request opts out via header: the failure hook
stamps the raw error text; the restamp must not overwrite it with the
redaction marker."""
import litellm
from litellm.proxy._types import UserAPIKeyAuth
litellm.turn_off_message_logging = True
try:
logger, exporter = _logger()
server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME)
set_request_root_span(server)
secret = "secret-prompt-marker"
exc = _proxy_exc(f"Unsupported content: {secret}", 400)
asyncio.run(
logger.async_post_call_failure_hook(
request_data={"metadata": {"headers": {"litellm-disable-message-redaction": "true"}}},
original_exception=exc,
user_api_key_dict=UserAPIKeyAuth(),
traceback_str=f"Traceback ... {secret} ...",
)
)
logger.record_error_attributes_on_span(server, exc, 400)
server.end()
(span,) = exporter.get_finished_spans()
assert secret in span.attributes["error.message"]
assert span.attributes["litellm.provider.error.code"] == "400"
finally:
litellm.turn_off_message_logging = False
def test_start_phase_span_does_not_record_raw_exception_when_gated():
"""With message redaction on, an exception raised inside a phase span must
not leak through use_span's automatic exception recording: exactly one
exception event lands, carrying only the redaction marker."""
import litellm
litellm.turn_off_message_logging = True
try:
logger, exporter = _logger()
secret = "secret-prompt-marker"
with pytest.raises(RuntimeError):
with logger.start_phase_span("auth"):
raise RuntimeError(f"auth exploded: {secret}")
(span,) = exporter.get_finished_spans()
assert secret not in str(dict(span.attributes or {}))
assert secret not in str([dict(e.attributes or {}) for e in span.events])
assert secret not in str(span.status.description or "")
exception_events = [e for e in span.events if e.name == "exception"]
assert len(exception_events) == 1
assert (exception_events[0].attributes or {}).get("exception.message") == "redacted-by-litellm"
finally:
litellm.turn_off_message_logging = False

46
tests/unit/integrations/test_opentelemetry.py
View file

@ -6843,3 +6843,49 @@ class TestOpenTelemetryFailureHookRedaction(unittest.TestCase):
stamped = {call.args[0]: call.args[1] for call in span.set_attribute.call_args_list}
assert stamped.get("error.message") == "redacted-by-litellm"
assert stamped.get("error.type") == "BadRequestError"
def _restamped_span(self, stamped_message, redact, exception):
original = litellm.turn_off_message_logging
litellm.turn_off_message_logging = redact
try:
exporter = InMemorySpanExporter()
provider = TracerProvider()
provider.add_span_processor(SimpleSpanProcessor(exporter))
otel = OpenTelemetry()
otel.tracer = provider.get_tracer(__name__)
span = otel.tracer.start_span("Received Proxy Server Request")
otel.safe_set_attribute(span=span, key="error.message", value=stamped_message)
otel.record_error_attributes_on_span(span=span, exception=exception, status_code=400)
span.end()
return exporter.get_finished_spans()[0]
finally:
litellm.turn_off_message_logging = original
def test_record_error_attributes_on_span_preserves_request_opt_in_redaction(self):
"""Global flag off but the request opted in: the failure hook already
stamped the redaction marker on the SERVER span; the exception-handler
restamp must keep it instead of writing raw error text."""
span = self._restamped_span(
"redacted-by-litellm",
redact=False,
exception=litellm.BadRequestError(
message=f"Unsupported content: {SECRET_PROMPT}", model="gpt-4o", llm_provider="openai"
),
)
assert span.attributes["error.message"] == "redacted-by-litellm"
assert span.attributes["error.code"] == "400"
assert SECRET_PROMPT not in str(dict(span.attributes or {}))
def test_record_error_attributes_on_span_preserves_opt_out_raw_message(self):
"""Global flag on but the request opted out: the failure hook stamped
the raw error text; the restamp must not overwrite it with the
redaction marker."""
span = self._restamped_span(
f"Unsupported content: {SECRET_PROMPT}",
redact=True,
exception=litellm.BadRequestError(
message=f"Unsupported content: {SECRET_PROMPT}", model="gpt-4o", llm_provider="openai"
),
)
assert SECRET_PROMPT in span.attributes["error.message"]
assert span.attributes["error.code"] == "400"

20
tests/unit/litellm_core_utils/test_redact_messages.py
View file

@ -1177,3 +1177,23 @@ class TestShouldRedactFailedRequest:
litellm.turn_off_message_logging = True
assert should_redact_failed_request(_request_data(turn_off_message_logging=False)) is False
def test_dynamic_param_in_metadata_slot(self):
from litellm.litellm_core_utils.redact_messages import should_redact_failed_request
assert should_redact_failed_request(_request_data(metadata={"turn_off_message_logging": True})) is True
def test_dynamic_param_in_litellm_metadata_slot(self):
from litellm.litellm_core_utils.redact_messages import should_redact_failed_request
assert should_redact_failed_request(_request_data(litellm_metadata={"turn_off_message_logging": True})) is True
def test_top_level_dynamic_param_beats_metadata_slot(self):
from litellm.litellm_core_utils.redact_messages import should_redact_failed_request
litellm.turn_off_message_logging = True
request_data = _request_data(
metadata={"turn_off_message_logging": True},
turn_off_message_logging=False,
)
assert should_redact_failed_request(request_data) is False