diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 0ff42736310..2b3c9e398b3 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -379,6 +379,23 @@ class OpenTelemetryConfig: ) +def _server_span_failure_redact(span: "Span") -> bool: + """Redaction decision for re-stamping a SERVER span the failure hook may + have already marked: an existing error message/stack-trace attribute keeps + its request-aware value, so a restamp with the global-only probe can't leak + an opt-in's raw text or clobber a valid opt-out.""" + from litellm.integrations._types.open_inference import ErrorAttributes + from litellm.litellm_core_utils.redact_messages import should_redact_message_logging + + attributes: Final = getattr(span, "attributes", None) or {} + if ErrorAttributes.ERROR_MESSAGE in attributes or ErrorAttributes.ERROR_STACK_TRACE in attributes: + return ( + attributes.get(ErrorAttributes.ERROR_MESSAGE) == REDACTED_BY_LITELLM + or attributes.get(ErrorAttributes.ERROR_STACK_TRACE) == REDACTED_BY_LITELLM + ) + return should_redact_message_logging({}) # mutable-ok: read-only probe for the global flag + + class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): def __init__( self, @@ -3596,12 +3613,9 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): from litellm.litellm_core_utils.litellm_logging import ( StandardLoggingPayloadSetup, ) - from litellm.litellm_core_utils.redact_messages import ( - redact_error_information, - should_redact_message_logging, - ) + from litellm.litellm_core_utils.redact_messages import redact_error_information - redact: Final = should_redact_message_logging({}) # mutable-ok: read-only probe for the global flag + redact: Final = _server_span_failure_redact(span) error_information: Final = StandardLoggingPayloadSetup.get_error_information(original_exception=exception) error_information["error_code"] = str(status_code) self._record_exception_on_span( diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index b974cf6086b..817b44fd984 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -49,7 +49,7 @@ from litellm.integrations.otel.model.payloads import ( is_mcp_list_tools, is_mcp_tool_call, ) -from litellm.integrations.otel.model.semconv import Error +from litellm.integrations.otel.model.semconv import Error, LiteLLMError from litellm.integrations.otel.model.spans import SpanRole, span_role_for_service from litellm.integrations.otel.model.utils import to_ns from litellm.integrations.otel.plumbing.context import ( @@ -100,6 +100,20 @@ _published_v2_provider: ApiTracerProvider | None = None _GLOBAL_REDACTION_PROBE: Final[dict[str, object]] = {} # mutable-ok: read-only global-redaction probe +def _span_failure_redact(span: "Span") -> bool: + """Redaction decision for re-stamping a span the failure hook may have + already marked: an existing ``error.message``/stack-trace attribute keeps + its request-aware value, so a later restamp with the global probe can't + leak an opt-in's raw text or clobber a valid opt-out.""" + attributes: Final = getattr(span, "attributes", None) or {} + if Error.MESSAGE in attributes or LiteLLMError.STACK_TRACE in attributes: + return ( + attributes.get(Error.MESSAGE) == REDACTED_BY_LITELLM + or attributes.get(LiteLLMError.STACK_TRACE) == REDACTED_BY_LITELLM + ) + return should_redact_message_logging(_GLOBAL_REDACTION_PROBE) + + def _span_error_from_exception( exception: "Exception | None", *, @@ -747,18 +761,22 @@ class OpenTelemetryV2(CustomLogger): @contextmanager def start_phase_span(self, name: str) -> "Iterator[Span]": span: Final = self._emitter.start_span(SpanRole.SERVICE, name) - with use_span(span, end_on_exit=True): + redact: Final = should_redact_message_logging(_GLOBAL_REDACTION_PROBE) + with use_span( + span, + end_on_exit=True, + record_exception=not redact, + set_status_on_exception=not redact, + ): try: yield span except Exception as exc: if is_recordable_span(span): stamp_error( span, - _span_error_from_exception( - exc, redact_content=should_redact_message_logging(_GLOBAL_REDACTION_PROBE) - ), - record_event=False, - set_status=False, + _span_error_from_exception(exc, redact_content=redact), + record_event=redact, + set_status=redact, ) raise @@ -803,7 +821,7 @@ class OpenTelemetryV2(CustomLogger): _span_error_from_exception( exception, status_code=status_code, - redact_content=should_redact_message_logging(_GLOBAL_REDACTION_PROBE), + redact_content=_span_failure_redact(span), ), record_event=not already_stamped, ) diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index f846bb8ac4f..a4b01c22875 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -21,6 +21,9 @@ from litellm.litellm_core_utils.classifier_logging import without_classifier_aud from litellm.litellm_core_utils.core_helpers import ( get_metadata_variable_name_from_kwargs, ) +from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + iter_client_callback_metadata_dicts, +) from litellm.litellm_core_utils.served_output_texts import SERVED_OUTPUT_TEXTS_KEY from litellm.llms.vertex_ai.common_utils import ( redact_vertex_ai_metadata_from_litellm_params, @@ -221,7 +224,23 @@ def should_redact_failed_request(request_data: Mapping[str, object]) -> bool: only receive ``request_data`` (the Logging object is popped before hooks run). ``litellm_metadata`` is included only when present so ``get_metadata_variable_name_from_kwargs`` resolves ``metadata`` for chat routes. + ``turn_off_message_logging`` resolves like ``initialize_standard_callback_dynamic_params``: + the top-level value when present, else the first client-metadata slot carrying it. """ + dynamic_param: Final = ( + request_data["turn_off_message_logging"] + if "turn_off_message_logging" in request_data + else next( + ( + slot["turn_off_message_logging"] + for _, slot in iter_client_callback_metadata_dicts( + cast(dict[str, Any], request_data) # cast-ok: the helper only reads mapping keys + ) + if "turn_off_message_logging" in slot + ), + None, + ) + ) litellm_params: Final = MappingProxyType( { key: request_data.get(key) @@ -233,7 +252,7 @@ def should_redact_failed_request(request_data: Mapping[str, object]) -> bool: { # mutable-ok: the model_call_details shape the decision helper reads "litellm_params": litellm_params, "standard_callback_dynamic_params": { # mutable-ok: dynamic-params slot the helper reads - "turn_off_message_logging": request_data.get("turn_off_message_logging") + "turn_off_message_logging": dynamic_param }, } ) diff --git a/tests/unit/integrations/otel/test_otel_v2_logger.py b/tests/unit/integrations/otel/test_otel_v2_logger.py index 1efc7a1053b..95b9b7df05a 100644 --- a/tests/unit/integrations/otel/test_otel_v2_logger.py +++ b/tests/unit/integrations/otel/test_otel_v2_logger.py @@ -3292,3 +3292,92 @@ def test_async_post_call_failure_hook_keeps_error_text_when_not_gated(): server.end() (span,) = exporter.get_finished_spans() assert secret in span.attributes["error.message"] + + +def test_record_error_attributes_on_span_preserves_request_opt_in_redaction(): + """Global flag off but the request opts in via header: the failure hook + stamps redacted values on the SERVER span; the exception-handler restamp + must keep them instead of re-writing raw error text from the global probe.""" + import litellm + from litellm.proxy._types import UserAPIKeyAuth + + litellm.turn_off_message_logging = False + try: + logger, exporter = _logger() + server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME) + set_request_root_span(server) + secret = "secret-prompt-marker" + exc = _proxy_exc(f"Unsupported content: {secret}", 400) + asyncio.run( + logger.async_post_call_failure_hook( + request_data={"metadata": {"headers": {"x-litellm-enable-message-redaction": "true"}}}, + original_exception=exc, + user_api_key_dict=UserAPIKeyAuth(), + traceback_str=f"Traceback ... {secret} ...", + ) + ) + logger.record_error_attributes_on_span(server, exc, 400) + server.end() + (span,) = exporter.get_finished_spans() + assert secret not in str(dict(span.attributes or {})) + assert secret not in str([dict(e.attributes or {}) for e in span.events]) + assert secret not in str(span.status.description or "") + assert span.attributes["error.message"] == "redacted-by-litellm" + assert span.attributes["litellm.provider.error.code"] == "400" + finally: + litellm.turn_off_message_logging = False + + +def test_record_error_attributes_on_span_preserves_opt_out_raw_message(): + """Global flag on but the request opts out via header: the failure hook + stamps the raw error text; the restamp must not overwrite it with the + redaction marker.""" + import litellm + from litellm.proxy._types import UserAPIKeyAuth + + litellm.turn_off_message_logging = True + try: + logger, exporter = _logger() + server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME) + set_request_root_span(server) + secret = "secret-prompt-marker" + exc = _proxy_exc(f"Unsupported content: {secret}", 400) + asyncio.run( + logger.async_post_call_failure_hook( + request_data={"metadata": {"headers": {"litellm-disable-message-redaction": "true"}}}, + original_exception=exc, + user_api_key_dict=UserAPIKeyAuth(), + traceback_str=f"Traceback ... {secret} ...", + ) + ) + logger.record_error_attributes_on_span(server, exc, 400) + server.end() + (span,) = exporter.get_finished_spans() + assert secret in span.attributes["error.message"] + assert span.attributes["litellm.provider.error.code"] == "400" + finally: + litellm.turn_off_message_logging = False + + +def test_start_phase_span_does_not_record_raw_exception_when_gated(): + """With message redaction on, an exception raised inside a phase span must + not leak through use_span's automatic exception recording: exactly one + exception event lands, carrying only the redaction marker.""" + import litellm + + litellm.turn_off_message_logging = True + try: + logger, exporter = _logger() + secret = "secret-prompt-marker" + with pytest.raises(RuntimeError): + with logger.start_phase_span("auth"): + raise RuntimeError(f"auth exploded: {secret}") + (span,) = exporter.get_finished_spans() + assert secret not in str(dict(span.attributes or {})) + assert secret not in str([dict(e.attributes or {}) for e in span.events]) + assert secret not in str(span.status.description or "") + exception_events = [e for e in span.events if e.name == "exception"] + assert len(exception_events) == 1 + assert (exception_events[0].attributes or {}).get("exception.message") == "redacted-by-litellm" + finally: + litellm.turn_off_message_logging = False diff --git a/tests/unit/integrations/test_opentelemetry.py b/tests/unit/integrations/test_opentelemetry.py index 8f6cb5f47ab..dcb553ca18e 100644 --- a/tests/unit/integrations/test_opentelemetry.py +++ b/tests/unit/integrations/test_opentelemetry.py @@ -6843,3 +6843,49 @@ class TestOpenTelemetryFailureHookRedaction(unittest.TestCase): stamped = {call.args[0]: call.args[1] for call in span.set_attribute.call_args_list} assert stamped.get("error.message") == "redacted-by-litellm" assert stamped.get("error.type") == "BadRequestError" + + def _restamped_span(self, stamped_message, redact, exception): + original = litellm.turn_off_message_logging + litellm.turn_off_message_logging = redact + try: + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + otel = OpenTelemetry() + otel.tracer = provider.get_tracer(__name__) + span = otel.tracer.start_span("Received Proxy Server Request") + otel.safe_set_attribute(span=span, key="error.message", value=stamped_message) + otel.record_error_attributes_on_span(span=span, exception=exception, status_code=400) + span.end() + return exporter.get_finished_spans()[0] + finally: + litellm.turn_off_message_logging = original + + def test_record_error_attributes_on_span_preserves_request_opt_in_redaction(self): + """Global flag off but the request opted in: the failure hook already + stamped the redaction marker on the SERVER span; the exception-handler + restamp must keep it instead of writing raw error text.""" + span = self._restamped_span( + "redacted-by-litellm", + redact=False, + exception=litellm.BadRequestError( + message=f"Unsupported content: {SECRET_PROMPT}", model="gpt-4o", llm_provider="openai" + ), + ) + assert span.attributes["error.message"] == "redacted-by-litellm" + assert span.attributes["error.code"] == "400" + assert SECRET_PROMPT not in str(dict(span.attributes or {})) + + def test_record_error_attributes_on_span_preserves_opt_out_raw_message(self): + """Global flag on but the request opted out: the failure hook stamped + the raw error text; the restamp must not overwrite it with the + redaction marker.""" + span = self._restamped_span( + f"Unsupported content: {SECRET_PROMPT}", + redact=True, + exception=litellm.BadRequestError( + message=f"Unsupported content: {SECRET_PROMPT}", model="gpt-4o", llm_provider="openai" + ), + ) + assert SECRET_PROMPT in span.attributes["error.message"] + assert span.attributes["error.code"] == "400" diff --git a/tests/unit/litellm_core_utils/test_redact_messages.py b/tests/unit/litellm_core_utils/test_redact_messages.py index d51737dc182..c21893bb5c3 100644 --- a/tests/unit/litellm_core_utils/test_redact_messages.py +++ b/tests/unit/litellm_core_utils/test_redact_messages.py @@ -1177,3 +1177,23 @@ class TestShouldRedactFailedRequest: litellm.turn_off_message_logging = True assert should_redact_failed_request(_request_data(turn_off_message_logging=False)) is False + + def test_dynamic_param_in_metadata_slot(self): + from litellm.litellm_core_utils.redact_messages import should_redact_failed_request + + assert should_redact_failed_request(_request_data(metadata={"turn_off_message_logging": True})) is True + + def test_dynamic_param_in_litellm_metadata_slot(self): + from litellm.litellm_core_utils.redact_messages import should_redact_failed_request + + assert should_redact_failed_request(_request_data(litellm_metadata={"turn_off_message_logging": True})) is True + + def test_top_level_dynamic_param_beats_metadata_slot(self): + from litellm.litellm_core_utils.redact_messages import should_redact_failed_request + + litellm.turn_off_message_logging = True + request_data = _request_data( + metadata={"turn_off_message_logging": True}, + turn_off_message_logging=False, + ) + assert should_redact_failed_request(request_data) is False