From 1c2ebb37b419629fabe52725125fed3b33741166 Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 29 Oct 2026 15:59:35 -0700 Subject: [PATCH] fix(logging): scrub the request's prompt out of failure error text when message logging is redacted When message logging is redacted (globally, per request, or per callback), the failure path still sent the provider's error text, which often quotes the prompt, to every callback and to the proxy's spend log, DataDog and OTel server span. Scrub the request's own text out of error_str, error_information and traceback_exception instead of blanking them, so the error type, status code and the rest of the wording stay. Also carry a per-request redaction opt-in over to the proxy-only failure log, whose field allowlist left it out. --- litellm/integrations/custom_logger.py | 45 +-- litellm/integrations/datadog/datadog.py | 14 +- litellm/integrations/mlflow.py | 17 +- litellm/integrations/opentelemetry.py | 10 +- litellm/integrations/otel/logger.py | 37 ++- litellm/litellm_core_utils/litellm_logging.py | 35 ++- litellm/litellm_core_utils/redact_messages.py | 256 +++++++++++++++++- .../proxy/hooks/proxy_track_cost_callback.py | 16 +- litellm/proxy/utils.py | 4 + .../datadog/test_datadog_logger_batching.py | 25 ++ .../integrations/otel/test_otel_v2_logger.py | 31 +++ tests/unit/integrations/test_mlflow.py | 51 ++++ tests/unit/integrations/test_opentelemetry.py | 36 +++ .../test_litellm_logging.py | 157 +++++++++++ .../test_redact_messages.py | 88 ++++++ .../hooks/test_proxy_track_cost_callback.py | 27 ++ ..._utils_model_creation_and_error_logging.py | 30 ++ 17 files changed, 806 insertions(+), 73 deletions(-) diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 0579eb15baf..9330e7c16f7 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -3,13 +3,12 @@ import re import traceback from collections.abc import AsyncGenerator, Mapping, Sequence -from types import MappingProxyType from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional from pydantic import BaseModel from litellm._logging import verbose_logger -from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER, EMPTY_MAPPING +from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER from litellm.types.integrations.argilla import ArgillaItem from litellm.types.integrations.custom_logger import AgenticLoopPlan from litellm.types.llms.openai import AllMessageValues, ChatCompletionRequest @@ -912,46 +911,12 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac This is useful for logging payloads that contain sensitive information. """ import litellm - from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS - from litellm.litellm_core_utils.redact_messages import ( - redacted_litellm_params, - redacted_standard_logging_payload, + from litellm.litellm_core_utils.redact_messages import redacted_standard_logging_details + + return redacted_standard_logging_details( + model_call_details, self, excluded_fields=getattr(litellm, "standard_logging_payload_excluded_fields", None) ) - turn_off_message_logging: Final[bool] = getattr(self, "turn_off_message_logging", False) - excluded_fields: Final[list[str] | None] = getattr(litellm, "standard_logging_payload_excluded_fields", None) - - # Early return if no processing needed - if turn_off_message_logging is False and not excluded_fields: - return model_call_details - - params: Final = model_call_details.get("litellm_params") - redacted_params: Final = ( - MappingProxyType({"litellm_params": redacted_litellm_params(params)}) - if turn_off_message_logging and isinstance(params, Mapping) - else EMPTY_MAPPING - ) - standard_logging_object: Final = model_call_details.get("standard_logging_object") - if standard_logging_object is None: - return {**model_call_details, **redacted_params} - - # Make a copy of just the standard_logging_object to avoid modifying the original - standard_logging_object_copy: Final = { - key: value - for key, value in standard_logging_object.items() - if key not in (excluded_fields or ()) and not (turn_off_message_logging and key in CLASSIFIER_AUDIT_FIELDS) - } - - return { - **model_call_details, - **redacted_params, - "standard_logging_object": ( - redacted_standard_logging_payload(standard_logging_object_copy) - if turn_off_message_logging and not self.redacts_messages_itself() - else standard_logging_object_copy - ), - } - async def get_proxy_server_request_from_cold_storage_with_object_key( self, object_key: str, diff --git a/litellm/integrations/datadog/datadog.py b/litellm/integrations/datadog/datadog.py index 6518cda5ae3..3c569ff4fe9 100644 --- a/litellm/integrations/datadog/datadog.py +++ b/litellm/integrations/datadog/datadog.py @@ -18,7 +18,7 @@ import datetime import os import time import traceback -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from datetime import datetime as datetimeObj from typing import TYPE_CHECKING, Any, Final @@ -287,7 +287,7 @@ class DataDogLogger( async def async_post_call_failure_hook( self, - request_data: dict, + request_data: Mapping[str, object], original_exception: Exception, user_api_key_dict: "UserAPIKeyAuth", traceback_str: str | None = None, @@ -303,11 +303,15 @@ class DataDogLogger( from litellm.litellm_core_utils.litellm_logging import ( StandardLoggingPayloadSetup, ) + from litellm.litellm_core_utils.redact_messages import scrubbed_error_information_for_request from litellm.litellm_core_utils.safe_json_dumps import safe_dumps - error_information: Final = StandardLoggingPayloadSetup.get_error_information( - original_exception=original_exception, - traceback_str=traceback_str, + error_information: Final = scrubbed_error_information_for_request( + StandardLoggingPayloadSetup.get_error_information( + original_exception=original_exception, + traceback_str=traceback_str, + ), + request_data, ) _code: Final = error_information.get("error_code") or "" status_code: int | None = None diff --git a/litellm/integrations/mlflow.py b/litellm/integrations/mlflow.py index 8731e96440f..740ac36716a 100644 --- a/litellm/integrations/mlflow.py +++ b/litellm/integrations/mlflow.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.redact_messages import scrubbed_exception_text if TYPE_CHECKING: from litellm.types.utils import StandardLoggingPayload @@ -86,9 +87,21 @@ class MlflowLogger(CustomLogger): end_time_ns: Final = int(end_time.timestamp() * 1e9) - # Record exception info as event + # Record exception info as event, with the logged text when redaction scrubbed it if exception := kwargs.get("exception"): - span.add_event(SpanEvent.from_exception(exception)) + scrubbed: Final = scrubbed_exception_text(kwargs) + span.add_event( + SpanEvent.from_exception(exception) + if scrubbed is None + else SpanEvent( + name="exception", + attributes={ + "exception.message": scrubbed[0], + "exception.type": exception.__class__.__name__, + "exception.stacktrace": scrubbed[1], + }, + ) + ) self._extract_and_set_chat_attributes(span, kwargs, response_obj) self._end_span_or_trace( diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 0d0e309034a..e5d936f8dce 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -36,6 +36,7 @@ from litellm.integrations.otel.model.semconv import LiteLLM, Metric from litellm.integrations.otel.plumbing.otlp_tls import resolve_otlp_http_tls from litellm.integrations.otel.routing import routing_decision_attributes from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params +from litellm.litellm_core_utils.redact_messages import scrubbed_exception_text from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.secret_redaction import redact_string from litellm.litellm_core_utils.service_tier_utils import ( @@ -2271,9 +2272,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): # Get the exception object if available exception: Final = kwargs.get("exception") + scrubbed: Final = scrubbed_exception_text(kwargs) - # Record the exception using OTEL's standard method - if exception is not None: + # Record the exception using OTEL's standard method, with the logged text when redaction scrubbed it + if exception is not None and scrubbed is not None: + span.record_exception( + exception, attributes={"exception.message": scrubbed[0], "exception.stacktrace": scrubbed[1]} + ) + elif exception is not None: span.record_exception(exception) # Get StandardLoggingPayload for structured error information diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index b5b65d299e0..6297abaf4ac 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -27,6 +27,7 @@ from typing_extensions import TypedDict, Unpack import litellm from litellm._logging import verbose_logger +from litellm.constants import REDACTED_BY_LITELLM from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.otel.emitter import SpanEmitter, stamp_error from litellm.integrations.otel.mappers import resolve_mappers @@ -49,7 +50,7 @@ from litellm.integrations.otel.model.payloads import ( is_mcp_list_tools, is_mcp_tool_call, ) -from litellm.integrations.otel.model.semconv import Error +from litellm.integrations.otel.model.semconv import Error, LiteLLMError from litellm.integrations.otel.model.spans import SpanRole, span_role_for_service from litellm.integrations.otel.model.utils import to_ns from litellm.integrations.otel.plumbing.context import ( @@ -106,18 +107,22 @@ def _span_error_from_exception( *, status_code: int | None = None, traceback_str: str | None = None, + request_data: Mapping[str, object] | None = None, ) -> SpanError: """A ``SpanError`` for a proxy-level failure that never produced a ``StandardLoggingPayload`` (auth / validation / malformed-body rejections), mirroring ``_parse_error``'s field mapping so it stamps the same v2 keys a failed LLM call does. ``status_code`` pins ``error.code`` to the real response - status, matching v1's SERVER-span behavior.""" + status, matching v1's SERVER-span behavior. ``request_data`` scrubs the + request's own text out of the error when its message logging is redacted.""" from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.litellm_core_utils.redact_messages import scrubbed_error_information_for_request - info: Final = StandardLoggingPayloadSetup.get_error_information( + raw_info: Final = StandardLoggingPayloadSetup.get_error_information( original_exception=exception, traceback_str=traceback_str, ) + info: Final = raw_info if request_data is None else scrubbed_error_information_for_request(raw_info, request_data) return SpanError( error_type=info.get("error_class") or info.get("error_code") or None, message=info.get("error_message") or None, @@ -127,6 +132,20 @@ def _span_error_from_exception( ) +def _keeping_scrubbed_text(error: SpanError, attributes: Mapping[str, object]) -> SpanError: + """``error`` with the span's already stamped message and stack trace when the failure hook scrubbed + the request's text out of them, so a restamp from the raw exception does not put it back.""" + message: Final = attributes.get(Error.MESSAGE) + stack_trace: Final = attributes.get(LiteLLMError.STACK_TRACE) + if not any(isinstance(text, str) and REDACTED_BY_LITELLM in text for text in (message, stack_trace)): + return error + return replace( + error, + message=message if isinstance(message, str) else None, + stack_trace=stack_trace if isinstance(stack_trace, str) else None, + ) + + # Any callback whose class belongs to one of these modules is "the OTel # callback" for proxy-global-registration purposes. _OTEL_MODULES: Final = ( @@ -846,16 +865,17 @@ class OpenTelemetryV2(CustomLogger): response status.""" if span is None or not is_recordable_span(span): return - already_stamped: Final = Error.TYPE in (getattr(span, "attributes", None) or ()) + attributes: Final[Mapping[str, object]] = getattr(span, "attributes", None) or {} + already_stamped: Final = Error.TYPE in attributes stamp_error( span, - _span_error_from_exception(exception, status_code=status_code), + _keeping_scrubbed_text(_span_error_from_exception(exception, status_code=status_code), attributes), record_event=not already_stamped, ) async def async_post_call_failure_hook( self, - request_data: dict, + request_data: Mapping[str, object], original_exception: Exception, user_api_key_dict: "UserAPIKeyAuth", traceback_str: "str | None" = None, @@ -875,7 +895,10 @@ class OpenTelemetryV2(CustomLogger): span: Final = mcp_message_transport_span() or request_root_span() or user_api_key_dict.parent_otel_span if span is None or not is_recordable_span(span): return - stamp_error(span, _span_error_from_exception(original_exception, traceback_str=traceback_str)) + stamp_error( + span, + _span_error_from_exception(original_exception, traceback_str=traceback_str, request_data=request_data), + ) return def emit_guardrail_span(self, entry: "StandardLoggingGuardrailInformation") -> None: diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 34c98efc22f..ed37d373a7f 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -124,10 +124,16 @@ from litellm.litellm_core_utils.logging_utils import ( from litellm.litellm_core_utils.model_param_helper import ModelParamHelper from litellm.litellm_core_utils.ptu_pricing import is_spilled_over_ptu_request from litellm.litellm_core_utils.redact_messages import ( + failure_details_for_custom_logger, + failure_request_texts, redact_message_input_output_from_custom_logger, redact_message_input_output_from_logging, redacted_model_call_details_for_custom_logger, redacted_response_obj_for_custom_logger, + scrub_request_text, + scrubbed_error_fields, + scrubbed_exception_text, + should_redact_failure_logging, should_redact_message_logging, ) from litellm.litellm_core_utils.served_output_texts import ( @@ -4272,8 +4278,9 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["log_event_type"] = "failed_api_call" self.model_call_details["exception"] = exception + traceback_text: Final = redact_string(traceback_exception) if isinstance(traceback_exception, str) else None self.model_call_details["traceback_exception"] = ( - redact_string(traceback_exception) if isinstance(traceback_exception, str) else traceback_exception + traceback_exception if traceback_text is None else traceback_text ) self.model_call_details["end_time"] = end_time self.model_call_details.setdefault("original_response", None) @@ -4291,9 +4298,15 @@ class Logging(LiteLLMLoggingBaseClass): metadata: Final = self.model_call_details["litellm_params"].get("metadata", {}) or {} metadata.update(exception.headers) + # Building the payload redacts the messages in place, so read the request text first + redact: Final = should_redact_failure_logging(self.model_call_details) + request_texts: Final[frozenset[str]] = ( + failure_request_texts(self.model_call_details, getattr(self, "messages", None)) if redact else frozenset() + ) + ## STANDARDIZED LOGGING PAYLOAD - self.model_call_details["standard_logging_object"] = get_standard_logging_object_payload( + payload: Final = get_standard_logging_object_payload( kwargs=self.model_call_details, init_response_obj={}, start_time=start_time, @@ -4304,6 +4317,11 @@ class Logging(LiteLLMLoggingBaseClass): original_exception=exception, standard_built_in_tools_params=self.standard_built_in_tools_params, ) + self.model_call_details["standard_logging_object"] = ( + {**payload, **scrubbed_error_fields(payload, request_texts)} if redact and payload is not None else payload + ) + if redact and traceback_text is not None: + self.model_call_details["traceback_exception"] = scrub_request_text(traceback_text, request_texts) return start_time, end_time async def special_failure_handlers(self, exception: Exception): @@ -4399,6 +4417,9 @@ class Logging(LiteLLMLoggingBaseClass): ) ) self.mark_logging_complete(event_type="sync_failure") + scrubbed_exception: Final = scrubbed_exception_text(self.model_call_details) + logged_error: Final = str(exception) if scrubbed_exception is None else scrubbed_exception[0] + logged_traceback: Final = traceback_exception if scrubbed_exception is None else scrubbed_exception[1] for callback in callbacks: try: should_run = self.should_run_callback( @@ -4424,7 +4445,7 @@ class Logging(LiteLLMLoggingBaseClass): user_id=self._model_call_details_view.get("user", "default"), model=model, input=input, - error=traceback_exception, + error=logged_traceback, run_id=self.litellm_call_id, start_time=start_time, end_time=end_time, @@ -4471,7 +4492,7 @@ class Logging(LiteLLMLoggingBaseClass): start_time=start_time, end_time=end_time, response_obj=result, - kwargs=self.model_call_details, + kwargs=failure_details_for_custom_logger(self.model_call_details, callback), ) if callback == "langfuse": global langFuseLogger @@ -4491,7 +4512,7 @@ class Logging(LiteLLMLoggingBaseClass): end_time=end_time, response_obj=None, user_id=kwargs.get("user", None), - status_message=str(exception), + status_message=logged_error, level="ERROR", kwargs=kwargs, ) @@ -4510,7 +4531,7 @@ class Logging(LiteLLMLoggingBaseClass): response_obj=None, user_id=self.model_call_details.get("user", None), print_verbose=print_verbose, - status_message=str(exception), + status_message=logged_error, level="ERROR", kwargs=self.model_call_details, ) @@ -4606,7 +4627,7 @@ class Logging(LiteLLMLoggingBaseClass): continue if isinstance(callback, CustomLogger): # custom logger class await callback.async_log_failure_event( - kwargs=self.model_call_details, + kwargs=failure_details_for_custom_logger(self.model_call_details, callback), response_obj=result, start_time=start_time, end_time=end_time, diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index 548d088d833..340bcf32c3b 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -10,19 +10,26 @@ import asyncio import copy import inspect -from collections.abc import Mapping, Sequence +import itertools +import json +import re +from collections.abc import Collection, Mapping, Sequence from dataclasses import replace +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final -from pydantic import JsonValue, TypeAdapter, ValidationError +from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError import litellm -from litellm.constants import REDACTED_BY_LITELLM +from litellm.constants import EMPTY_MAPPING, REDACTED_BY_LITELLM from litellm.integrations.custom_logger import CustomLogger -from litellm.litellm_core_utils.classifier_logging import without_classifier_audit +from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS, without_classifier_audit from litellm.litellm_core_utils.core_helpers import ( get_metadata_variable_name_from_kwargs, ) +from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + initialize_standard_callback_dynamic_params, +) from litellm.litellm_core_utils.served_output_texts import SERVED_OUTPUT_TEXTS_KEY from litellm.llms.base_llm.guardrail_translation.utils import stream_item_field, stream_item_items from litellm.llms.vertex_ai.common_utils import ( @@ -31,7 +38,7 @@ from litellm.llms.vertex_ai.common_utils import ( ) from litellm.secret_managers.main import str_to_bool from litellm.types.router import BaselineRouteStamp -from litellm.types.utils import StandardCallbackDynamicParams +from litellm.types.utils import StandardCallbackDynamicParams, StandardLoggingPayloadErrorInformation if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import ( @@ -57,6 +64,80 @@ def redact_message_input_output_from_custom_logger( return result +def failure_details_for_custom_logger( + model_call_details: dict[str, object], # mutable-ok: the kwargs dict a CustomLogger failure hook receives + custom_logger: CustomLogger, +) -> dict[str, object]: # mutable-ok: the kwargs dict a CustomLogger failure hook receives + """ + The per-callback message redaction the success path applies, for a failure event, with the + request's text also scrubbed out of the error. The excluded fields are left out, since several + callbacks index the payload's fields on failure. + """ + redacted: Final = redacted_standard_logging_details(model_call_details, custom_logger, excluded_fields=None) + if redacted is model_call_details and not _opted_out_of_message_logging(custom_logger): + return model_call_details + texts: Final = failure_request_texts(model_call_details) + payload: Final = _as_mapping(redacted.get("standard_logging_object")) + traceback_exception: Final = redacted.get("traceback_exception") + return redacted_model_call_details_for_custom_logger( + { + **redacted, + **( + {"standard_logging_object": {**payload, **scrubbed_error_fields(payload, texts)}} + if payload is not None + else {} + ), + **( + {"traceback_exception": scrub_request_text(traceback_exception, texts)} + if isinstance(traceback_exception, str) + else {} + ), + }, + custom_logger, + ) + + +def redacted_standard_logging_details( + model_call_details: dict[str, object], # mutable-ok: the kwargs dict a CustomLogger hook receives + custom_logger: CustomLogger, + *, + excluded_fields: Sequence[str] | None, +) -> dict[str, object]: # mutable-ok: the kwargs dict a CustomLogger hook receives + """``CustomLogger.redact_standard_logging_payload_from_model_call_details`` with the excluded fields passed in.""" + turn_off_message_logging: Final[bool] = getattr(custom_logger, "turn_off_message_logging", False) + + # Early return if no processing needed + if turn_off_message_logging is False and not excluded_fields: + return model_call_details + + params: Final = _as_mapping(model_call_details.get("litellm_params")) + redacted_params: Final = ( + MappingProxyType({"litellm_params": redacted_litellm_params(params)}) + if turn_off_message_logging and params is not None + else EMPTY_MAPPING + ) + standard_logging_object: Final = _as_mapping(model_call_details.get("standard_logging_object")) + if standard_logging_object is None: + return {**model_call_details, **redacted_params} + + # Make a copy of just the standard_logging_object to avoid modifying the original + standard_logging_object_copy: Final = { + key: value + for key, value in standard_logging_object.items() + if key not in (excluded_fields or ()) and not (turn_off_message_logging and key in CLASSIFIER_AUDIT_FIELDS) + } + + return { + **model_call_details, + **redacted_params, + "standard_logging_object": ( + redacted_standard_logging_payload(standard_logging_object_copy) + if turn_off_message_logging and not custom_logger.redacts_messages_itself() + else standard_logging_object_copy + ), + } + + def redacted_model_call_details_for_custom_logger(model_call_details: dict, custom_logger: CustomLogger) -> dict: """ Returns a copy of model_call_details with the request messages, instructions and streaming responses @@ -222,6 +303,10 @@ def _as_mapping(value: object) -> Mapping[str, object] | None: return value if isinstance(value, Mapping) else None # pyright: ignore[reportUnknownVariableType] # keys are read one by one +def _as_items(value: object) -> tuple[object, ...] | None: + return tuple(value) if isinstance(value, (list, tuple)) else None # pyright: ignore[reportUnknownVariableType, reportUnknownArgumentType] # items are walked one by one + + def _redact_provider_specific_fields(message: object) -> None: if (fields := _as_mapping(getattr(message, "provider_specific_fields", None))) is not None: setattr(message, "provider_specific_fields", _redacted_provider_specific_fields(fields)) # noqa: B010 # typed as object @@ -360,6 +445,126 @@ def _redacted_responses_api_response( return _with_redacted_instructions(redacted) if redact_instructions else redacted +_REQUEST_TEXT_KEYS: Final = ("messages", "input", "prompt", "system", "instructions") +_STRUCTURAL_KEYS: Final = frozenset({"role", "type", "id", "tool_call_id", "name"}) +_MIN_SCRUBBED_TEXT_LENGTH: Final = 4 +_ECHO_FRAGMENT_LENGTH: Final = 24 +_MAX_SCRUB_COMPARISONS: Final = 10_000_000 + + +def request_texts(*values: object) -> frozenset[str]: + """ + The strings a request carries under ``values``, skipping structural fields such as ``role`` and + ``type`` and strings too short to point at the prompt. + """ + texts: Final[set[str]] = set() # mutable-ok: filled by the walk below + pending: Final[list[object]] = list(values) # mutable-ok: the walk's work stack + while pending: + value = pending.pop() + if isinstance(value, str): + if len(value) >= _MIN_SCRUBBED_TEXT_LENGTH: + texts.add(value) + elif (mapping := _as_mapping(value)) is not None: + pending.extend(item for key, item in mapping.items() if key not in _STRUCTURAL_KEYS) + elif (items := _as_items(value)) is not None: + pending.extend(items) + elif isinstance(value, BaseModel): + pending.append(value.model_dump()) + return frozenset(texts) + + +def failure_request_texts(model_call_details: Mapping[str, object], *originals: object) -> frozenset[str]: + """The request text a failure on this call can quote: its messages, input, prompt, system prompt and instructions.""" + optional_params: Final = _as_mapping(model_call_details.get("optional_params")) or {} + return request_texts( + *originals, + *(model_call_details.get(key) for key in _REQUEST_TEXT_KEYS), + *(optional_params.get(key) for key in ("system", "instructions")), + ) + + +def _echo_forms(text: str) -> tuple[str, ...]: + """How an error can quote ``text``: verbatim or JSON or repr escaped, whole or cut down to its head and tail.""" + whole: Final = {text, json.dumps(text)[1:-1], json.dumps(text, ensure_ascii=False)[1:-1], repr(text)[1:-1]} + long_forms: Final = tuple(form for form in whole if len(form) >= 2 * _ECHO_FRAGMENT_LENGTH) + return ( + *whole, + *(form[:_ECHO_FRAGMENT_LENGTH] for form in long_forms), + *(form[-_ECHO_FRAGMENT_LENGTH:] for form in long_forms), + ) + + +def scrub_request_text(text: str, texts: Collection[str]) -> str: + """ + ``text`` with every quote of the request's own ``texts`` replaced by ``REDACTED_BY_LITELLM``, so an + error that does not echo the prompt keeps its wording. A search too large for a failure path + replaces the whole text. + """ + if not text or not texts: + return text + if len(text) * len(texts) > _MAX_SCRUB_COMPARISONS: + return REDACTED_BY_LITELLM + forms: Final = frozenset(itertools.chain.from_iterable(map(_echo_forms, texts))) + quoted: Final = sorted((form for form in forms if form in text), key=len, reverse=True) + if not quoted: + return text + return re.sub("|".join(map(re.escape, quoted)), REDACTED_BY_LITELLM, text) + + +def scrubbed_error_information( + error_information: StandardLoggingPayloadErrorInformation, texts: Collection[str] +) -> StandardLoggingPayloadErrorInformation: + """``error_information`` with the request's ``texts`` scrubbed out of its message and traceback. The type, + status code and every other field carry no prompt and are left as they are.""" + scrubbed: Final = error_information.copy() + error_message: Final = scrubbed.get("error_message") + if isinstance(error_message, str): + scrubbed["error_message"] = scrub_request_text(error_message, texts) + traceback_text: Final = scrubbed.get("traceback") + if isinstance(traceback_text, str): + scrubbed["traceback"] = scrub_request_text(traceback_text, texts) + return scrubbed + + +def _with_scrubbed_text( + mapping: Mapping[str, object], fields: tuple[str, ...], texts: Collection[str] +) -> Mapping[str, object]: + return { + key: scrub_request_text(value, texts) if key in fields and isinstance(value, str) else value + for key, value in mapping.items() + } + + +def scrubbed_error_fields(payload: Mapping[str, object], texts: Collection[str]) -> Mapping[str, object]: + """The ``error_str`` and ``error_information`` entries of a ``StandardLoggingPayload`` with the request's + ``texts`` scrubbed out, to merge over a copy of the payload. Absent entries are left out.""" + error_str: Final = payload.get("error_str") + error_information: Final = _as_mapping(payload.get("error_information")) + return { + **({"error_str": scrub_request_text(error_str, texts)} if isinstance(error_str, str) else {}), + **( + {"error_information": _with_scrubbed_text(error_information, ("error_message", "traceback"), texts)} + if error_information is not None + else {} + ), + } + + +def scrubbed_exception_text(model_call_details: Mapping[str, object]) -> tuple[str, str] | None: + """ + The error message and traceback a failure payload logged when redaction scrubbed the request's text + out of them, for callbacks that would otherwise record the raw exception. ``None`` when nothing was + scrubbed. + """ + payload: Final = _as_mapping(model_call_details.get("standard_logging_object")) or {} + error_information: Final = _as_mapping(payload.get("error_information")) or {} + message: Final = error_information.get("error_message") + if not isinstance(message, str) or REDACTED_BY_LITELLM not in message: + return None + traceback_text: Final = error_information.get("traceback") + return message, traceback_text if isinstance(traceback_text, str) else "" + + def redacted_standard_logging_payload(payload: Mapping[str, object]) -> Mapping[str, object]: """ Return a copy of a ``StandardLoggingPayload`` with its messages and response redacted. @@ -643,6 +848,47 @@ def should_redact_message_logging(model_call_details: dict) -> bool: return litellm.turn_off_message_logging is True +def should_redact_failure_logging(model_call_details: Mapping[str, object]) -> bool: + """ + ``should_redact_message_logging`` for the failure path, which must log the failure whatever the + request carried. Headers that are not a mapping make the payload build fail on the success and + failure paths alike, so the failure is logged unredacted, as before. + """ + try: + return should_redact_message_logging(dict(model_call_details)) + except AttributeError: + return False + + +def should_redact_failed_request(request_data: Mapping[str, object]) -> bool: + """ + The redaction decision for proxy failure hooks, which get the request body but not its logging + object: the same metadata, headers and dynamic params the logging object would read. Callback + params the logging object rejects leave the failure unredacted, as before. + """ + body: Final = dict(request_data) + try: + dynamic_params: Final = initialize_standard_callback_dynamic_params(body) + except ValueError: + return False + return should_redact_failure_logging( + { + "litellm_params": {key: body[key] for key in ("metadata", "litellm_metadata") if key in body}, + "standard_callback_dynamic_params": dynamic_params, + } + ) + + +def scrubbed_error_information_for_request( + error_information: StandardLoggingPayloadErrorInformation, request_data: Mapping[str, object] +) -> StandardLoggingPayloadErrorInformation: + """``error_information`` built from a proxy failure hook's exception, with the request's own text + scrubbed out when the request's message logging is redacted.""" + if not should_redact_failed_request(request_data): + return error_information + return scrubbed_error_information(error_information, failure_request_texts(request_data)) + + def redact_message_input_output_from_logging(model_call_details: dict, result, input: object | None = None) -> Any: """ Removes messages, prompts, input, response from logging. This modifies the data in-place diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 7d8b2e8329c..73eaa299530 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -22,7 +22,10 @@ from litellm.litellm_core_utils.core_helpers import ( ) from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import guardrail_information_cost -from litellm.litellm_core_utils.redact_messages import MESSAGE_REDACTION_ENABLE_HEADERS +from litellm.litellm_core_utils.redact_messages import ( + MESSAGE_REDACTION_ENABLE_HEADERS, + scrubbed_error_information_for_request, +) from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.auth_checks import ( @@ -427,9 +430,13 @@ class _ProxyDBLogger(CustomLogger): ) _metadata["user_api_key"] = LiteLLMProxyRequestSetup.get_logged_api_key(user_api_key_dict) _metadata["status"] = "failure" - _error_information = StandardLoggingPayloadSetup.get_error_information( - original_exception=original_exception, - traceback_str=traceback_str, + request_fields: Final = _str_keyed_fields(request_data) + _error_information = scrubbed_error_information_for_request( + StandardLoggingPayloadSetup.get_error_information( + original_exception=original_exception, + traceback_str=traceback_str, + ), + request_fields, ) if should_suppress_spend_log_tracebacks(): # Drop the traceback key entirely so the per-row Metadata pane in @@ -451,7 +458,6 @@ class _ProxyDBLogger(CustomLogger): metadata=_metadata, original_exception=original_exception ) - request_fields: Final = _str_keyed_fields(request_data) _metadata["used_client_oauth_token"] = _proxy_stamped_used_client_oauth_token(request_fields, request_route) existing_metadata: Final[dict] = request_data.get("metadata", None) or {} diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 9e125b95a68..d03bc297cb8 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -162,6 +162,7 @@ from litellm.litellm_core_utils.duration_parser import get_budget_window_start from litellm.litellm_core_utils.env_utils import get_env_float, get_env_int from litellm.litellm_core_utils.litellm_logging import BilledRoundsDispatch, Logging, deployment_pricing_model_info from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.redact_messages import should_redact_failed_request from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.served_output_texts import ( @@ -3972,6 +3973,9 @@ class ProxyLogging: ), ) request_data["litellm_logging_obj"] = litellm_logging_obj # rebind-ok: lifted then popped by the caller + if should_redact_failed_request(request_fields): + # The allowlist leaves out the request's redaction opt-in, so carry the decision over + litellm_logging_obj.standard_callback_dynamic_params["turn_off_message_logging"] = True request_data.update({REJECTED_BEFORE_REQUEST_SETUP_KEY: REJECTED_BEFORE_REQUEST_SETUP}) if "metadata" not in request_data: request_data["metadata"] = {} diff --git a/tests/unit/integrations/datadog/test_datadog_logger_batching.py b/tests/unit/integrations/datadog/test_datadog_logger_batching.py index e2707d321bf..512d30d7fa6 100644 --- a/tests/unit/integrations/datadog/test_datadog_logger_batching.py +++ b/tests/unit/integrations/datadog/test_datadog_logger_batching.py @@ -1,4 +1,5 @@ import asyncio +import json from unittest.mock import AsyncMock, Mock, patch import httpx @@ -6,6 +7,7 @@ import pytest from httpx import Request, Response from pydantic import BaseModel, computed_field +import litellm from litellm.integrations.datadog.datadog import DataDogLogger from litellm.llms.custom_httpx.http_handler import MaskedHTTPStatusError from litellm.types.integrations.datadog import ( @@ -558,3 +560,26 @@ async def test_raised_intake_error_preserves_datadog_requeue_behavior(datadog_en await logger.async_send_batch() assert [event["message"] for event in logger.log_queue] == ['{"event": 0}', '{"event": 1}'] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("redact", [True, False], ids=["redaction_on", "redaction_off"]) +async def test_failure_hook_scrubs_the_prompt_out_of_the_logged_exception(datadog_env, monkeypatch, redact): + secret = "my ssn is 123-45-6789" + monkeypatch.setattr(litellm, "turn_off_message_logging", redact) + with patch("asyncio.create_task"): + logger = DataDogLogger() + + await logger.async_post_call_failure_hook( + request_data={"messages": [{"role": "user", "content": secret}], "metadata": {}}, + original_exception=litellm.BadRequestError( + message=f"Unsupported content: {secret}", model="gpt-4o", llm_provider="openai" + ), + user_api_key_dict=type("UserKey", (), {})(), + traceback_str=f"Traceback: {secret}", + ) + + message = json.loads(logger.log_queue[0]["message"]) + assert (secret in json.dumps(message)) is not redact + assert "Unsupported content: " in message["exception"] + assert message["error_class"] == "BadRequestError" diff --git a/tests/unit/integrations/otel/test_otel_v2_logger.py b/tests/unit/integrations/otel/test_otel_v2_logger.py index 1f9d3af71d9..b2fd9e8638c 100644 --- a/tests/unit/integrations/otel/test_otel_v2_logger.py +++ b/tests/unit/integrations/otel/test_otel_v2_logger.py @@ -1548,6 +1548,37 @@ def test_record_error_attributes_on_span_decorates_without_ending(): assert [e.name for e in span.events] == ["exception"] +@pytest.mark.parametrize("redact", [True, False], ids=["redaction_on", "redaction_off"]) +def test_failure_hook_scrubs_the_prompt_out_of_the_server_span_and_the_restamp_keeps_it(monkeypatch, redact): + import json + + import litellm + from litellm.proxy._types import UserAPIKeyAuth + + secret = "my ssn is 123-45-6789" + monkeypatch.setattr(litellm, "turn_off_message_logging", redact) + logger, exporter = _logger() + server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME) + set_request_root_span(server) + exc = _proxy_exc(f"blocked: {secret}", 400) + asyncio.run( + logger.async_post_call_failure_hook( + request_data={"messages": [{"role": "user", "content": secret}]}, + original_exception=exc, + user_api_key_dict=UserAPIKeyAuth(), + ) + ) + logger.record_error_attributes_on_span(server, exc, 400) + server.end() + (span,) = exporter.get_finished_spans() + logged = json.dumps( + [dict(span.attributes), [dict(event.attributes or {}) for event in span.events], span.status.description] + ) + assert (secret in logged) is not redact + assert span.attributes["error.message"].startswith("blocked: ") + assert span.attributes["error.type"] == "ProxyException" + + def test_record_error_attributes_on_span_does_not_duplicate_an_already_stamped_error(): """A failure that already went through ``async_post_call_failure_hook`` reaches the exception handler too; the second stamp must keep one exception event while diff --git a/tests/unit/integrations/test_mlflow.py b/tests/unit/integrations/test_mlflow.py index 058d0ae885d..73fdd7e568b 100644 --- a/tests/unit/integrations/test_mlflow.py +++ b/tests/unit/integrations/test_mlflow.py @@ -268,3 +268,54 @@ def test_mlflow_end_span_or_trace_works_with_mlflow_2x_client(): span=child_span, outputs="out", end_time_ns=1, status="OK" ) assert client.ended_spans == [("req-2", "span-2")] + + +class _ExceptionSpanEvent: + def __init__(self, name, attributes): + self.name = name + self.attributes = attributes + + @classmethod + def from_exception(cls, exception): + return cls( + "exception", + { + "exception.message": str(exception), + "exception.type": type(exception).__name__, + "exception.stacktrace": "raw", + }, + ) + + +@pytest.mark.parametrize("scrubbed", [True, False], ids=["scrubbed", "not_scrubbed"]) +def test_mlflow_failure_event_records_the_error_text_the_payload_logged(scrubbed): + secret = "my ssn is 123-45-6789" + modules = _mock_mlflow_modules() + modules["mlflow.entities"].SpanEvent = _ExceptionSpanEvent + with patch.dict("sys.modules", modules): + from litellm.integrations.mlflow import MlflowLogger + + mlflow_logger = MlflowLogger() + span = MagicMock() + mlflow_logger._start_span_or_trace = MagicMock(return_value=span) + mlflow_logger._end_span_or_trace = MagicMock() + mlflow_logger._extract_and_set_chat_attributes = MagicMock() + error_message = "bad input: redacted-by-litellm" if scrubbed else f"bad input: {secret}" + + mlflow_logger._handle_failure( + kwargs={ + "litellm_call_id": "x", + "exception": ValueError(f"bad input: {secret}"), + "standard_logging_object": { + "error_information": {"error_message": error_message, "traceback": "frames"} + }, + }, + response_obj=None, + start_time=datetime.now(), + end_time=datetime.now(), + ) + + event = span.add_event.call_args.args[0] + assert event.attributes["exception.type"] == "ValueError" + assert event.attributes["exception.message"] == error_message + assert event.attributes["exception.stacktrace"] == ("frames" if scrubbed else "raw") diff --git a/tests/unit/integrations/test_opentelemetry.py b/tests/unit/integrations/test_opentelemetry.py index f0fca94c370..3380b83f000 100644 --- a/tests/unit/integrations/test_opentelemetry.py +++ b/tests/unit/integrations/test_opentelemetry.py @@ -7631,3 +7631,39 @@ class TestOpentelemetryUnitTests: # Assert: error.message should be set from error_str using ErrorAttributes constant mock_span.set_attribute.assert_called_with(ErrorAttributes.ERROR_MESSAGE, "Fallback error message") + + +def test_record_exception_on_span_records_the_scrubbed_error_text_with_the_same_event_shape(): + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + + from litellm.integrations.opentelemetry import OpenTelemetry + + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + secret = "my ssn is 123-45-6789" + exception = ValueError(f"bad input: {secret}") + + with provider.get_tracer(__name__).start_as_current_span("failure") as span: + OpenTelemetry()._record_exception_on_span( + span=span, + kwargs={ + "exception": exception, + "standard_logging_object": { + "error_information": { + "error_class": "ValueError", + "error_message": "bad input: redacted-by-litellm", + "traceback": "frames", + }, + }, + }, + ) + + (event,) = exporter.get_finished_spans()[0].events + assert secret not in str(dict(event.attributes)) + assert event.attributes["exception.message"] == "bad input: redacted-by-litellm" + assert event.attributes["exception.stacktrace"] == "frames" + assert event.attributes["exception.type"] == "ValueError" + assert event.attributes["exception.escaped"] == "False" diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index a6d99a5ba55..b4378e12ec1 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -11648,6 +11648,163 @@ def test_uncached_price_keeps_an_earlier_price_failure(pr44187_priced_model: str assert logging_obj.model_call_details["response_cost_failure_debug_information"] == earlier_failure +SCRUB_PROMPT: Final = "my ssn is 123-45-6789" + + +class _FailureCapture(CustomLogger): + def __init__(self, **kwargs: bool) -> None: + super().__init__(**kwargs) + self.sync_kwargs: dict | None = None + self.async_kwargs: dict | None = None + + def log_failure_event(self, kwargs, response_obj, start_time, end_time): + self.sync_kwargs = kwargs + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + self.async_kwargs = kwargs + + +class _SelfRedactingFailureCapture(_FailureCapture): + def redacts_messages_itself(self) -> bool: + return True + + +def _prompt_failure_logging_obj(litellm_params: Mapping[str, object] | None = None) -> LitellmLogging: + logging_obj: Final = LitellmLogging( + model="gpt-4o", + messages=[{"role": "user", "content": SCRUB_PROMPT}], + stream=False, + call_type="completion", + start_time=time.time(), + litellm_call_id="failure-scrub", + function_id="failure-scrub", + ) + logging_obj.model_call_details["messages"] = logging_obj.messages # as update_environment_variables sets it + if litellm_params is not None: + logging_obj.model_call_details["litellm_params"] = dict(litellm_params) + return logging_obj + + +def _prompt_echo_error() -> litellm.BadRequestError: + return litellm.BadRequestError( + message=f"Invalid message passed in {{'role': 'user', 'content': '{SCRUB_PROMPT}'}}", + model="gpt-4o", + llm_provider="openai", + ) + + +async def _async_failure_payload( + monkeypatch: pytest.MonkeyPatch, *, redact: bool, error: Exception, capture: _FailureCapture | None = None +) -> dict: + logger: Final = capture or _FailureCapture() + monkeypatch.setattr(litellm, "turn_off_message_logging", redact) + monkeypatch.setattr(litellm, "_async_failure_callback", [logger]) + await _prompt_failure_logging_obj().async_failure_handler(error, f"Traceback: {error}") + assert logger.async_kwargs is not None + return logger.async_kwargs + + +@pytest.mark.asyncio +async def test_failure_with_redaction_on_scrubs_the_prompt_out_of_the_error_and_keeps_the_rest(monkeypatch): + kwargs: Final = await _async_failure_payload(monkeypatch, redact=True, error=_prompt_echo_error()) + + payload: Final = kwargs["standard_logging_object"] + assert SCRUB_PROMPT not in json.dumps(payload, default=str) + assert SCRUB_PROMPT not in kwargs["traceback_exception"] + assert "Invalid message passed in" in payload["error_str"] + assert "'content': 'redacted-by-litellm'" in payload["error_information"]["error_message"] + assert payload["error_information"]["error_class"] == "BadRequestError" + assert payload["error_information"]["error_code"] == "400" + + +@pytest.mark.asyncio +async def test_failure_with_redaction_on_keeps_error_text_that_does_not_quote_the_prompt(monkeypatch): + error: Final = litellm.AuthenticationError( + message="Incorrect API key provided", model="gpt-4o", llm_provider="openai" + ) + + redacted: Final = (await _async_failure_payload(monkeypatch, redact=True, error=error))["standard_logging_object"] + raw: Final = (await _async_failure_payload(monkeypatch, redact=False, error=error))["standard_logging_object"] + + assert redacted["error_str"] == raw["error_str"] + assert redacted["error_information"] == raw["error_information"] + + +@pytest.mark.asyncio +async def test_failure_with_redaction_off_logs_the_error_as_raised(monkeypatch): + kwargs: Final = await _async_failure_payload(monkeypatch, redact=False, error=_prompt_echo_error()) + + assert SCRUB_PROMPT in kwargs["standard_logging_object"]["error_information"]["error_message"] + assert SCRUB_PROMPT in kwargs["traceback_exception"] + + +@pytest.mark.asyncio +async def test_failure_redacts_messages_and_error_for_a_callback_that_opted_out_of_message_logging(monkeypatch): + opted_out: Final = _FailureCapture(message_logging=False) + logging_everything: Final = _FailureCapture() + monkeypatch.setattr(litellm, "turn_off_message_logging", False) + monkeypatch.setattr(litellm, "_async_failure_callback", [opted_out, logging_everything]) + + await _prompt_failure_logging_obj().async_failure_handler(_prompt_echo_error(), "Traceback") + + assert opted_out.async_kwargs is not None and logging_everything.async_kwargs is not None + assert SCRUB_PROMPT not in json.dumps(opted_out.async_kwargs["standard_logging_object"], default=str) + assert opted_out.async_kwargs["messages"] == [{"role": "user", "content": "redacted-by-litellm"}] + assert SCRUB_PROMPT in logging_everything.async_kwargs["standard_logging_object"]["error_str"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("capture_type", [_FailureCapture, _SelfRedactingFailureCapture]) +async def test_failure_scrubs_the_error_for_a_callback_with_its_own_turn_off_message_logging(monkeypatch, capture_type): + capture: Final = capture_type(turn_off_message_logging=True) + + kwargs: Final = await _async_failure_payload(monkeypatch, redact=False, error=_prompt_echo_error(), capture=capture) + + assert SCRUB_PROMPT not in kwargs["standard_logging_object"]["error_str"] + assert SCRUB_PROMPT not in kwargs["standard_logging_object"]["error_information"]["error_message"] + + +@pytest.mark.asyncio +async def test_failure_payload_keeps_the_excluded_fields(monkeypatch): + monkeypatch.setattr(litellm, "standard_logging_payload_excluded_fields", ["metadata"]) + + kwargs: Final = await _async_failure_payload(monkeypatch, redact=False, error=_prompt_echo_error()) + + assert "metadata" in kwargs["standard_logging_object"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("headers", [None, "x", ["a"]], ids=["none", "str", "list"]) +@pytest.mark.parametrize("redact", [True, False], ids=["redaction_on", "redaction_off"]) +async def test_failure_with_non_mapping_headers_is_still_logged(monkeypatch, headers, redact): + capture: Final = _FailureCapture() + monkeypatch.setattr(litellm, "turn_off_message_logging", redact) + monkeypatch.setattr(litellm, "_async_failure_callback", [capture]) + logging_obj: Final = _prompt_failure_logging_obj({"metadata": {"headers": headers}}) + + await logging_obj.async_failure_handler(_prompt_echo_error(), "Traceback") + + assert capture.async_kwargs is not None + + +def test_sync_failure_hands_callbacks_and_langfuse_the_scrubbed_error(monkeypatch): + capture: Final = _FailureCapture() + langfuse: Final = MagicMock() + monkeypatch.setattr(litellm, "turn_off_message_logging", True) + monkeypatch.setattr(litellm, "failure_callback", [capture, "langfuse"]) + monkeypatch.setattr( + litellm_logging.LangFuseHandler, "get_langfuse_logger_for_request", MagicMock(return_value=langfuse) + ) + + _prompt_failure_logging_obj().failure_handler(_prompt_echo_error(), f"Traceback: {SCRUB_PROMPT}") + + assert capture.sync_kwargs is not None + assert SCRUB_PROMPT not in json.dumps(capture.sync_kwargs["standard_logging_object"], default=str) + status_message: Final = langfuse.log_event_on_langfuse.call_args.kwargs["status_message"] + assert "Invalid message passed in" in status_message + assert SCRUB_PROMPT not in status_message + + def test_enterprise_alerting_loggers_resolves_real_classes(): from litellm_enterprise.enterprise_callbacks.pagerduty.pagerduty import PagerDutyAlerting from litellm_enterprise.enterprise_callbacks.send_emails.resend_email import ResendEmailLogger diff --git a/tests/unit/litellm_core_utils/test_redact_messages.py b/tests/unit/litellm_core_utils/test_redact_messages.py index 5b64e42d8a8..8eedd755e60 100644 --- a/tests/unit/litellm_core_utils/test_redact_messages.py +++ b/tests/unit/litellm_core_utils/test_redact_messages.py @@ -33,6 +33,10 @@ from litellm.litellm_core_utils.redact_messages import ( redacted_model_call_details_for_custom_logger, redacted_response_obj_for_custom_logger, redacted_standard_logging_payload, + request_texts, + scrub_request_text, + scrubbed_error_information_for_request, + should_redact_failed_request, should_redact_message_logging, ) from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler @@ -43,6 +47,7 @@ from litellm.types.utils import ( ModelResponseStream, ResponsesAPIResponse, StandardLoggingPayload, + StandardLoggingPayloadErrorInformation, TextCompletionResponse, ) from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome @@ -2075,3 +2080,86 @@ def test_perform_redaction_strips_the_matched_text_from_the_bedrock_guardrail_tr assert "US_SSN" in redacted.model_dump_json(), "the finding stays, only its matched text goes" assert "123-45-6789" not in str(redacted_dict) assert "123-45-6789" in response.model_dump_json(), "the caller's own response is not redacted" + + +PROMPT: Final = "my ssn is 123-45-6789\nand my pin is 4321, keep both safe" + + +@pytest.mark.parametrize( + "quoted", + [PROMPT, json.dumps(PROMPT)[1:-1], repr(PROMPT)[1:-1], PROMPT[:24] + "...", "..." + PROMPT[-24:]], + ids=["verbatim", "json", "repr", "head", "tail"], +) +def test_scrub_request_text_replaces_each_way_an_error_quotes_the_prompt(quoted: str) -> None: + scrubbed: Final = scrub_request_text(f"Invalid content: <{quoted}> rejected", request_texts(PROMPT)) + + assert "123-45-6789" not in scrubbed and "4321" not in scrubbed + assert "redacted-by-litellm" in scrubbed + assert scrubbed.startswith("Invalid content: <") and scrubbed.endswith("> rejected") + + +def test_scrub_request_text_leaves_an_error_that_does_not_quote_the_prompt() -> None: + assert scrub_request_text("Incorrect API key provided", request_texts(PROMPT, "hi")) == "Incorrect API key provided" + + +def test_scrub_request_text_replaces_the_whole_text_when_the_search_is_too_large() -> None: + texts: Final = request_texts([f"message number {index}" for index in range(5_000)]) + + assert scrub_request_text("x" * 5_000, texts) == "redacted-by-litellm" + + +def test_request_texts_reads_content_and_skips_structural_fields() -> None: + texts: Final = request_texts( + [ + {"role": "assistant", "content": [{"type": "text", "text": "part one text"}]}, + {"role": "tool", "tool_call_id": "call_123456", "name": "lookup", "content": "tool output"}, + litellm.Message(role="assistant", content="from a model object"), + ] + ) + + assert texts == {"part one text", "tool output", "from a model object"} + + +@pytest.mark.parametrize( + "request_data", + [ + {"turn_off_message_logging": True}, + {"metadata": {"turn_off_message_logging": True}}, + {"metadata": {"headers": {"x-litellm-enable-message-redaction": "true"}}}, + ], + ids=["body_flag", "metadata_flag", "enable_header"], +) +def test_should_redact_failed_request_follows_the_request_opt_ins(request_data: dict) -> None: + assert should_redact_failed_request(request_data) is True + assert should_redact_failed_request({}) is False + + +@pytest.mark.parametrize( + "request_data", + [{"metadata": {"headers": "x"}}, {"turn_off_message_logging": "os.environ/REDACT"}], + ids=["non_mapping_headers", "env_reference"], +) +def test_should_redact_failed_request_leaves_an_undecidable_request_unredacted( + monkeypatch: pytest.MonkeyPatch, request_data: dict +) -> None: + monkeypatch.setattr(litellm, "turn_off_message_logging", True) + + assert should_redact_failed_request(request_data) is False + + +def test_scrubbed_error_information_for_request_scrubs_only_when_the_request_redacts() -> None: + error_information: Final = StandardLoggingPayloadErrorInformation( + error_code="400", error_class="BadRequestError", error_message=f"bad: {PROMPT}", traceback=f"tb {PROMPT}" + ) + messages: Final = [{"role": "user", "content": PROMPT}] + + scrubbed: Final = scrubbed_error_information_for_request( + error_information, {"messages": messages, "turn_off_message_logging": True} + ) + + assert scrubbed == { + **error_information, + "error_message": "bad: redacted-by-litellm", + "traceback": "tb redacted-by-litellm", + } + assert scrubbed_error_information_for_request(error_information, {"messages": messages}) == error_information diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py index dc46b5faf4e..b578a3e96b9 100644 --- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py @@ -3674,3 +3674,30 @@ async def test_async_post_call_failure_hook_disable_pre_routing_error_logs( user_api_key_dict=user_api_key_dict, ) assert len(writer.rows) == rows_written + + +@pytest.mark.asyncio +@pytest.mark.parametrize("redact", [True, False], ids=["redaction_on", "redaction_off"]) +async def test_async_post_call_failure_hook_scrubs_the_prompt_out_of_the_persisted_error( + monkeypatch: pytest.MonkeyPatch, redact: bool +): + secret: Final = "my ssn is 123-45-6789" + monkeypatch.setattr(litellm, "turn_off_message_logging", redact) + spend_writer: Final = create_autospec(DBSpendUpdateWriter, instance=True) + + await ProxyDBLogger(spend_writer=lambda: spend_writer).async_post_call_failure_hook( + request_data={"model": "gpt-4o", "messages": [{"role": "user", "content": secret}], "metadata": {}}, + original_exception=litellm.BadRequestError( + message=f"Unsupported content: {secret}", model="gpt-4o", llm_provider="openai" + ), + user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key"), + traceback_str=f"Traceback: {secret}", + ) + + persisted: Final = spend_writer.update_database.call_args.kwargs["kwargs"]["litellm_params"]["metadata"][ + "error_information" + ] + assert (secret in json.dumps(persisted)) is not redact + assert "Unsupported content: " in persisted["error_message"] + assert persisted["error_class"] == "BadRequestError" + assert persisted["error_code"] == "400" diff --git a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py index 3cdd2edb2fb..2b317a9dc97 100644 --- a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py +++ b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py @@ -202,6 +202,36 @@ async def test_proxy_only_error_log_keeps_tags_and_drops_caller_logging_fields( assert details["litellm_params"][metadata_field]["user_api_key_team_id"] == "team-a" +@pytest.mark.asyncio +@pytest.mark.parametrize( + "opt_in", + [ + {"turn_off_message_logging": True}, + {"metadata": {"turn_off_message_logging": True}}, + {"metadata": {"headers": {"x-litellm-enable-message-redaction": "true"}}}, + ], + ids=["body_flag", "metadata_flag", "enable_header"], +) +async def test_proxy_only_error_log_keeps_the_request_redaction_opt_in(opt_in: Mapping[str, object]) -> None: + secret: Final = "my ssn is 123-45-6789" + request_data: Final[dict[str, object]] = { + "model": "gpt-4o", + "messages": [{"role": "user", "content": secret}], + **opt_in, + } + + await ProxyLogging(user_api_key_cache=DualCache())._handle_logging_proxy_only_error( + request_data=request_data, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-rejected", team_id="team-a", request_route="/v1/rerank"), + route="/v1/rerank", + original_exception=HTTPException(status_code=400, detail=f"blocked: {secret}"), + ) + + payload: Final = request_data["litellm_logging_obj"].model_call_details["standard_logging_object"] + assert secret not in json.dumps(payload, default=str) + assert "blocked: redacted-by-litellm" in payload["error_str"] + + @pytest.mark.asyncio @pytest.mark.parametrize("metadata_field", ["metadata", "litellm_metadata"]) async def test_proxy_only_error_log_drops_malformed_metadata_tags(metadata_field: str) -> None: