diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 0579eb15baf..9330e7c16f7 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -3,13 +3,12 @@ import re import traceback from collections.abc import AsyncGenerator, Mapping, Sequence -from types import MappingProxyType from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional from pydantic import BaseModel from litellm._logging import verbose_logger -from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER, EMPTY_MAPPING +from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER from litellm.types.integrations.argilla import ArgillaItem from litellm.types.integrations.custom_logger import AgenticLoopPlan from litellm.types.llms.openai import AllMessageValues, ChatCompletionRequest @@ -912,46 +911,12 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac This is useful for logging payloads that contain sensitive information. """ import litellm - from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS - from litellm.litellm_core_utils.redact_messages import ( - redacted_litellm_params, - redacted_standard_logging_payload, + from litellm.litellm_core_utils.redact_messages import redacted_standard_logging_details + + return redacted_standard_logging_details( + model_call_details, self, excluded_fields=getattr(litellm, "standard_logging_payload_excluded_fields", None) ) - turn_off_message_logging: Final[bool] = getattr(self, "turn_off_message_logging", False) - excluded_fields: Final[list[str] | None] = getattr(litellm, "standard_logging_payload_excluded_fields", None) - - # Early return if no processing needed - if turn_off_message_logging is False and not excluded_fields: - return model_call_details - - params: Final = model_call_details.get("litellm_params") - redacted_params: Final = ( - MappingProxyType({"litellm_params": redacted_litellm_params(params)}) - if turn_off_message_logging and isinstance(params, Mapping) - else EMPTY_MAPPING - ) - standard_logging_object: Final = model_call_details.get("standard_logging_object") - if standard_logging_object is None: - return {**model_call_details, **redacted_params} - - # Make a copy of just the standard_logging_object to avoid modifying the original - standard_logging_object_copy: Final = { - key: value - for key, value in standard_logging_object.items() - if key not in (excluded_fields or ()) and not (turn_off_message_logging and key in CLASSIFIER_AUDIT_FIELDS) - } - - return { - **model_call_details, - **redacted_params, - "standard_logging_object": ( - redacted_standard_logging_payload(standard_logging_object_copy) - if turn_off_message_logging and not self.redacts_messages_itself() - else standard_logging_object_copy - ), - } - async def get_proxy_server_request_from_cold_storage_with_object_key( self, object_key: str, diff --git a/litellm/integrations/datadog/datadog.py b/litellm/integrations/datadog/datadog.py index 6518cda5ae3..3c569ff4fe9 100644 --- a/litellm/integrations/datadog/datadog.py +++ b/litellm/integrations/datadog/datadog.py @@ -18,7 +18,7 @@ import datetime import os import time import traceback -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from datetime import datetime as datetimeObj from typing import TYPE_CHECKING, Any, Final @@ -287,7 +287,7 @@ class DataDogLogger( async def async_post_call_failure_hook( self, - request_data: dict, + request_data: Mapping[str, object], original_exception: Exception, user_api_key_dict: "UserAPIKeyAuth", traceback_str: str | None = None, @@ -303,11 +303,15 @@ class DataDogLogger( from litellm.litellm_core_utils.litellm_logging import ( StandardLoggingPayloadSetup, ) + from litellm.litellm_core_utils.redact_messages import scrubbed_error_information_for_request from litellm.litellm_core_utils.safe_json_dumps import safe_dumps - error_information: Final = StandardLoggingPayloadSetup.get_error_information( - original_exception=original_exception, - traceback_str=traceback_str, + error_information: Final = scrubbed_error_information_for_request( + StandardLoggingPayloadSetup.get_error_information( + original_exception=original_exception, + traceback_str=traceback_str, + ), + request_data, ) _code: Final = error_information.get("error_code") or "" status_code: int | None = None diff --git a/litellm/integrations/mlflow.py b/litellm/integrations/mlflow.py index 8731e96440f..740ac36716a 100644 --- a/litellm/integrations/mlflow.py +++ b/litellm/integrations/mlflow.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.redact_messages import scrubbed_exception_text if TYPE_CHECKING: from litellm.types.utils import StandardLoggingPayload @@ -86,9 +87,21 @@ class MlflowLogger(CustomLogger): end_time_ns: Final = int(end_time.timestamp() * 1e9) - # Record exception info as event + # Record exception info as event, with the logged text when redaction scrubbed it if exception := kwargs.get("exception"): - span.add_event(SpanEvent.from_exception(exception)) + scrubbed: Final = scrubbed_exception_text(kwargs) + span.add_event( + SpanEvent.from_exception(exception) + if scrubbed is None + else SpanEvent( + name="exception", + attributes={ + "exception.message": scrubbed[0], + "exception.type": exception.__class__.__name__, + "exception.stacktrace": scrubbed[1], + }, + ) + ) self._extract_and_set_chat_attributes(span, kwargs, response_obj) self._end_span_or_trace( diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 0d0e309034a..e5d936f8dce 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -36,6 +36,7 @@ from litellm.integrations.otel.model.semconv import LiteLLM, Metric from litellm.integrations.otel.plumbing.otlp_tls import resolve_otlp_http_tls from litellm.integrations.otel.routing import routing_decision_attributes from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params +from litellm.litellm_core_utils.redact_messages import scrubbed_exception_text from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.secret_redaction import redact_string from litellm.litellm_core_utils.service_tier_utils import ( @@ -2271,9 +2272,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): # Get the exception object if available exception: Final = kwargs.get("exception") + scrubbed: Final = scrubbed_exception_text(kwargs) - # Record the exception using OTEL's standard method - if exception is not None: + # Record the exception using OTEL's standard method, with the logged text when redaction scrubbed it + if exception is not None and scrubbed is not None: + span.record_exception( + exception, attributes={"exception.message": scrubbed[0], "exception.stacktrace": scrubbed[1]} + ) + elif exception is not None: span.record_exception(exception) # Get StandardLoggingPayload for structured error information diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index b5b65d299e0..6297abaf4ac 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -27,6 +27,7 @@ from typing_extensions import TypedDict, Unpack import litellm from litellm._logging import verbose_logger +from litellm.constants import REDACTED_BY_LITELLM from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.otel.emitter import SpanEmitter, stamp_error from litellm.integrations.otel.mappers import resolve_mappers @@ -49,7 +50,7 @@ from litellm.integrations.otel.model.payloads import ( is_mcp_list_tools, is_mcp_tool_call, ) -from litellm.integrations.otel.model.semconv import Error +from litellm.integrations.otel.model.semconv import Error, LiteLLMError from litellm.integrations.otel.model.spans import SpanRole, span_role_for_service from litellm.integrations.otel.model.utils import to_ns from litellm.integrations.otel.plumbing.context import ( @@ -106,18 +107,22 @@ def _span_error_from_exception( *, status_code: int | None = None, traceback_str: str | None = None, + request_data: Mapping[str, object] | None = None, ) -> SpanError: """A ``SpanError`` for a proxy-level failure that never produced a ``StandardLoggingPayload`` (auth / validation / malformed-body rejections), mirroring ``_parse_error``'s field mapping so it stamps the same v2 keys a failed LLM call does. ``status_code`` pins ``error.code`` to the real response - status, matching v1's SERVER-span behavior.""" + status, matching v1's SERVER-span behavior. ``request_data`` scrubs the + request's own text out of the error when its message logging is redacted.""" from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.litellm_core_utils.redact_messages import scrubbed_error_information_for_request - info: Final = StandardLoggingPayloadSetup.get_error_information( + raw_info: Final = StandardLoggingPayloadSetup.get_error_information( original_exception=exception, traceback_str=traceback_str, ) + info: Final = raw_info if request_data is None else scrubbed_error_information_for_request(raw_info, request_data) return SpanError( error_type=info.get("error_class") or info.get("error_code") or None, message=info.get("error_message") or None, @@ -127,6 +132,20 @@ def _span_error_from_exception( ) +def _keeping_scrubbed_text(error: SpanError, attributes: Mapping[str, object]) -> SpanError: + """``error`` with the span's already stamped message and stack trace when the failure hook scrubbed + the request's text out of them, so a restamp from the raw exception does not put it back.""" + message: Final = attributes.get(Error.MESSAGE) + stack_trace: Final = attributes.get(LiteLLMError.STACK_TRACE) + if not any(isinstance(text, str) and REDACTED_BY_LITELLM in text for text in (message, stack_trace)): + return error + return replace( + error, + message=message if isinstance(message, str) else None, + stack_trace=stack_trace if isinstance(stack_trace, str) else None, + ) + + # Any callback whose class belongs to one of these modules is "the OTel # callback" for proxy-global-registration purposes. _OTEL_MODULES: Final = ( @@ -846,16 +865,17 @@ class OpenTelemetryV2(CustomLogger): response status.""" if span is None or not is_recordable_span(span): return - already_stamped: Final = Error.TYPE in (getattr(span, "attributes", None) or ()) + attributes: Final[Mapping[str, object]] = getattr(span, "attributes", None) or {} + already_stamped: Final = Error.TYPE in attributes stamp_error( span, - _span_error_from_exception(exception, status_code=status_code), + _keeping_scrubbed_text(_span_error_from_exception(exception, status_code=status_code), attributes), record_event=not already_stamped, ) async def async_post_call_failure_hook( self, - request_data: dict, + request_data: Mapping[str, object], original_exception: Exception, user_api_key_dict: "UserAPIKeyAuth", traceback_str: "str | None" = None, @@ -875,7 +895,10 @@ class OpenTelemetryV2(CustomLogger): span: Final = mcp_message_transport_span() or request_root_span() or user_api_key_dict.parent_otel_span if span is None or not is_recordable_span(span): return - stamp_error(span, _span_error_from_exception(original_exception, traceback_str=traceback_str)) + stamp_error( + span, + _span_error_from_exception(original_exception, traceback_str=traceback_str, request_data=request_data), + ) return def emit_guardrail_span(self, entry: "StandardLoggingGuardrailInformation") -> None: diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 34c98efc22f..ed37d373a7f 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -124,10 +124,16 @@ from litellm.litellm_core_utils.logging_utils import ( from litellm.litellm_core_utils.model_param_helper import ModelParamHelper from litellm.litellm_core_utils.ptu_pricing import is_spilled_over_ptu_request from litellm.litellm_core_utils.redact_messages import ( + failure_details_for_custom_logger, + failure_request_texts, redact_message_input_output_from_custom_logger, redact_message_input_output_from_logging, redacted_model_call_details_for_custom_logger, redacted_response_obj_for_custom_logger, + scrub_request_text, + scrubbed_error_fields, + scrubbed_exception_text, + should_redact_failure_logging, should_redact_message_logging, ) from litellm.litellm_core_utils.served_output_texts import ( @@ -4272,8 +4278,9 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["log_event_type"] = "failed_api_call" self.model_call_details["exception"] = exception + traceback_text: Final = redact_string(traceback_exception) if isinstance(traceback_exception, str) else None self.model_call_details["traceback_exception"] = ( - redact_string(traceback_exception) if isinstance(traceback_exception, str) else traceback_exception + traceback_exception if traceback_text is None else traceback_text ) self.model_call_details["end_time"] = end_time self.model_call_details.setdefault("original_response", None) @@ -4291,9 +4298,15 @@ class Logging(LiteLLMLoggingBaseClass): metadata: Final = self.model_call_details["litellm_params"].get("metadata", {}) or {} metadata.update(exception.headers) + # Building the payload redacts the messages in place, so read the request text first + redact: Final = should_redact_failure_logging(self.model_call_details) + request_texts: Final[frozenset[str]] = ( + failure_request_texts(self.model_call_details, getattr(self, "messages", None)) if redact else frozenset() + ) + ## STANDARDIZED LOGGING PAYLOAD - self.model_call_details["standard_logging_object"] = get_standard_logging_object_payload( + payload: Final = get_standard_logging_object_payload( kwargs=self.model_call_details, init_response_obj={}, start_time=start_time, @@ -4304,6 +4317,11 @@ class Logging(LiteLLMLoggingBaseClass): original_exception=exception, standard_built_in_tools_params=self.standard_built_in_tools_params, ) + self.model_call_details["standard_logging_object"] = ( + {**payload, **scrubbed_error_fields(payload, request_texts)} if redact and payload is not None else payload + ) + if redact and traceback_text is not None: + self.model_call_details["traceback_exception"] = scrub_request_text(traceback_text, request_texts) return start_time, end_time async def special_failure_handlers(self, exception: Exception): @@ -4399,6 +4417,9 @@ class Logging(LiteLLMLoggingBaseClass): ) ) self.mark_logging_complete(event_type="sync_failure") + scrubbed_exception: Final = scrubbed_exception_text(self.model_call_details) + logged_error: Final = str(exception) if scrubbed_exception is None else scrubbed_exception[0] + logged_traceback: Final = traceback_exception if scrubbed_exception is None else scrubbed_exception[1] for callback in callbacks: try: should_run = self.should_run_callback( @@ -4424,7 +4445,7 @@ class Logging(LiteLLMLoggingBaseClass): user_id=self._model_call_details_view.get("user", "default"), model=model, input=input, - error=traceback_exception, + error=logged_traceback, run_id=self.litellm_call_id, start_time=start_time, end_time=end_time, @@ -4471,7 +4492,7 @@ class Logging(LiteLLMLoggingBaseClass): start_time=start_time, end_time=end_time, response_obj=result, - kwargs=self.model_call_details, + kwargs=failure_details_for_custom_logger(self.model_call_details, callback), ) if callback == "langfuse": global langFuseLogger @@ -4491,7 +4512,7 @@ class Logging(LiteLLMLoggingBaseClass): end_time=end_time, response_obj=None, user_id=kwargs.get("user", None), - status_message=str(exception), + status_message=logged_error, level="ERROR", kwargs=kwargs, ) @@ -4510,7 +4531,7 @@ class Logging(LiteLLMLoggingBaseClass): response_obj=None, user_id=self.model_call_details.get("user", None), print_verbose=print_verbose, - status_message=str(exception), + status_message=logged_error, level="ERROR", kwargs=self.model_call_details, ) @@ -4606,7 +4627,7 @@ class Logging(LiteLLMLoggingBaseClass): continue if isinstance(callback, CustomLogger): # custom logger class await callback.async_log_failure_event( - kwargs=self.model_call_details, + kwargs=failure_details_for_custom_logger(self.model_call_details, callback), response_obj=result, start_time=start_time, end_time=end_time, diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index 548d088d833..340bcf32c3b 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -10,19 +10,26 @@ import asyncio import copy import inspect -from collections.abc import Mapping, Sequence +import itertools +import json +import re +from collections.abc import Collection, Mapping, Sequence from dataclasses import replace +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final -from pydantic import JsonValue, TypeAdapter, ValidationError +from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError import litellm -from litellm.constants import REDACTED_BY_LITELLM +from litellm.constants import EMPTY_MAPPING, REDACTED_BY_LITELLM from litellm.integrations.custom_logger import CustomLogger -from litellm.litellm_core_utils.classifier_logging import without_classifier_audit +from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS, without_classifier_audit from litellm.litellm_core_utils.core_helpers import ( get_metadata_variable_name_from_kwargs, ) +from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + initialize_standard_callback_dynamic_params, +) from litellm.litellm_core_utils.served_output_texts import SERVED_OUTPUT_TEXTS_KEY from litellm.llms.base_llm.guardrail_translation.utils import stream_item_field, stream_item_items from litellm.llms.vertex_ai.common_utils import ( @@ -31,7 +38,7 @@ from litellm.llms.vertex_ai.common_utils import ( ) from litellm.secret_managers.main import str_to_bool from litellm.types.router import BaselineRouteStamp -from litellm.types.utils import StandardCallbackDynamicParams +from litellm.types.utils import StandardCallbackDynamicParams, StandardLoggingPayloadErrorInformation if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import ( @@ -57,6 +64,80 @@ def redact_message_input_output_from_custom_logger( return result +def failure_details_for_custom_logger( + model_call_details: dict[str, object], # mutable-ok: the kwargs dict a CustomLogger failure hook receives + custom_logger: CustomLogger, +) -> dict[str, object]: # mutable-ok: the kwargs dict a CustomLogger failure hook receives + """ + The per-callback message redaction the success path applies, for a failure event, with the + request's text also scrubbed out of the error. The excluded fields are left out, since several + callbacks index the payload's fields on failure. + """ + redacted: Final = redacted_standard_logging_details(model_call_details, custom_logger, excluded_fields=None) + if redacted is model_call_details and not _opted_out_of_message_logging(custom_logger): + return model_call_details + texts: Final = failure_request_texts(model_call_details) + payload: Final = _as_mapping(redacted.get("standard_logging_object")) + traceback_exception: Final = redacted.get("traceback_exception") + return redacted_model_call_details_for_custom_logger( + { + **redacted, + **( + {"standard_logging_object": {**payload, **scrubbed_error_fields(payload, texts)}} + if payload is not None + else {} + ), + **( + {"traceback_exception": scrub_request_text(traceback_exception, texts)} + if isinstance(traceback_exception, str) + else {} + ), + }, + custom_logger, + ) + + +def redacted_standard_logging_details( + model_call_details: dict[str, object], # mutable-ok: the kwargs dict a CustomLogger hook receives + custom_logger: CustomLogger, + *, + excluded_fields: Sequence[str] | None, +) -> dict[str, object]: # mutable-ok: the kwargs dict a CustomLogger hook receives + """``CustomLogger.redact_standard_logging_payload_from_model_call_details`` with the excluded fields passed in.""" + turn_off_message_logging: Final[bool] = getattr(custom_logger, "turn_off_message_logging", False) + + # Early return if no processing needed + if turn_off_message_logging is False and not excluded_fields: + return model_call_details + + params: Final = _as_mapping(model_call_details.get("litellm_params")) + redacted_params: Final = ( + MappingProxyType({"litellm_params": redacted_litellm_params(params)}) + if turn_off_message_logging and params is not None + else EMPTY_MAPPING + ) + standard_logging_object: Final = _as_mapping(model_call_details.get("standard_logging_object")) + if standard_logging_object is None: + return {**model_call_details, **redacted_params} + + # Make a copy of just the standard_logging_object to avoid modifying the original + standard_logging_object_copy: Final = { + key: value + for key, value in standard_logging_object.items() + if key not in (excluded_fields or ()) and not (turn_off_message_logging and key in CLASSIFIER_AUDIT_FIELDS) + } + + return { + **model_call_details, + **redacted_params, + "standard_logging_object": ( + redacted_standard_logging_payload(standard_logging_object_copy) + if turn_off_message_logging and not custom_logger.redacts_messages_itself() + else standard_logging_object_copy + ), + } + + def redacted_model_call_details_for_custom_logger(model_call_details: dict, custom_logger: CustomLogger) -> dict: """ Returns a copy of model_call_details with the request messages, instructions and streaming responses @@ -222,6 +303,10 @@ def _as_mapping(value: object) -> Mapping[str, object] | None: return value if isinstance(value, Mapping) else None # pyright: ignore[reportUnknownVariableType] # keys are read one by one +def _as_items(value: object) -> tuple[object, ...] | None: + return tuple(value) if isinstance(value, (list, tuple)) else None # pyright: ignore[reportUnknownVariableType, reportUnknownArgumentType] # items are walked one by one + + def _redact_provider_specific_fields(message: object) -> None: if (fields := _as_mapping(getattr(message, "provider_specific_fields", None))) is not None: setattr(message, "provider_specific_fields", _redacted_provider_specific_fields(fields)) # noqa: B010 # typed as object @@ -360,6 +445,126 @@ def _redacted_responses_api_response( return _with_redacted_instructions(redacted) if redact_instructions else redacted +_REQUEST_TEXT_KEYS: Final = ("messages", "input", "prompt", "system", "instructions") +_STRUCTURAL_KEYS: Final = frozenset({"role", "type", "id", "tool_call_id", "name"}) +_MIN_SCRUBBED_TEXT_LENGTH: Final = 4 +_ECHO_FRAGMENT_LENGTH: Final = 24 +_MAX_SCRUB_COMPARISONS: Final = 10_000_000 + + +def request_texts(*values: object) -> frozenset[str]: + """ + The strings a request carries under ``values``, skipping structural fields such as ``role`` and + ``type`` and strings too short to point at the prompt. + """ + texts: Final[set[str]] = set() # mutable-ok: filled by the walk below + pending: Final[list[object]] = list(values) # mutable-ok: the walk's work stack + while pending: + value = pending.pop() + if isinstance(value, str): + if len(value) >= _MIN_SCRUBBED_TEXT_LENGTH: + texts.add(value) + elif (mapping := _as_mapping(value)) is not None: + pending.extend(item for key, item in mapping.items() if key not in _STRUCTURAL_KEYS) + elif (items := _as_items(value)) is not None: + pending.extend(items) + elif isinstance(value, BaseModel): + pending.append(value.model_dump()) + return frozenset(texts) + + +def failure_request_texts(model_call_details: Mapping[str, object], *originals: object) -> frozenset[str]: + """The request text a failure on this call can quote: its messages, input, prompt, system prompt and instructions.""" + optional_params: Final = _as_mapping(model_call_details.get("optional_params")) or {} + return request_texts( + *originals, + *(model_call_details.get(key) for key in _REQUEST_TEXT_KEYS), + *(optional_params.get(key) for key in ("system", "instructions")), + ) + + +def _echo_forms(text: str) -> tuple[str, ...]: + """How an error can quote ``text``: verbatim or JSON or repr escaped, whole or cut down to its head and tail.""" + whole: Final = {text, json.dumps(text)[1:-1], json.dumps(text, ensure_ascii=False)[1:-1], repr(text)[1:-1]} + long_forms: Final = tuple(form for form in whole if len(form) >= 2 * _ECHO_FRAGMENT_LENGTH) + return ( + *whole, + *(form[:_ECHO_FRAGMENT_LENGTH] for form in long_forms), + *(form[-_ECHO_FRAGMENT_LENGTH:] for form in long_forms), + ) + + +def scrub_request_text(text: str, texts: Collection[str]) -> str: + """ + ``text`` with every quote of the request's own ``texts`` replaced by ``REDACTED_BY_LITELLM``, so an + error that does not echo the prompt keeps its wording. A search too large for a failure path + replaces the whole text. + """ + if not text or not texts: + return text + if len(text) * len(texts) > _MAX_SCRUB_COMPARISONS: + return REDACTED_BY_LITELLM + forms: Final = frozenset(itertools.chain.from_iterable(map(_echo_forms, texts))) + quoted: Final = sorted((form for form in forms if form in text), key=len, reverse=True) + if not quoted: + return text + return re.sub("|".join(map(re.escape, quoted)), REDACTED_BY_LITELLM, text) + + +def scrubbed_error_information( + error_information: StandardLoggingPayloadErrorInformation, texts: Collection[str] +) -> StandardLoggingPayloadErrorInformation: + """``error_information`` with the request's ``texts`` scrubbed out of its message and traceback. The type, + status code and every other field carry no prompt and are left as they are.""" + scrubbed: Final = error_information.copy() + error_message: Final = scrubbed.get("error_message") + if isinstance(error_message, str): + scrubbed["error_message"] = scrub_request_text(error_message, texts) + traceback_text: Final = scrubbed.get("traceback") + if isinstance(traceback_text, str): + scrubbed["traceback"] = scrub_request_text(traceback_text, texts) + return scrubbed + + +def _with_scrubbed_text( + mapping: Mapping[str, object], fields: tuple[str, ...], texts: Collection[str] +) -> Mapping[str, object]: + return { + key: scrub_request_text(value, texts) if key in fields and isinstance(value, str) else value + for key, value in mapping.items() + } + + +def scrubbed_error_fields(payload: Mapping[str, object], texts: Collection[str]) -> Mapping[str, object]: + """The ``error_str`` and ``error_information`` entries of a ``StandardLoggingPayload`` with the request's + ``texts`` scrubbed out, to merge over a copy of the payload. Absent entries are left out.""" + error_str: Final = payload.get("error_str") + error_information: Final = _as_mapping(payload.get("error_information")) + return { + **({"error_str": scrub_request_text(error_str, texts)} if isinstance(error_str, str) else {}), + **( + {"error_information": _with_scrubbed_text(error_information, ("error_message", "traceback"), texts)} + if error_information is not None + else {} + ), + } + + +def scrubbed_exception_text(model_call_details: Mapping[str, object]) -> tuple[str, str] | None: + """ + The error message and traceback a failure payload logged when redaction scrubbed the request's text + out of them, for callbacks that would otherwise record the raw exception. ``None`` when nothing was + scrubbed. + """ + payload: Final = _as_mapping(model_call_details.get("standard_logging_object")) or {} + error_information: Final = _as_mapping(payload.get("error_information")) or {} + message: Final = error_information.get("error_message") + if not isinstance(message, str) or REDACTED_BY_LITELLM not in message: + return None + traceback_text: Final = error_information.get("traceback") + return message, traceback_text if isinstance(traceback_text, str) else "" + + def redacted_standard_logging_payload(payload: Mapping[str, object]) -> Mapping[str, object]: """ Return a copy of a ``StandardLoggingPayload`` with its messages and response redacted. @@ -643,6 +848,47 @@ def should_redact_message_logging(model_call_details: dict) -> bool: return litellm.turn_off_message_logging is True +def should_redact_failure_logging(model_call_details: Mapping[str, object]) -> bool: + """ + ``should_redact_message_logging`` for the failure path, which must log the failure whatever the + request carried. Headers that are not a mapping make the payload build fail on the success and + failure paths alike, so the failure is logged unredacted, as before. + """ + try: + return should_redact_message_logging(dict(model_call_details)) + except AttributeError: + return False + + +def should_redact_failed_request(request_data: Mapping[str, object]) -> bool: + """ + The redaction decision for proxy failure hooks, which get the request body but not its logging + object: the same metadata, headers and dynamic params the logging object would read. Callback + params the logging object rejects leave the failure unredacted, as before. + """ + body: Final = dict(request_data) + try: + dynamic_params: Final = initialize_standard_callback_dynamic_params(body) + except ValueError: + return False + return should_redact_failure_logging( + { + "litellm_params": {key: body[key] for key in ("metadata", "litellm_metadata") if key in body}, + "standard_callback_dynamic_params": dynamic_params, + } + ) + + +def scrubbed_error_information_for_request( + error_information: StandardLoggingPayloadErrorInformation, request_data: Mapping[str, object] +) -> StandardLoggingPayloadErrorInformation: + """``error_information`` built from a proxy failure hook's exception, with the request's own text + scrubbed out when the request's message logging is redacted.""" + if not should_redact_failed_request(request_data): + return error_information + return scrubbed_error_information(error_information, failure_request_texts(request_data)) + + def redact_message_input_output_from_logging(model_call_details: dict, result, input: object | None = None) -> Any: """ Removes messages, prompts, input, response from logging. This modifies the data in-place diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 7d8b2e8329c..73eaa299530 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -22,7 +22,10 @@ from litellm.litellm_core_utils.core_helpers import ( ) from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import guardrail_information_cost -from litellm.litellm_core_utils.redact_messages import MESSAGE_REDACTION_ENABLE_HEADERS +from litellm.litellm_core_utils.redact_messages import ( + MESSAGE_REDACTION_ENABLE_HEADERS, + scrubbed_error_information_for_request, +) from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.auth_checks import ( @@ -427,9 +430,13 @@ class _ProxyDBLogger(CustomLogger): ) _metadata["user_api_key"] = LiteLLMProxyRequestSetup.get_logged_api_key(user_api_key_dict) _metadata["status"] = "failure" - _error_information = StandardLoggingPayloadSetup.get_error_information( - original_exception=original_exception, - traceback_str=traceback_str, + request_fields: Final = _str_keyed_fields(request_data) + _error_information = scrubbed_error_information_for_request( + StandardLoggingPayloadSetup.get_error_information( + original_exception=original_exception, + traceback_str=traceback_str, + ), + request_fields, ) if should_suppress_spend_log_tracebacks(): # Drop the traceback key entirely so the per-row Metadata pane in @@ -451,7 +458,6 @@ class _ProxyDBLogger(CustomLogger): metadata=_metadata, original_exception=original_exception ) - request_fields: Final = _str_keyed_fields(request_data) _metadata["used_client_oauth_token"] = _proxy_stamped_used_client_oauth_token(request_fields, request_route) existing_metadata: Final[dict] = request_data.get("metadata", None) or {} diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 9e125b95a68..d03bc297cb8 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -162,6 +162,7 @@ from litellm.litellm_core_utils.duration_parser import get_budget_window_start from litellm.litellm_core_utils.env_utils import get_env_float, get_env_int from litellm.litellm_core_utils.litellm_logging import BilledRoundsDispatch, Logging, deployment_pricing_model_info from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.redact_messages import should_redact_failed_request from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.served_output_texts import ( @@ -3972,6 +3973,9 @@ class ProxyLogging: ), ) request_data["litellm_logging_obj"] = litellm_logging_obj # rebind-ok: lifted then popped by the caller + if should_redact_failed_request(request_fields): + # The allowlist leaves out the request's redaction opt-in, so carry the decision over + litellm_logging_obj.standard_callback_dynamic_params["turn_off_message_logging"] = True request_data.update({REJECTED_BEFORE_REQUEST_SETUP_KEY: REJECTED_BEFORE_REQUEST_SETUP}) if "metadata" not in request_data: request_data["metadata"] = {} diff --git a/tests/unit/integrations/datadog/test_datadog_logger_batching.py b/tests/unit/integrations/datadog/test_datadog_logger_batching.py index e2707d321bf..512d30d7fa6 100644 --- a/tests/unit/integrations/datadog/test_datadog_logger_batching.py +++ b/tests/unit/integrations/datadog/test_datadog_logger_batching.py @@ -1,4 +1,5 @@ import asyncio +import json from unittest.mock import AsyncMock, Mock, patch import httpx @@ -6,6 +7,7 @@ import pytest from httpx import Request, Response from pydantic import BaseModel, computed_field +import litellm from litellm.integrations.datadog.datadog import DataDogLogger from litellm.llms.custom_httpx.http_handler import MaskedHTTPStatusError from litellm.types.integrations.datadog import ( @@ -558,3 +560,26 @@ async def test_raised_intake_error_preserves_datadog_requeue_behavior(datadog_en await logger.async_send_batch() assert [event["message"] for event in logger.log_queue] == ['{"event": 0}', '{"event": 1}'] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("redact", [True, False], ids=["redaction_on", "redaction_off"]) +async def test_failure_hook_scrubs_the_prompt_out_of_the_logged_exception(datadog_env, monkeypatch, redact): + secret = "my ssn is 123-45-6789" + monkeypatch.setattr(litellm, "turn_off_message_logging", redact) + with patch("asyncio.create_task"): + logger = DataDogLogger() + + await logger.async_post_call_failure_hook( + request_data={"messages": [{"role": "user", "content": secret}], "metadata": {}}, + original_exception=litellm.BadRequestError( + message=f"Unsupported content: {secret}", model="gpt-4o", llm_provider="openai" + ), + user_api_key_dict=type("UserKey", (), {})(), + traceback_str=f"Traceback: {secret}", + ) + + message = json.loads(logger.log_queue[0]["message"]) + assert (secret in json.dumps(message)) is not redact + assert "Unsupported content: " in message["exception"] + assert message["error_class"] == "BadRequestError" diff --git a/tests/unit/integrations/otel/test_otel_v2_logger.py b/tests/unit/integrations/otel/test_otel_v2_logger.py index 1f9d3af71d9..b2fd9e8638c 100644 --- a/tests/unit/integrations/otel/test_otel_v2_logger.py +++ b/tests/unit/integrations/otel/test_otel_v2_logger.py @@ -1548,6 +1548,37 @@ def test_record_error_attributes_on_span_decorates_without_ending(): assert [e.name for e in span.events] == ["exception"] +@pytest.mark.parametrize("redact", [True, False], ids=["redaction_on", "redaction_off"]) +def test_failure_hook_scrubs_the_prompt_out_of_the_server_span_and_the_restamp_keeps_it(monkeypatch, redact): + import json + + import litellm + from litellm.proxy._types import UserAPIKeyAuth + + secret = "my ssn is 123-45-6789" + monkeypatch.setattr(litellm, "turn_off_message_logging", redact) + logger, exporter = _logger() + server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME) + set_request_root_span(server) + exc = _proxy_exc(f"blocked: {secret}", 400) + asyncio.run( + logger.async_post_call_failure_hook( + request_data={"messages": [{"role": "user", "content": secret}]}, + original_exception=exc, + user_api_key_dict=UserAPIKeyAuth(), + ) + ) + logger.record_error_attributes_on_span(server, exc, 400) + server.end() + (span,) = exporter.get_finished_spans() + logged = json.dumps( + [dict(span.attributes), [dict(event.attributes or {}) for event in span.events], span.status.description] + ) + assert (secret in logged) is not redact + assert span.attributes["error.message"].startswith("blocked: ") + assert span.attributes["error.type"] == "ProxyException" + + def test_record_error_attributes_on_span_does_not_duplicate_an_already_stamped_error(): """A failure that already went through ``async_post_call_failure_hook`` reaches the exception handler too; the second stamp must keep one exception event while diff --git a/tests/unit/integrations/test_mlflow.py b/tests/unit/integrations/test_mlflow.py index 058d0ae885d..73fdd7e568b 100644 --- a/tests/unit/integrations/test_mlflow.py +++ b/tests/unit/integrations/test_mlflow.py @@ -268,3 +268,54 @@ def test_mlflow_end_span_or_trace_works_with_mlflow_2x_client(): span=child_span, outputs="out", end_time_ns=1, status="OK" ) assert client.ended_spans == [("req-2", "span-2")] + + +class _ExceptionSpanEvent: + def __init__(self, name, attributes): + self.name = name + self.attributes = attributes + + @classmethod + def from_exception(cls, exception): + return cls( + "exception", + { + "exception.message": str(exception), + "exception.type": type(exception).__name__, + "exception.stacktrace": "raw", + }, + ) + + +@pytest.mark.parametrize("scrubbed", [True, False], ids=["scrubbed", "not_scrubbed"]) +def test_mlflow_failure_event_records_the_error_text_the_payload_logged(scrubbed): + secret = "my ssn is 123-45-6789" + modules = _mock_mlflow_modules() + modules["mlflow.entities"].SpanEvent = _ExceptionSpanEvent + with patch.dict("sys.modules", modules): + from litellm.integrations.mlflow import MlflowLogger + + mlflow_logger = MlflowLogger() + span = MagicMock() + mlflow_logger._start_span_or_trace = MagicMock(return_value=span) + mlflow_logger._end_span_or_trace = MagicMock() + mlflow_logger._extract_and_set_chat_attributes = MagicMock() + error_message = "bad input: redacted-by-litellm" if scrubbed else f"bad input: {secret}" + + mlflow_logger._handle_failure( + kwargs={ + "litellm_call_id": "x", + "exception": ValueError(f"bad input: {secret}"), + "standard_logging_object": { + "error_information": {"error_message": error_message, "traceback": "frames"} + }, + }, + response_obj=None, + start_time=datetime.now(), + end_time=datetime.now(), + ) + + event = span.add_event.call_args.args[0] + assert event.attributes["exception.type"] == "ValueError" + assert event.attributes["exception.message"] == error_message + assert event.attributes["exception.stacktrace"] == ("frames" if scrubbed else "raw") diff --git a/tests/unit/integrations/test_opentelemetry.py b/tests/unit/integrations/test_opentelemetry.py index f0fca94c370..3380b83f000 100644 --- a/tests/unit/integrations/test_opentelemetry.py +++ b/tests/unit/integrations/test_opentelemetry.py @@ -7631,3 +7631,39 @@ class TestOpentelemetryUnitTests: # Assert: error.message should be set from error_str using ErrorAttributes constant mock_span.set_attribute.assert_called_with(ErrorAttributes.ERROR_MESSAGE, "Fallback error message") + + +def test_record_exception_on_span_records_the_scrubbed_error_text_with_the_same_event_shape(): + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + + from litellm.integrations.opentelemetry import OpenTelemetry + + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + secret = "my ssn is 123-45-6789" + exception = ValueError(f"bad input: {secret}") + + with provider.get_tracer(__name__).start_as_current_span("failure") as span: + OpenTelemetry()._record_exception_on_span( + span=span, + kwargs={ + "exception": exception, + "standard_logging_object": { + "error_information": { + "error_class": "ValueError", + "error_message": "bad input: redacted-by-litellm", + "traceback": "frames", + }, + }, + }, + ) + + (event,) = exporter.get_finished_spans()[0].events + assert secret not in str(dict(event.attributes)) + assert event.attributes["exception.message"] == "bad input: redacted-by-litellm" + assert event.attributes["exception.stacktrace"] == "frames" + assert event.attributes["exception.type"] == "ValueError" + assert event.attributes["exception.escaped"] == "False" diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index a6d99a5ba55..b4378e12ec1 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -11648,6 +11648,163 @@ def test_uncached_price_keeps_an_earlier_price_failure(pr44187_priced_model: str assert logging_obj.model_call_details["response_cost_failure_debug_information"] == earlier_failure +SCRUB_PROMPT: Final = "my ssn is 123-45-6789" + + +class _FailureCapture(CustomLogger): + def __init__(self, **kwargs: bool) -> None: + super().__init__(**kwargs) + self.sync_kwargs: dict | None = None + self.async_kwargs: dict | None = None + + def log_failure_event(self, kwargs, response_obj, start_time, end_time): + self.sync_kwargs = kwargs + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + self.async_kwargs = kwargs + + +class _SelfRedactingFailureCapture(_FailureCapture): + def redacts_messages_itself(self) -> bool: + return True + + +def _prompt_failure_logging_obj(litellm_params: Mapping[str, object] | None = None) -> LitellmLogging: + logging_obj: Final = LitellmLogging( + model="gpt-4o", + messages=[{"role": "user", "content": SCRUB_PROMPT}], + stream=False, + call_type="completion", + start_time=time.time(), + litellm_call_id="failure-scrub", + function_id="failure-scrub", + ) + logging_obj.model_call_details["messages"] = logging_obj.messages # as update_environment_variables sets it + if litellm_params is not None: + logging_obj.model_call_details["litellm_params"] = dict(litellm_params) + return logging_obj + + +def _prompt_echo_error() -> litellm.BadRequestError: + return litellm.BadRequestError( + message=f"Invalid message passed in {{'role': 'user', 'content': '{SCRUB_PROMPT}'}}", + model="gpt-4o", + llm_provider="openai", + ) + + +async def _async_failure_payload( + monkeypatch: pytest.MonkeyPatch, *, redact: bool, error: Exception, capture: _FailureCapture | None = None +) -> dict: + logger: Final = capture or _FailureCapture() + monkeypatch.setattr(litellm, "turn_off_message_logging", redact) + monkeypatch.setattr(litellm, "_async_failure_callback", [logger]) + await _prompt_failure_logging_obj().async_failure_handler(error, f"Traceback: {error}") + assert logger.async_kwargs is not None + return logger.async_kwargs + + +@pytest.mark.asyncio +async def test_failure_with_redaction_on_scrubs_the_prompt_out_of_the_error_and_keeps_the_rest(monkeypatch): + kwargs: Final = await _async_failure_payload(monkeypatch, redact=True, error=_prompt_echo_error()) + + payload: Final = kwargs["standard_logging_object"] + assert SCRUB_PROMPT not in json.dumps(payload, default=str) + assert SCRUB_PROMPT not in kwargs["traceback_exception"] + assert "Invalid message passed in" in payload["error_str"] + assert "'content': 'redacted-by-litellm'" in payload["error_information"]["error_message"] + assert payload["error_information"]["error_class"] == "BadRequestError" + assert payload["error_information"]["error_code"] == "400" + + +@pytest.mark.asyncio +async def test_failure_with_redaction_on_keeps_error_text_that_does_not_quote_the_prompt(monkeypatch): + error: Final = litellm.AuthenticationError( + message="Incorrect API key provided", model="gpt-4o", llm_provider="openai" + ) + + redacted: Final = (await _async_failure_payload(monkeypatch, redact=True, error=error))["standard_logging_object"] + raw: Final = (await _async_failure_payload(monkeypatch, redact=False, error=error))["standard_logging_object"] + + assert redacted["error_str"] == raw["error_str"] + assert redacted["error_information"] == raw["error_information"] + + +@pytest.mark.asyncio +async def test_failure_with_redaction_off_logs_the_error_as_raised(monkeypatch): + kwargs: Final = await _async_failure_payload(monkeypatch, redact=False, error=_prompt_echo_error()) + + assert SCRUB_PROMPT in kwargs["standard_logging_object"]["error_information"]["error_message"] + assert SCRUB_PROMPT in kwargs["traceback_exception"] + + +@pytest.mark.asyncio +async def test_failure_redacts_messages_and_error_for_a_callback_that_opted_out_of_message_logging(monkeypatch): + opted_out: Final = _FailureCapture(message_logging=False) + logging_everything: Final = _FailureCapture() + monkeypatch.setattr(litellm, "turn_off_message_logging", False) + monkeypatch.setattr(litellm, "_async_failure_callback", [opted_out, logging_everything]) + + await _prompt_failure_logging_obj().async_failure_handler(_prompt_echo_error(), "Traceback") + + assert opted_out.async_kwargs is not None and logging_everything.async_kwargs is not None + assert SCRUB_PROMPT not in json.dumps(opted_out.async_kwargs["standard_logging_object"], default=str) + assert opted_out.async_kwargs["messages"] == [{"role": "user", "content": "redacted-by-litellm"}] + assert SCRUB_PROMPT in logging_everything.async_kwargs["standard_logging_object"]["error_str"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("capture_type", [_FailureCapture, _SelfRedactingFailureCapture]) +async def test_failure_scrubs_the_error_for_a_callback_with_its_own_turn_off_message_logging(monkeypatch, capture_type): + capture: Final = capture_type(turn_off_message_logging=True) + + kwargs: Final = await _async_failure_payload(monkeypatch, redact=False, error=_prompt_echo_error(), capture=capture) + + assert SCRUB_PROMPT not in kwargs["standard_logging_object"]["error_str"] + assert SCRUB_PROMPT not in kwargs["standard_logging_object"]["error_information"]["error_message"] + + +@pytest.mark.asyncio +async def test_failure_payload_keeps_the_excluded_fields(monkeypatch): + monkeypatch.setattr(litellm, "standard_logging_payload_excluded_fields", ["metadata"]) + + kwargs: Final = await _async_failure_payload(monkeypatch, redact=False, error=_prompt_echo_error()) + + assert "metadata" in kwargs["standard_logging_object"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("headers", [None, "x", ["a"]], ids=["none", "str", "list"]) +@pytest.mark.parametrize("redact", [True, False], ids=["redaction_on", "redaction_off"]) +async def test_failure_with_non_mapping_headers_is_still_logged(monkeypatch, headers, redact): + capture: Final = _FailureCapture() + monkeypatch.setattr(litellm, "turn_off_message_logging", redact) + monkeypatch.setattr(litellm, "_async_failure_callback", [capture]) + logging_obj: Final = _prompt_failure_logging_obj({"metadata": {"headers": headers}}) + + await logging_obj.async_failure_handler(_prompt_echo_error(), "Traceback") + + assert capture.async_kwargs is not None + + +def test_sync_failure_hands_callbacks_and_langfuse_the_scrubbed_error(monkeypatch): + capture: Final = _FailureCapture() + langfuse: Final = MagicMock() + monkeypatch.setattr(litellm, "turn_off_message_logging", True) + monkeypatch.setattr(litellm, "failure_callback", [capture, "langfuse"]) + monkeypatch.setattr( + litellm_logging.LangFuseHandler, "get_langfuse_logger_for_request", MagicMock(return_value=langfuse) + ) + + _prompt_failure_logging_obj().failure_handler(_prompt_echo_error(), f"Traceback: {SCRUB_PROMPT}") + + assert capture.sync_kwargs is not None + assert SCRUB_PROMPT not in json.dumps(capture.sync_kwargs["standard_logging_object"], default=str) + status_message: Final = langfuse.log_event_on_langfuse.call_args.kwargs["status_message"] + assert "Invalid message passed in" in status_message + assert SCRUB_PROMPT not in status_message + + def test_enterprise_alerting_loggers_resolves_real_classes(): from litellm_enterprise.enterprise_callbacks.pagerduty.pagerduty import PagerDutyAlerting from litellm_enterprise.enterprise_callbacks.send_emails.resend_email import ResendEmailLogger diff --git a/tests/unit/litellm_core_utils/test_redact_messages.py b/tests/unit/litellm_core_utils/test_redact_messages.py index 5b64e42d8a8..8eedd755e60 100644 --- a/tests/unit/litellm_core_utils/test_redact_messages.py +++ b/tests/unit/litellm_core_utils/test_redact_messages.py @@ -33,6 +33,10 @@ from litellm.litellm_core_utils.redact_messages import ( redacted_model_call_details_for_custom_logger, redacted_response_obj_for_custom_logger, redacted_standard_logging_payload, + request_texts, + scrub_request_text, + scrubbed_error_information_for_request, + should_redact_failed_request, should_redact_message_logging, ) from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler @@ -43,6 +47,7 @@ from litellm.types.utils import ( ModelResponseStream, ResponsesAPIResponse, StandardLoggingPayload, + StandardLoggingPayloadErrorInformation, TextCompletionResponse, ) from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome @@ -2075,3 +2080,86 @@ def test_perform_redaction_strips_the_matched_text_from_the_bedrock_guardrail_tr assert "US_SSN" in redacted.model_dump_json(), "the finding stays, only its matched text goes" assert "123-45-6789" not in str(redacted_dict) assert "123-45-6789" in response.model_dump_json(), "the caller's own response is not redacted" + + +PROMPT: Final = "my ssn is 123-45-6789\nand my pin is 4321, keep both safe" + + +@pytest.mark.parametrize( + "quoted", + [PROMPT, json.dumps(PROMPT)[1:-1], repr(PROMPT)[1:-1], PROMPT[:24] + "...", "..." + PROMPT[-24:]], + ids=["verbatim", "json", "repr", "head", "tail"], +) +def test_scrub_request_text_replaces_each_way_an_error_quotes_the_prompt(quoted: str) -> None: + scrubbed: Final = scrub_request_text(f"Invalid content: <{quoted}> rejected", request_texts(PROMPT)) + + assert "123-45-6789" not in scrubbed and "4321" not in scrubbed + assert "redacted-by-litellm" in scrubbed + assert scrubbed.startswith("Invalid content: <") and scrubbed.endswith("> rejected") + + +def test_scrub_request_text_leaves_an_error_that_does_not_quote_the_prompt() -> None: + assert scrub_request_text("Incorrect API key provided", request_texts(PROMPT, "hi")) == "Incorrect API key provided" + + +def test_scrub_request_text_replaces_the_whole_text_when_the_search_is_too_large() -> None: + texts: Final = request_texts([f"message number {index}" for index in range(5_000)]) + + assert scrub_request_text("x" * 5_000, texts) == "redacted-by-litellm" + + +def test_request_texts_reads_content_and_skips_structural_fields() -> None: + texts: Final = request_texts( + [ + {"role": "assistant", "content": [{"type": "text", "text": "part one text"}]}, + {"role": "tool", "tool_call_id": "call_123456", "name": "lookup", "content": "tool output"}, + litellm.Message(role="assistant", content="from a model object"), + ] + ) + + assert texts == {"part one text", "tool output", "from a model object"} + + +@pytest.mark.parametrize( + "request_data", + [ + {"turn_off_message_logging": True}, + {"metadata": {"turn_off_message_logging": True}}, + {"metadata": {"headers": {"x-litellm-enable-message-redaction": "true"}}}, + ], + ids=["body_flag", "metadata_flag", "enable_header"], +) +def test_should_redact_failed_request_follows_the_request_opt_ins(request_data: dict) -> None: + assert should_redact_failed_request(request_data) is True + assert should_redact_failed_request({}) is False + + +@pytest.mark.parametrize( + "request_data", + [{"metadata": {"headers": "x"}}, {"turn_off_message_logging": "os.environ/REDACT"}], + ids=["non_mapping_headers", "env_reference"], +) +def test_should_redact_failed_request_leaves_an_undecidable_request_unredacted( + monkeypatch: pytest.MonkeyPatch, request_data: dict +) -> None: + monkeypatch.setattr(litellm, "turn_off_message_logging", True) + + assert should_redact_failed_request(request_data) is False + + +def test_scrubbed_error_information_for_request_scrubs_only_when_the_request_redacts() -> None: + error_information: Final = StandardLoggingPayloadErrorInformation( + error_code="400", error_class="BadRequestError", error_message=f"bad: {PROMPT}", traceback=f"tb {PROMPT}" + ) + messages: Final = [{"role": "user", "content": PROMPT}] + + scrubbed: Final = scrubbed_error_information_for_request( + error_information, {"messages": messages, "turn_off_message_logging": True} + ) + + assert scrubbed == { + **error_information, + "error_message": "bad: redacted-by-litellm", + "traceback": "tb redacted-by-litellm", + } + assert scrubbed_error_information_for_request(error_information, {"messages": messages}) == error_information diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py index dc46b5faf4e..b578a3e96b9 100644 --- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py @@ -3674,3 +3674,30 @@ async def test_async_post_call_failure_hook_disable_pre_routing_error_logs( user_api_key_dict=user_api_key_dict, ) assert len(writer.rows) == rows_written + + +@pytest.mark.asyncio +@pytest.mark.parametrize("redact", [True, False], ids=["redaction_on", "redaction_off"]) +async def test_async_post_call_failure_hook_scrubs_the_prompt_out_of_the_persisted_error( + monkeypatch: pytest.MonkeyPatch, redact: bool +): + secret: Final = "my ssn is 123-45-6789" + monkeypatch.setattr(litellm, "turn_off_message_logging", redact) + spend_writer: Final = create_autospec(DBSpendUpdateWriter, instance=True) + + await ProxyDBLogger(spend_writer=lambda: spend_writer).async_post_call_failure_hook( + request_data={"model": "gpt-4o", "messages": [{"role": "user", "content": secret}], "metadata": {}}, + original_exception=litellm.BadRequestError( + message=f"Unsupported content: {secret}", model="gpt-4o", llm_provider="openai" + ), + user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key"), + traceback_str=f"Traceback: {secret}", + ) + + persisted: Final = spend_writer.update_database.call_args.kwargs["kwargs"]["litellm_params"]["metadata"][ + "error_information" + ] + assert (secret in json.dumps(persisted)) is not redact + assert "Unsupported content: " in persisted["error_message"] + assert persisted["error_class"] == "BadRequestError" + assert persisted["error_code"] == "400" diff --git a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py index 3cdd2edb2fb..2b317a9dc97 100644 --- a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py +++ b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py @@ -202,6 +202,36 @@ async def test_proxy_only_error_log_keeps_tags_and_drops_caller_logging_fields( assert details["litellm_params"][metadata_field]["user_api_key_team_id"] == "team-a" +@pytest.mark.asyncio +@pytest.mark.parametrize( + "opt_in", + [ + {"turn_off_message_logging": True}, + {"metadata": {"turn_off_message_logging": True}}, + {"metadata": {"headers": {"x-litellm-enable-message-redaction": "true"}}}, + ], + ids=["body_flag", "metadata_flag", "enable_header"], +) +async def test_proxy_only_error_log_keeps_the_request_redaction_opt_in(opt_in: Mapping[str, object]) -> None: + secret: Final = "my ssn is 123-45-6789" + request_data: Final[dict[str, object]] = { + "model": "gpt-4o", + "messages": [{"role": "user", "content": secret}], + **opt_in, + } + + await ProxyLogging(user_api_key_cache=DualCache())._handle_logging_proxy_only_error( + request_data=request_data, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-rejected", team_id="team-a", request_route="/v1/rerank"), + route="/v1/rerank", + original_exception=HTTPException(status_code=400, detail=f"blocked: {secret}"), + ) + + payload: Final = request_data["litellm_logging_obj"].model_call_details["standard_logging_object"] + assert secret not in json.dumps(payload, default=str) + assert "blocked: redacted-by-litellm" in payload["error_str"] + + @pytest.mark.asyncio @pytest.mark.parametrize("metadata_field", ["metadata", "litellm_metadata"]) async def test_proxy_only_error_log_drops_malformed_metadata_tags(metadata_field: str) -> None: