From c8601d733bfbf580e846b99cfcdf0c9df778ddae Mon Sep 17 00:00:00 2001 From: yucheng Date: Fri, 30 Oct 2026 18:39:28 -0700 Subject: [PATCH] fix(logging): apply a callback's own message redaction to its failure events --- litellm/integrations/custom_logger.py | 45 ++--------- litellm/litellm_core_utils/litellm_logging.py | 5 +- litellm/litellm_core_utils/redact_messages.py | 79 ++++++++++++++++++- .../test_litellm_logging.py | 40 ++++++++++ 4 files changed, 125 insertions(+), 44 deletions(-) diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 0579eb15baf..9330e7c16f7 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -3,13 +3,12 @@ import re import traceback from collections.abc import AsyncGenerator, Mapping, Sequence -from types import MappingProxyType from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional from pydantic import BaseModel from litellm._logging import verbose_logger -from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER, EMPTY_MAPPING +from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER from litellm.types.integrations.argilla import ArgillaItem from litellm.types.integrations.custom_logger import AgenticLoopPlan from litellm.types.llms.openai import AllMessageValues, ChatCompletionRequest @@ -912,46 +911,12 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac This is useful for logging payloads that contain sensitive information. """ import litellm - from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS - from litellm.litellm_core_utils.redact_messages import ( - redacted_litellm_params, - redacted_standard_logging_payload, + from litellm.litellm_core_utils.redact_messages import redacted_standard_logging_details + + return redacted_standard_logging_details( + model_call_details, self, excluded_fields=getattr(litellm, "standard_logging_payload_excluded_fields", None) ) - turn_off_message_logging: Final[bool] = getattr(self, "turn_off_message_logging", False) - excluded_fields: Final[list[str] | None] = getattr(litellm, "standard_logging_payload_excluded_fields", None) - - # Early return if no processing needed - if turn_off_message_logging is False and not excluded_fields: - return model_call_details - - params: Final = model_call_details.get("litellm_params") - redacted_params: Final = ( - MappingProxyType({"litellm_params": redacted_litellm_params(params)}) - if turn_off_message_logging and isinstance(params, Mapping) - else EMPTY_MAPPING - ) - standard_logging_object: Final = model_call_details.get("standard_logging_object") - if standard_logging_object is None: - return {**model_call_details, **redacted_params} - - # Make a copy of just the standard_logging_object to avoid modifying the original - standard_logging_object_copy: Final = { - key: value - for key, value in standard_logging_object.items() - if key not in (excluded_fields or ()) and not (turn_off_message_logging and key in CLASSIFIER_AUDIT_FIELDS) - } - - return { - **model_call_details, - **redacted_params, - "standard_logging_object": ( - redacted_standard_logging_payload(standard_logging_object_copy) - if turn_off_message_logging and not self.redacts_messages_itself() - else standard_logging_object_copy - ), - } - async def get_proxy_server_request_from_cold_storage_with_object_key( self, object_key: str, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index c2b2f803366..06c95d5e23a 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -124,6 +124,7 @@ from litellm.litellm_core_utils.logging_utils import ( from litellm.litellm_core_utils.model_param_helper import ModelParamHelper from litellm.litellm_core_utils.ptu_pricing import is_spilled_over_ptu_request from litellm.litellm_core_utils.redact_messages import ( + failure_details_for_custom_logger, failure_request_texts, redact_message_input_output_from_custom_logger, redact_message_input_output_from_logging, @@ -4497,7 +4498,7 @@ class Logging(LiteLLMLoggingBaseClass): start_time=start_time, end_time=end_time, response_obj=result, - kwargs=self.model_call_details, + kwargs=failure_details_for_custom_logger(self.model_call_details, callback), ) if callback == "langfuse": global langFuseLogger @@ -4632,7 +4633,7 @@ class Logging(LiteLLMLoggingBaseClass): continue if isinstance(callback, CustomLogger): # custom logger class await callback.async_log_failure_event( - kwargs=self.model_call_details, + kwargs=failure_details_for_custom_logger(self.model_call_details, callback), response_obj=result, start_time=start_time, end_time=end_time, diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index c976a5cb811..1e438288ea7 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -16,14 +16,15 @@ import json import re from collections.abc import Collection, Mapping, Sequence from dataclasses import replace +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError import litellm -from litellm.constants import REDACTED_BY_LITELLM +from litellm.constants import EMPTY_MAPPING, REDACTED_BY_LITELLM from litellm.integrations.custom_logger import CustomLogger -from litellm.litellm_core_utils.classifier_logging import without_classifier_audit +from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS, without_classifier_audit from litellm.litellm_core_utils.core_helpers import ( get_metadata_variable_name_from_kwargs, ) @@ -65,6 +66,80 @@ def redact_message_input_output_from_custom_logger( return result +def failure_details_for_custom_logger( + model_call_details: dict[str, object], # mutable-ok: the kwargs dict a CustomLogger failure hook receives + custom_logger: CustomLogger, +) -> dict[str, object]: # mutable-ok: the kwargs dict a CustomLogger failure hook receives + """ + The per-callback message redaction the success path applies, for a failure event, with the + request's text also scrubbed out of the error. The excluded fields are left out, since several + callbacks index the payload's fields on failure. + """ + redacted: Final = redacted_standard_logging_details(model_call_details, custom_logger, excluded_fields=None) + if redacted is model_call_details and not _opted_out_of_message_logging(custom_logger): + return model_call_details + texts: Final = failure_request_texts(model_call_details) + payload: Final = _as_mapping(redacted.get("standard_logging_object")) + traceback_exception: Final = redacted.get("traceback_exception") + return redacted_model_call_details_for_custom_logger( + { + **redacted, + **( + {"standard_logging_object": {**payload, **scrubbed_error_fields(payload, texts)}} + if payload is not None + else {} + ), + **( + {"traceback_exception": scrub_request_text(traceback_exception, texts)} + if isinstance(traceback_exception, str) + else {} + ), + }, + custom_logger, + ) + + +def redacted_standard_logging_details( + model_call_details: dict[str, object], # mutable-ok: the kwargs dict a CustomLogger hook receives + custom_logger: CustomLogger, + *, + excluded_fields: Sequence[str] | None, +) -> dict[str, object]: # mutable-ok: the kwargs dict a CustomLogger hook receives + """``CustomLogger.redact_standard_logging_payload_from_model_call_details`` with the excluded fields passed in.""" + turn_off_message_logging: Final[bool] = getattr(custom_logger, "turn_off_message_logging", False) + + # Early return if no processing needed + if turn_off_message_logging is False and not excluded_fields: + return model_call_details + + params: Final = _as_mapping(model_call_details.get("litellm_params")) + redacted_params: Final = ( + MappingProxyType({"litellm_params": redacted_litellm_params(params)}) + if turn_off_message_logging and params is not None + else EMPTY_MAPPING + ) + standard_logging_object: Final = _as_mapping(model_call_details.get("standard_logging_object")) + if standard_logging_object is None: + return {**model_call_details, **redacted_params} + + # Make a copy of just the standard_logging_object to avoid modifying the original + standard_logging_object_copy: Final = { + key: value + for key, value in standard_logging_object.items() + if key not in (excluded_fields or ()) and not (turn_off_message_logging and key in CLASSIFIER_AUDIT_FIELDS) + } + + return { + **model_call_details, + **redacted_params, + "standard_logging_object": ( + redacted_standard_logging_payload(standard_logging_object_copy) + if turn_off_message_logging and not custom_logger.redacts_messages_itself() + else standard_logging_object_copy + ), + } + + def redacted_model_call_details_for_custom_logger(model_call_details: dict, custom_logger: CustomLogger) -> dict: """ Returns a copy of model_call_details with the request messages, instructions and streaming responses diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 8750c11c8d3..3ff8314925e 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -11750,6 +11750,11 @@ class _FailureCapture(CustomLogger): self.async_kwargs = kwargs +class _SelfRedactingFailureCapture(_FailureCapture): + def redacts_messages_itself(self) -> bool: + return True + + def _prompt_failure_logging_obj(litellm_params: Mapping[str, object] | None = None) -> LitellmLogging: logging_obj: Final = LitellmLogging( model="gpt-4o", @@ -11819,6 +11824,41 @@ async def test_failure_with_redaction_off_logs_the_error_as_raised(monkeypatch): assert SCRUB_PROMPT in kwargs["traceback_exception"] +@pytest.mark.asyncio +async def test_failure_redacts_messages_and_error_for_a_callback_that_opted_out_of_message_logging(monkeypatch): + opted_out: Final = _FailureCapture(message_logging=False) + logging_everything: Final = _FailureCapture() + monkeypatch.setattr(litellm, "turn_off_message_logging", False) + monkeypatch.setattr(litellm, "_async_failure_callback", [opted_out, logging_everything]) + + await _prompt_failure_logging_obj().async_failure_handler(_prompt_echo_error(), "Traceback") + + assert opted_out.async_kwargs is not None and logging_everything.async_kwargs is not None + assert SCRUB_PROMPT not in json.dumps(opted_out.async_kwargs["standard_logging_object"], default=str) + assert opted_out.async_kwargs["messages"] == [{"role": "user", "content": "redacted-by-litellm"}] + assert SCRUB_PROMPT in logging_everything.async_kwargs["standard_logging_object"]["error_str"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("capture_type", [_FailureCapture, _SelfRedactingFailureCapture]) +async def test_failure_scrubs_the_error_for_a_callback_with_its_own_turn_off_message_logging(monkeypatch, capture_type): + capture: Final = capture_type(turn_off_message_logging=True) + + kwargs: Final = await _async_failure_payload(monkeypatch, redact=False, error=_prompt_echo_error(), capture=capture) + + assert SCRUB_PROMPT not in kwargs["standard_logging_object"]["error_str"] + assert SCRUB_PROMPT not in kwargs["standard_logging_object"]["error_information"]["error_message"] + + +@pytest.mark.asyncio +async def test_failure_payload_keeps_the_excluded_fields(monkeypatch): + monkeypatch.setattr(litellm, "standard_logging_payload_excluded_fields", ["metadata"]) + + kwargs: Final = await _async_failure_payload(monkeypatch, redact=False, error=_prompt_echo_error()) + + assert "metadata" in kwargs["standard_logging_object"] + + @pytest.mark.asyncio @pytest.mark.parametrize("headers", [None, "x", ["a"]], ids=["none", "str", "list"]) @pytest.mark.parametrize("redact", [True, False], ids=["redaction_on", "redaction_off"])