fix(logging): apply a callback's own message redaction to its failure events

litellm/integrations/custom_logger.py

@@ -3,13 +3,12 @@
 import re
 import traceback
 from collections.abc import AsyncGenerator, Mapping, Sequence
-from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional
 
 from pydantic import BaseModel
 
 from litellm._logging import verbose_logger
-from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER, EMPTY_MAPPING
+from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER
 from litellm.types.integrations.argilla import ArgillaItem
 from litellm.types.integrations.custom_logger import AgenticLoopPlan
 from litellm.types.llms.openai import AllMessageValues, ChatCompletionRequest
@@ -912,46 +911,12 @@ class CustomLogger:  # https://docs.litellm.ai/docs/observability/custom_callbac
         This is useful for logging payloads that contain sensitive information.
         """
         import litellm
-        from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS
-        from litellm.litellm_core_utils.redact_messages import (
-            redacted_litellm_params,
-            redacted_standard_logging_payload,
+        from litellm.litellm_core_utils.redact_messages import redacted_standard_logging_details
+
+        return redacted_standard_logging_details(
+            model_call_details, self, excluded_fields=getattr(litellm, "standard_logging_payload_excluded_fields", None)
         )
 
-        turn_off_message_logging: Final[bool] = getattr(self, "turn_off_message_logging", False)
-        excluded_fields: Final[list[str] | None] = getattr(litellm, "standard_logging_payload_excluded_fields", None)
-
-        # Early return if no processing needed
-        if turn_off_message_logging is False and not excluded_fields:
-            return model_call_details
-
-        params: Final = model_call_details.get("litellm_params")
-        redacted_params: Final = (
-            MappingProxyType({"litellm_params": redacted_litellm_params(params)})
-            if turn_off_message_logging and isinstance(params, Mapping)
-            else EMPTY_MAPPING
-        )
-        standard_logging_object: Final = model_call_details.get("standard_logging_object")
-        if standard_logging_object is None:
-            return {**model_call_details, **redacted_params}
-
-        # Make a copy of just the standard_logging_object to avoid modifying the original
-        standard_logging_object_copy: Final = {
-            key: value
-            for key, value in standard_logging_object.items()
-            if key not in (excluded_fields or ()) and not (turn_off_message_logging and key in CLASSIFIER_AUDIT_FIELDS)
-        }
-
-        return {
-            **model_call_details,
-            **redacted_params,
-            "standard_logging_object": (
-                redacted_standard_logging_payload(standard_logging_object_copy)
-                if turn_off_message_logging and not self.redacts_messages_itself()
-                else standard_logging_object_copy
-            ),
-        }
-
     async def get_proxy_server_request_from_cold_storage_with_object_key(
         self,
         object_key: str,

litellm/litellm_core_utils/litellm_logging.py

@@ -124,6 +124,7 @@ from litellm.litellm_core_utils.logging_utils import (
 from litellm.litellm_core_utils.model_param_helper import ModelParamHelper
 from litellm.litellm_core_utils.ptu_pricing import is_spilled_over_ptu_request
 from litellm.litellm_core_utils.redact_messages import (
+    failure_details_for_custom_logger,
     failure_request_texts,
     redact_message_input_output_from_custom_logger,
     redact_message_input_output_from_logging,
@@ -4497,7 +4498,7 @@ class Logging(LiteLLMLoggingBaseClass):
                             start_time=start_time,
                             end_time=end_time,
                             response_obj=result,
-                            kwargs=self.model_call_details,
+                            kwargs=failure_details_for_custom_logger(self.model_call_details, callback),
                         )
                     if callback == "langfuse":
                         global langFuseLogger
@@ -4632,7 +4633,7 @@ class Logging(LiteLLMLoggingBaseClass):
                     continue
                 if isinstance(callback, CustomLogger):  # custom logger class
                     await callback.async_log_failure_event(
-                        kwargs=self.model_call_details,
+                        kwargs=failure_details_for_custom_logger(self.model_call_details, callback),
                         response_obj=result,
                         start_time=start_time,
                         end_time=end_time,

litellm/litellm_core_utils/redact_messages.py

@@ -16,14 +16,15 @@ import json
 import re
 from collections.abc import Collection, Mapping, Sequence
 from dataclasses import replace
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final
 
 from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError
 
 import litellm
-from litellm.constants import REDACTED_BY_LITELLM
+from litellm.constants import EMPTY_MAPPING, REDACTED_BY_LITELLM
 from litellm.integrations.custom_logger import CustomLogger
-from litellm.litellm_core_utils.classifier_logging import without_classifier_audit
+from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS, without_classifier_audit
 from litellm.litellm_core_utils.core_helpers import (
     get_metadata_variable_name_from_kwargs,
 )
@@ -65,6 +66,80 @@ def redact_message_input_output_from_custom_logger(
     return result
 
 
+def failure_details_for_custom_logger(
+    model_call_details: dict[str, object],  # mutable-ok: the kwargs dict a CustomLogger failure hook receives
+    custom_logger: CustomLogger,
+) -> dict[str, object]:  # mutable-ok: the kwargs dict a CustomLogger failure hook receives
+    """
+    The per-callback message redaction the success path applies, for a failure event, with the
+    request's text also scrubbed out of the error. The excluded fields are left out, since several
+    callbacks index the payload's fields on failure.
+    """
+    redacted: Final = redacted_standard_logging_details(model_call_details, custom_logger, excluded_fields=None)
+    if redacted is model_call_details and not _opted_out_of_message_logging(custom_logger):
+        return model_call_details
+    texts: Final = failure_request_texts(model_call_details)
+    payload: Final = _as_mapping(redacted.get("standard_logging_object"))
+    traceback_exception: Final = redacted.get("traceback_exception")
+    return redacted_model_call_details_for_custom_logger(
+        {
+            **redacted,
+            **(
+                {"standard_logging_object": {**payload, **scrubbed_error_fields(payload, texts)}}
+                if payload is not None
+                else {}
+            ),
+            **(
+                {"traceback_exception": scrub_request_text(traceback_exception, texts)}
+                if isinstance(traceback_exception, str)
+                else {}
+            ),
+        },
+        custom_logger,
+    )
+
+
+def redacted_standard_logging_details(
+    model_call_details: dict[str, object],  # mutable-ok: the kwargs dict a CustomLogger hook receives
+    custom_logger: CustomLogger,
+    *,
+    excluded_fields: Sequence[str] | None,
+) -> dict[str, object]:  # mutable-ok: the kwargs dict a CustomLogger hook receives
+    """``CustomLogger.redact_standard_logging_payload_from_model_call_details`` with the excluded fields passed in."""
+    turn_off_message_logging: Final[bool] = getattr(custom_logger, "turn_off_message_logging", False)
+
+    # Early return if no processing needed
+    if turn_off_message_logging is False and not excluded_fields:
+        return model_call_details
+
+    params: Final = _as_mapping(model_call_details.get("litellm_params"))
+    redacted_params: Final = (
+        MappingProxyType({"litellm_params": redacted_litellm_params(params)})
+        if turn_off_message_logging and params is not None
+        else EMPTY_MAPPING
+    )
+    standard_logging_object: Final = _as_mapping(model_call_details.get("standard_logging_object"))
+    if standard_logging_object is None:
+        return {**model_call_details, **redacted_params}
+
+    # Make a copy of just the standard_logging_object to avoid modifying the original
+    standard_logging_object_copy: Final = {
+        key: value
+        for key, value in standard_logging_object.items()
+        if key not in (excluded_fields or ()) and not (turn_off_message_logging and key in CLASSIFIER_AUDIT_FIELDS)
+    }
+
+    return {
+        **model_call_details,
+        **redacted_params,
+        "standard_logging_object": (
+            redacted_standard_logging_payload(standard_logging_object_copy)
+            if turn_off_message_logging and not custom_logger.redacts_messages_itself()
+            else standard_logging_object_copy
+        ),
+    }
+
+
 def redacted_model_call_details_for_custom_logger(model_call_details: dict, custom_logger: CustomLogger) -> dict:
     """
     Returns a copy of model_call_details with the request messages, instructions and streaming responses

tests/unit/litellm_core_utils/test_litellm_logging.py

@@ -11750,6 +11750,11 @@ class _FailureCapture(CustomLogger):
         self.async_kwargs = kwargs
 
 
+class _SelfRedactingFailureCapture(_FailureCapture):
+    def redacts_messages_itself(self) -> bool:
+        return True
+
+
 def _prompt_failure_logging_obj(litellm_params: Mapping[str, object] | None = None) -> LitellmLogging:
     logging_obj: Final = LitellmLogging(
         model="gpt-4o",
@@ -11819,6 +11824,41 @@ async def test_failure_with_redaction_off_logs_the_error_as_raised(monkeypatch):
     assert SCRUB_PROMPT in kwargs["traceback_exception"]
 
 
+@pytest.mark.asyncio
+async def test_failure_redacts_messages_and_error_for_a_callback_that_opted_out_of_message_logging(monkeypatch):
+    opted_out: Final = _FailureCapture(message_logging=False)
+    logging_everything: Final = _FailureCapture()
+    monkeypatch.setattr(litellm, "turn_off_message_logging", False)
+    monkeypatch.setattr(litellm, "_async_failure_callback", [opted_out, logging_everything])
+
+    await _prompt_failure_logging_obj().async_failure_handler(_prompt_echo_error(), "Traceback")
+
+    assert opted_out.async_kwargs is not None and logging_everything.async_kwargs is not None
+    assert SCRUB_PROMPT not in json.dumps(opted_out.async_kwargs["standard_logging_object"], default=str)
+    assert opted_out.async_kwargs["messages"] == [{"role": "user", "content": "redacted-by-litellm"}]
+    assert SCRUB_PROMPT in logging_everything.async_kwargs["standard_logging_object"]["error_str"]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("capture_type", [_FailureCapture, _SelfRedactingFailureCapture])
+async def test_failure_scrubs_the_error_for_a_callback_with_its_own_turn_off_message_logging(monkeypatch, capture_type):
+    capture: Final = capture_type(turn_off_message_logging=True)
+
+    kwargs: Final = await _async_failure_payload(monkeypatch, redact=False, error=_prompt_echo_error(), capture=capture)
+
+    assert SCRUB_PROMPT not in kwargs["standard_logging_object"]["error_str"]
+    assert SCRUB_PROMPT not in kwargs["standard_logging_object"]["error_information"]["error_message"]
+
+
+@pytest.mark.asyncio
+async def test_failure_payload_keeps_the_excluded_fields(monkeypatch):
+    monkeypatch.setattr(litellm, "standard_logging_payload_excluded_fields", ["metadata"])
+
+    kwargs: Final = await _async_failure_payload(monkeypatch, redact=False, error=_prompt_echo_error())
+
+    assert "metadata" in kwargs["standard_logging_object"]
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize("headers", [None, "x", ["a"]], ids=["none", "str", "list"])
 @pytest.mark.parametrize("redact", [True, False], ids=["redaction_on", "redaction_off"])