diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index c244363e389..83f55bed24f 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -771,7 +771,9 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac return OLD_LITELLM_METADATA_FIELD def redact_standard_logging_payload_from_model_call_details( - self, model_call_details: Dict + self, + model_call_details: Dict, + global_redaction_applied: bool = False, ) -> Dict: """ Redacts or excludes fields from StandardLoggingPayload before callbacks receive it. @@ -784,6 +786,17 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac This is useful for logging payloads that contain sensitive information. """ + # Only skip if global redaction applied AND method is not overridden + if global_redaction_applied: + # Check if method was overridden anywhere in the inheritance chain (walks full MRO) + method_name = "redact_standard_logging_payload_from_model_call_details" + is_overridden = getattr(type(self), method_name) is not getattr(CustomLogger, method_name) + + if not is_overridden: + # Safe to skip - using default implementation + return model_call_details + # Method was overridden - might do additional redaction, so proceed + import litellm from copy import copy diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 25086352462..04916552fb7 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -68,6 +68,7 @@ from litellm.litellm_core_utils.model_param_helper import ModelParamHelper from litellm.litellm_core_utils.redact_messages import ( redact_message_input_output_from_custom_logger, redact_message_input_output_from_logging, + should_redact_message_logging, ) from litellm.llms.base_llm.ocr.transformation import OCRResponse from litellm.llms.base_llm.search.transformation import SearchResponse @@ -2500,11 +2501,12 @@ class Logging(LiteLLMLoggingBaseClass): global_callbacks=litellm._async_success_callback, ) + _model_call_details = self.model_call_details if hasattr(self, "model_call_details") else {} + global_redaction_applied = should_redact_message_logging(_model_call_details) result = redact_message_input_output_from_logging( - model_call_details=( - self.model_call_details if hasattr(self, "model_call_details") else {} - ), + model_call_details=_model_call_details, result=result, + should_redact=global_redaction_applied, ) ## LOGGING HOOK ## @@ -2529,7 +2531,10 @@ class Logging(LiteLLMLoggingBaseClass): ) elif isinstance(callback, CustomLogger): result = redact_message_input_output_from_custom_logger( - result=result, litellm_logging_obj=self, custom_logger=callback + result=result, + litellm_logging_obj=self, + custom_logger=callback, + global_redaction_applied=global_redaction_applied, ) self.model_call_details, result = await callback.async_logging_hook( kwargs=self.model_call_details, @@ -2584,7 +2589,8 @@ class Logging(LiteLLMLoggingBaseClass): ################################## # call redaction hook for custom logger model_call_details = callback.redact_standard_logging_payload_from_model_call_details( - model_call_details=model_call_details + model_call_details=model_call_details, + global_redaction_applied=global_redaction_applied, ) ################################## if self.stream is True: diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index 5d6d1fbc1c5..4fe19f042fd 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -30,8 +30,15 @@ else: def redact_message_input_output_from_custom_logger( - litellm_logging_obj: LiteLLMLoggingObject, result, custom_logger: CustomLogger + litellm_logging_obj: LiteLLMLoggingObject, + result, + custom_logger: CustomLogger, + global_redaction_applied: bool = False, ): + # skip redundant redaction if global redaction was already applied + if global_redaction_applied: + return result + if ( hasattr(custom_logger, "message_logging") and custom_logger.message_logging is not True @@ -72,6 +79,44 @@ def _redact_responses_api_output(output_items): summary_item.text = "redacted-by-litellm" +def _redact_standard_logging_object(model_call_details: dict): + """Redact messages and response inside standard_logging_object if present.""" + standard_logging_object = model_call_details.get("standard_logging_object") + if standard_logging_object is None: + return + + redacted_str = "redacted-by-litellm" + + if standard_logging_object.get("messages") is not None: + standard_logging_object["messages"] = [ + {"role": "user", "content": redacted_str} + ] + + response = standard_logging_object.get("response") + if response is not None: + if isinstance(response, dict) and "output" in response: + # ResponsesAPIResponse format - redact content in output items + if isinstance(response.get("output"), list): + for output_item in response["output"]: + if isinstance(output_item, dict) and "content" in output_item: + if isinstance(output_item["content"], list): + for content_item in output_item["content"]: + if ( + isinstance(content_item, dict) + and "text" in content_item + ): + content_item["text"] = redacted_str + elif isinstance(response, str): + standard_logging_object["response"] = redacted_str + else: + # Standard ModelResponse dict format + standard_logging_object["response"] = { + "choices": [ + {"message": {"content": redacted_str}} + ] + } + + def perform_redaction(model_call_details: dict, result): """ Performs the actual redaction on the logging object and result. @@ -83,6 +128,9 @@ def perform_redaction(model_call_details: dict, result): model_call_details["prompt"] = "" model_call_details["input"] = "" + # Redact standard_logging_object if present + _redact_standard_logging_object(model_call_details) + # Redact streaming response if ( model_call_details.get("stream", False) is True @@ -182,13 +230,18 @@ def should_redact_message_logging(model_call_details: dict) -> bool: def redact_message_input_output_from_logging( - model_call_details: dict, result, input: Optional[Any] = None + model_call_details: dict, + result, + input: Optional[Any] = None, + should_redact: Optional[bool] = None, ) -> Any: """ Removes messages, prompts, input, response from logging. This modifies the data in-place only redacts when litellm.turn_off_message_logging == True """ - if should_redact_message_logging(model_call_details): + if should_redact is None: + should_redact = should_redact_message_logging(model_call_details) + if should_redact: return perform_redaction(model_call_details, result) return result diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 639a7886cc0..22c22a97681 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -1443,3 +1443,276 @@ class TestProcessDynamicCallbacksOrdering: obj.process_dynamic_callbacks() assert call_order.index("failure") < call_order.index("async_failure") + + +def test_global_redaction_skips_per_callback_redaction(): + """ + When global_redaction_applied=True, per-callback redaction functions should + return early without processing to avoid redundant work. + """ + from litellm.litellm_core_utils.redact_messages import ( + redact_message_input_output_from_custom_logger, + ) + from litellm.integrations.custom_logger import CustomLogger + + # Test redact_message_input_output_from_custom_logger skips when global redaction applied + custom_logger = CustomLogger() + custom_logger.message_logging = False # Would normally trigger redaction + + mock_logging_obj = MagicMock() + mock_logging_obj.model_call_details = {"messages": [{"content": "secret"}]} + + original_result = {"response": "already-redacted"} + + result = redact_message_input_output_from_custom_logger( + litellm_logging_obj=mock_logging_obj, + result=original_result, + custom_logger=custom_logger, + global_redaction_applied=True, + ) + + assert result is original_result # Should return unchanged (early return) + + # Test CustomLogger.redact_standard_logging_payload_from_model_call_details skips + custom_logger.turn_off_message_logging = True + + model_call_details = { + "messages": [{"content": "secret"}], + "standard_logging_object": {"response": "sensitive"}, + } + + result = custom_logger.redact_standard_logging_payload_from_model_call_details( + model_call_details=model_call_details, + global_redaction_applied=True, + ) + + assert result is model_call_details # Should return unchanged (early return) + + +def test_perform_redaction_redacts_standard_logging_object(): + """ + When perform_redaction runs (global redaction path), it should also redact + the standard_logging_object inside model_call_details. + + This prevents sensitive data from leaking to callbacks (e.g. Langfuse, Datadog) + when per-callback redaction is skipped due to global_redaction_applied=True. + """ + from litellm.litellm_core_utils.redact_messages import perform_redaction + + # Standard ModelResponse format + model_call_details = { + "messages": [{"role": "user", "content": "my secret prompt"}], + "prompt": "my secret prompt", + "input": "my secret input", + "standard_logging_object": { + "messages": [{"role": "user", "content": "my secret prompt"}], + "response": { + "choices": [ + {"message": {"content": "sensitive response data"}} + ] + }, + }, + } + + perform_redaction(model_call_details, result=None) + + slo = model_call_details["standard_logging_object"] + # Messages should be redacted + assert slo["messages"] == [{"role": "user", "content": "redacted-by-litellm"}] + # Response should be redacted + assert slo["response"]["choices"][0]["message"]["content"] == "redacted-by-litellm" + + # ResponsesAPIResponse format + model_call_details_responses = { + "messages": [{"role": "user", "content": "my secret prompt"}], + "prompt": "", + "input": "", + "standard_logging_object": { + "messages": [{"role": "user", "content": "my secret prompt"}], + "response": { + "output": [ + { + "content": [ + {"type": "output_text", "text": "sensitive response"} + ] + } + ] + }, + }, + } + + perform_redaction(model_call_details_responses, result=None) + + slo = model_call_details_responses["standard_logging_object"] + assert slo["messages"] == [{"role": "user", "content": "redacted-by-litellm"}] + assert slo["response"]["output"][0]["content"][0]["text"] == "redacted-by-litellm" + + # No standard_logging_object - should not raise + model_call_details_none = { + "messages": [{"role": "user", "content": "prompt"}], + "prompt": "", + "input": "", + } + perform_redaction(model_call_details_none, result=None) # should not raise + + +def test_global_redaction_covers_standard_logging_object_for_callbacks(): + """ + End-to-end test: when global redaction is applied, per-callback redaction + is skipped, but standard_logging_object must still be redacted because + perform_redaction (the global path) now handles it. + """ + from litellm.litellm_core_utils.redact_messages import ( + redact_message_input_output_from_custom_logger, + redact_message_input_output_from_logging, + ) + from litellm.integrations.custom_logger import CustomLogger + + model_call_details = { + "messages": [{"role": "user", "content": "secret prompt"}], + "prompt": "secret prompt", + "input": "secret input", + "litellm_params": {}, + "standard_logging_object": { + "messages": [{"role": "user", "content": "secret prompt"}], + "response": "sensitive response string", + }, + } + + # Step 1: Global redaction (simulates what async_success_handler does) + redact_message_input_output_from_logging( + model_call_details=model_call_details, + result=None, + should_redact=True, + ) + + # Verify standard_logging_object was redacted by the global path + slo = model_call_details["standard_logging_object"] + assert slo["messages"] == [{"role": "user", "content": "redacted-by-litellm"}] + assert slo["response"] == "redacted-by-litellm" + + # Step 2: Per-callback redaction is skipped (as expected) + custom_logger = CustomLogger() + custom_logger.message_logging = False + + mock_logging_obj = MagicMock() + mock_logging_obj.model_call_details = model_call_details + + result = redact_message_input_output_from_custom_logger( + litellm_logging_obj=mock_logging_obj, + result=None, + custom_logger=custom_logger, + global_redaction_applied=True, + ) + + # Per-callback redaction skipped, but standard_logging_object is still redacted + # from the global path above + assert slo["messages"] == [{"role": "user", "content": "redacted-by-litellm"}] + assert slo["response"] == "redacted-by-litellm" + + +def test_per_callback_redaction_proceeds_when_global_redaction_not_applied(): + """ + When global_redaction_applied=False, per-callback redaction should still + proceed normally if the callback has redaction enabled. + """ + from litellm.litellm_core_utils.redact_messages import ( + redact_message_input_output_from_custom_logger, + ) + from litellm.integrations.custom_logger import CustomLogger + + # Test redact_message_input_output_from_custom_logger proceeds when global redaction NOT applied + custom_logger = CustomLogger() + custom_logger.message_logging = False # Triggers redaction + + mock_logging_obj = MagicMock() + mock_logging_obj.model_call_details = {"messages": [{"content": "secret"}]} + + original_result = {"response": "sensitive-data"} + + result = redact_message_input_output_from_custom_logger( + litellm_logging_obj=mock_logging_obj, + result=original_result, + custom_logger=custom_logger, + global_redaction_applied=False, + ) + + # Result should be different (redacted), not the same object + assert result is not original_result + + +def test_method_override_detection_walks_full_mro(): + """ + Test that method override detection correctly walks the full Method Resolution Order (MRO), + not just the immediate class's __dict__. + + This tests the fix for a bug where: + - `method_name in type(self).__dict__` only checks the immediate class + - `getattr(type(self), method_name) is not getattr(CustomLogger, method_name)` walks the full MRO + + The bug caused incorrect behavior when: + - ClassA(CustomLogger) overrides a method + - ClassB(ClassA) does NOT override the method + - ClassB instance should still detect the override from ClassA + """ + from litellm.integrations.custom_logger import CustomLogger + + # Case 1: Direct override - should be detected + class DirectOverrideLogger(CustomLogger): + def redact_standard_logging_payload_from_model_call_details( + self, model_call_details, global_redaction_applied=False + ): + # Custom implementation + return {"custom": "redacted"} + + direct_logger = DirectOverrideLogger() + method_name = "redact_standard_logging_payload_from_model_call_details" + + # Using the correct MRO-aware check + is_overridden_mro = getattr(type(direct_logger), method_name) is not getattr( + CustomLogger, method_name + ) + assert is_overridden_mro is True, "Direct override should be detected" + + # Case 2: Inherited override (the bug case) - should also be detected + class InheritedOverrideLogger(DirectOverrideLogger): + # Does NOT override the method - inherits from DirectOverrideLogger + pass + + inherited_logger = InheritedOverrideLogger() + + # The buggy check (only checks immediate class __dict__) would return False + buggy_check = method_name in type(inherited_logger).__dict__ + assert buggy_check is False, "Buggy check incorrectly misses inherited override" + + # The correct MRO-aware check should return True + is_overridden_mro = getattr(type(inherited_logger), method_name) is not getattr( + CustomLogger, method_name + ) + assert is_overridden_mro is True, "MRO check should detect inherited override" + + # Case 3: No override - should NOT be detected as overridden + class NoOverrideLogger(CustomLogger): + # Does NOT override the method + pass + + no_override_logger = NoOverrideLogger() + is_overridden_mro = getattr(type(no_override_logger), method_name) is not getattr( + CustomLogger, method_name + ) + assert is_overridden_mro is False, "No override should not be detected" + + # Case 4: Verify actual behavior - inherited override should NOT skip redaction + model_call_details = { + "messages": [{"content": "secret"}], + "standard_logging_object": {"response": "sensitive"}, + } + + # With global_redaction_applied=True and an inherited override, + # the method should NOT early-return (should proceed to custom implementation) + result = inherited_logger.redact_standard_logging_payload_from_model_call_details( + model_call_details=model_call_details, + global_redaction_applied=True, + ) + # DirectOverrideLogger returns {"custom": "redacted"} + assert result == {"custom": "redacted"}, "Inherited override should execute custom logic"