From 1b04f2f2bed56447870f6bbd44743e57a0e4dfba Mon Sep 17 00:00:00 2001 From: yucheng Date: Mon, 14 Sep 2026 18:34:06 +0000 Subject: [PATCH] fix(logging): apply callback_settings turn_off_message_logging per callback Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/litellm_logging.py | 45 +++++- .../test_litellm_logging.py | 140 ++++++++++++++++++ 2 files changed, 180 insertions(+), 5 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 9ba9fd082f3..ae608244b38 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2856,7 +2856,12 @@ class Logging(LiteLLMLoggingBaseClass): ): # custom logger class if self.stream and complete_streaming_response is None: callback.log_stream_event( - kwargs=self.model_call_details, + kwargs=redact_streaming_responses_for_custom_logger( + model_call_details=callback.redact_standard_logging_payload_from_model_call_details( + model_call_details=self.model_call_details + ), + custom_logger=callback, + ), response_obj=result, start_time=start_time, end_time=end_time, @@ -2869,7 +2874,12 @@ class Logging(LiteLLMLoggingBaseClass): result = self.model_call_details["complete_response"] callback.log_success_event( - kwargs=self.model_call_details, + kwargs=redact_streaming_responses_for_custom_logger( + model_call_details=callback.redact_standard_logging_payload_from_model_call_details( + model_call_details=self.model_call_details + ), + custom_logger=callback, + ), response_obj=result, start_time=start_time, end_time=end_time, @@ -3493,7 +3503,9 @@ class Logging(LiteLLMLoggingBaseClass): start_time=start_time, end_time=end_time, response_obj=result, - kwargs=self.model_call_details, + kwargs=callback.redact_standard_logging_payload_from_model_call_details( + model_call_details=self.model_call_details + ), ) if callback == "langfuse": global langFuseLogger @@ -3624,7 +3636,9 @@ class Logging(LiteLLMLoggingBaseClass): continue if isinstance(callback, CustomLogger): # custom logger class await callback.async_log_failure_event( - kwargs=self.model_call_details, + kwargs=callback.redact_standard_logging_payload_from_model_call_details( + model_call_details=self.model_call_details + ), response_obj=result, start_time=start_time, end_time=end_time, @@ -4236,8 +4250,27 @@ def _init_custom_logger_compatible_class( custom_logger_init_args: dict | None = {}, ) -> CustomLogger | None: """ - Initialize a custom logger compatible class + Initialize a custom logger compatible class and apply its proxy `callback_settings`. """ + custom_logger: Final = _construct_custom_logger_compatible_class( + logging_integration, internal_usage_cache, llm_router, custom_logger_init_args + ) + if custom_logger is None: + return None + turn_off_message_logging: Final = _get_custom_logger_settings_from_proxy_server( + callback_name=logging_integration + ).get("turn_off_message_logging") + if isinstance(turn_off_message_logging, bool): + custom_logger.turn_off_message_logging = turn_off_message_logging + return custom_logger + + +def _construct_custom_logger_compatible_class( + logging_integration: _custom_logger_compatible_callbacks_literal, + internal_usage_cache: DualCache | None, + llm_router: Any | None, # expect litellm.Router, but typing errors due to circular import + custom_logger_init_args: dict | None = None, +) -> CustomLogger | None: try: custom_logger_init_args = custom_logger_init_args or {} if logging_integration == "agentops": # Add AgentOps initialization @@ -5222,6 +5255,8 @@ def _get_custom_logger_settings_from_proxy_server(callback_name: str) -> dict: callback_settings: otel: message_logging: False + langsmith: + turn_off_message_logging: true """ if litellm.callback_settings: return dict(litellm.callback_settings.get(callback_name, {})) diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 70f9bae283b..9604f72daa0 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -45,6 +45,24 @@ def logging_obj(): ) +class _RecordingCustomLogger(CustomLogger): + def __init__(self, turn_off_message_logging: bool): + super().__init__(turn_off_message_logging=turn_off_message_logging) + self.received_kwargs: list[dict] = [] + + def log_stream_event(self, kwargs: dict, response_obj: object, start_time, end_time): + self.received_kwargs.append(kwargs) + + def log_success_event(self, kwargs: dict, response_obj: object, start_time, end_time): + self.received_kwargs.append(kwargs) + + def log_failure_event(self, kwargs: dict, response_obj: object, start_time, end_time): + self.received_kwargs.append(kwargs) + + async def async_log_failure_event(self, kwargs: dict, response_obj: object, start_time, end_time): + self.received_kwargs.append(kwargs) + + def test_get_combined_callback_list_preserves_insertion_order(logging_obj): assert logging_obj.get_combined_callback_list( dynamic_success_callbacks=["prometheus", "langfuse", "datadog", "otel", "s3"], @@ -976,6 +994,128 @@ async def test_datadog_logger_not_shadowed_by_llm_obs(monkeypatch): logging_module._in_memory_loggers.clear() +def test_init_custom_logger_applies_callback_settings_turn_off_message_logging(monkeypatch): + from litellm.litellm_core_utils import litellm_logging as logging_module + + monkeypatch.setattr(litellm, "callback_settings", {"langsmith": {"turn_off_message_logging": True}}) + logging_module._in_memory_loggers.clear() + try: + configured_logger = logging_module._init_custom_logger_compatible_class( + logging_integration="langsmith", + internal_usage_cache=None, + llm_router=None, + custom_logger_init_args={}, + ) + default_logger = logging_module._init_custom_logger_compatible_class( + logging_integration="literalai", + internal_usage_cache=None, + llm_router=None, + custom_logger_init_args={}, + ) + + assert configured_logger is not None + assert configured_logger.turn_off_message_logging is True + assert default_logger is not None + assert default_logger.turn_off_message_logging is False + + monkeypatch.setattr(litellm, "callback_settings", {}) + logging_module._in_memory_loggers.clear() + empty_settings_logger = logging_module._init_custom_logger_compatible_class( + logging_integration="langsmith", + internal_usage_cache=None, + llm_router=None, + custom_logger_init_args={}, + ) + + assert empty_settings_logger is not None + assert empty_settings_logger.turn_off_message_logging is False + finally: + logging_module._in_memory_loggers.clear() + + +def test_success_handler_redacts_custom_logger_payload_per_callback(logging_obj): + redacting_logger = _RecordingCustomLogger(turn_off_message_logging=True) + plain_logger = _RecordingCustomLogger(turn_off_message_logging=False) + logging_obj.stream = False + logging_obj.model_call_details["litellm_params"] = {} + standard_logging_object = { + "messages": [{"role": "user", "content": "original message"}], + "response": {"choices": []}, + } + + with patch.object( + logging_obj, + "get_combined_callback_list", + return_value=[redacting_logger, plain_logger], + ): + logging_obj.success_handler( + result={"id": "response"}, + standard_logging_object=standard_logging_object, + ) + + assert redacting_logger.received_kwargs[0]["standard_logging_object"]["messages"][0]["content"] == ( + "redacted-by-litellm" + ) + assert plain_logger.received_kwargs[0]["standard_logging_object"]["messages"][0]["content"] == "original message" + assert ( + logging_obj.model_call_details["standard_logging_object"]["messages"][0]["content"] == "original message" + ) + + +def test_failure_handler_redacts_custom_logger_payload_per_callback(logging_obj): + redacting_logger = _RecordingCustomLogger(turn_off_message_logging=True) + plain_logger = _RecordingCustomLogger(turn_off_message_logging=False) + logging_obj.stream = False + logging_obj.model_call_details["litellm_params"] = {} + logging_obj.model_call_details["messages"] = [{"role": "user", "content": "original message"}] + + with patch.object( + logging_obj, + "get_combined_callback_list", + return_value=[redacting_logger, plain_logger], + ): + logging_obj.failure_handler( + exception=Exception("test error"), + traceback_exception="", + ) + + assert redacting_logger.received_kwargs[0]["standard_logging_object"]["messages"][0]["content"] == ( + "redacted-by-litellm" + ) + assert plain_logger.received_kwargs[0]["standard_logging_object"]["messages"][0]["content"] == "original message" + assert ( + logging_obj.model_call_details["standard_logging_object"]["messages"][0]["content"] == "original message" + ) + + +@pytest.mark.asyncio +async def test_async_failure_handler_redacts_custom_logger_payload_per_callback(logging_obj): + redacting_logger = _RecordingCustomLogger(turn_off_message_logging=True) + plain_logger = _RecordingCustomLogger(turn_off_message_logging=False) + logging_obj.stream = False + logging_obj.call_type = "acompletion" + logging_obj.model_call_details["litellm_params"] = {} + logging_obj.model_call_details["messages"] = [{"role": "user", "content": "original message"}] + + with patch.object( + logging_obj, + "get_combined_callback_list", + return_value=[redacting_logger, plain_logger], + ): + await logging_obj.async_failure_handler( + exception=Exception("test error"), + traceback_exception="", + ) + + assert redacting_logger.received_kwargs[0]["standard_logging_object"]["messages"][0]["content"] == ( + "redacted-by-litellm" + ) + assert plain_logger.received_kwargs[0]["standard_logging_object"]["messages"][0]["content"] == "original message" + assert ( + logging_obj.model_call_details["standard_logging_object"]["messages"][0]["content"] == "original message" + ) + + @pytest.mark.asyncio async def test_logfire_logger_accepts_env_vars_for_base_url(monkeypatch): """Ensure Logfire logger uses LOGFIRE_BASE_URL to build the OTLP HTTP endpoint (/v1/traces)."""