From 9cb582307a5f811e51ed0aa2d6632e351788e9ff Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 13 Jun 2024 11:52:20 -0700 Subject: [PATCH 1/4] fix - fix redacting messages litellm --- litellm/litellm_core_utils/redact_messages.py | 63 +++++++++++++++++++ litellm/utils.py | 54 +++++----------- 2 files changed, 78 insertions(+), 39 deletions(-) create mode 100644 litellm/litellm_core_utils/redact_messages.py diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py new file mode 100644 index 00000000000..a93104e5dee --- /dev/null +++ b/litellm/litellm_core_utils/redact_messages.py @@ -0,0 +1,63 @@ +# +-----------------------------------------------+ +# | | +# | Give Feedback / Get Help | +# | https://github.com/BerriAI/litellm/issues/new | +# | | +# +-----------------------------------------------+ +# +# Thank you users! We ❤️ you! - Krrish & Ishaan + +import copy +from typing import TYPE_CHECKING, Any +import litellm + +if TYPE_CHECKING: + from litellm.utils import Logging as _LiteLLMLoggingObject + + LiteLLMLoggingObject = _LiteLLMLoggingObject +else: + LiteLLMLoggingObject = Any + + +def redact_message_input_output_from_logging( + litellm_logging_obj: LiteLLMLoggingObject, result +): + """ + Removes messages, prompts, input, response from logging. This modifies the data in-place + only redacts when litellm.turn_off_message_logging == True + """ + # check if user opted out of logging message/response to callbacks + if litellm.turn_off_message_logging is not True: + return result + + _result = copy.deepcopy(result) + # remove messages, prompts, input, response from logging + litellm_logging_obj.model_call_details["messages"] = [ + {"role": "user", "content": "redacted-by-litellm"} + ] + litellm_logging_obj.model_call_details["prompt"] = "" + litellm_logging_obj.model_call_details["input"] = "" + + # response cleaning + # ChatCompletion Responses + if ( + litellm_logging_obj.stream + and "complete_streaming_response" in litellm_logging_obj.model_call_details + ): + _streaming_response = litellm_logging_obj.model_call_details[ + "complete_streaming_response" + ] + for choice in _streaming_response.choices: + if isinstance(choice, litellm.Choices): + choice.message.content = "redacted-by-litellm" + elif isinstance(choice, litellm.utils.StreamingChoices): + choice.delta.content = "redacted-by-litellm" + else: + if _result is not None: + if isinstance(_result, litellm.ModelResponse): + if hasattr(_result, "choices") and _result.choices is not None: + for choice in _result.choices: + if isinstance(choice, litellm.Choices): + choice.message.content = "redacted-by-litellm" + elif isinstance(choice, litellm.utils.StreamingChoices): + choice.delta.content = "redacted-by-litellm" diff --git a/litellm/utils.py b/litellm/utils.py index dbde3c1a1a1..e3eaf93fe9c 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -35,6 +35,9 @@ import litellm._service_logger # for storing API inputs, outputs, and metadata from litellm.llms.custom_httpx.http_handler import HTTPHandler, AsyncHTTPHandler from litellm.caching import DualCache from litellm.types.utils import CostPerToken, ProviderField, ModelInfo +from litellm.litellm_core_utils.redact_messages import ( + redact_message_input_output_from_logging, +) oidc_cache = DualCache() @@ -1476,7 +1479,9 @@ class Logging: print_verbose( f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {traceback.format_exc()}" ) - self.redact_message_input_output_from_logging(result=original_response) + original_response = redact_message_input_output_from_logging( + litellm_logging_obj=self, result=original_response + ) # Input Integration Logging -> If you want to log the fact that an attempt to call the model was made callbacks = litellm.input_callback + self.dynamic_input_callbacks @@ -1667,7 +1672,9 @@ class Logging: else: callbacks = litellm.success_callback - self.redact_message_input_output_from_logging(result=result) + result = redact_message_input_output_from_logging( + result=result, litellm_logging_obj=self + ) for callback in callbacks: try: @@ -2294,7 +2301,9 @@ class Logging: else: callbacks = litellm._async_success_callback - self.redact_message_input_output_from_logging(result=result) + result = redact_message_input_output_from_logging( + result=result, litellm_logging_obj=self + ) for callback in callbacks: # check if callback can run for this request @@ -2504,7 +2513,9 @@ class Logging: result = None # result sent to all loggers, init this to None incase it's not created - self.redact_message_input_output_from_logging(result=result) + result = redact_message_input_output_from_logging( + result=result, litellm_logging_obj=self + ) for callback in callbacks: try: if callback == "lite_debugger": @@ -2728,41 +2739,6 @@ class Logging: f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while success logging {traceback.format_exc()}" ) - def redact_message_input_output_from_logging(self, result): - """ - Removes messages, prompts, input, response from logging. This modifies the data in-place - only redacts when litellm.turn_off_message_logging == True - """ - # check if user opted out of logging message/response to callbacks - if litellm.turn_off_message_logging is True: - # remove messages, prompts, input, response from logging - self.model_call_details["messages"] = [ - {"role": "user", "content": "redacted-by-litellm"} - ] - self.model_call_details["prompt"] = "" - self.model_call_details["input"] = "" - - # response cleaning - # ChatCompletion Responses - if self.stream and "complete_streaming_response" in self.model_call_details: - _streaming_response = self.model_call_details[ - "complete_streaming_response" - ] - for choice in _streaming_response.choices: - if isinstance(choice, litellm.Choices): - choice.message.content = "redacted-by-litellm" - elif isinstance(choice, litellm.utils.StreamingChoices): - choice.delta.content = "redacted-by-litellm" - else: - if result is not None: - if isinstance(result, litellm.ModelResponse): - if hasattr(result, "choices") and result.choices is not None: - for choice in result.choices: - if isinstance(choice, litellm.Choices): - choice.message.content = "redacted-by-litellm" - elif isinstance(choice, litellm.utils.StreamingChoices): - choice.delta.content = "redacted-by-litellm" - def exception_logging( additional_args={}, From ec110976b89b2cd66ebf9a8b5dd209495fa0f490 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 13 Jun 2024 11:53:04 -0700 Subject: [PATCH 2/4] fix config --- litellm/proxy/proxy_config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index a21378f31e4..d28f383c090 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -24,6 +24,7 @@ general_settings: litellm_settings: callbacks: ["otel"] store_audit_logs: true + turn_off_message_logging: true redact_messages_in_exceptions: True enforced_params: - user From b154a4a8add147f1157c13ccde7d7eef8f2f6a68 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 13 Jun 2024 11:59:18 -0700 Subject: [PATCH 3/4] fix - redacting messages --- litellm/litellm_core_utils/redact_messages.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index a93104e5dee..b76cd4a9e4f 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -61,3 +61,5 @@ def redact_message_input_output_from_logging( choice.message.content = "redacted-by-litellm" elif isinstance(choice, litellm.utils.StreamingChoices): choice.delta.content = "redacted-by-litellm" + + return _result From d274cfeb3f62a96557a4f078759f087ce3da02a0 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 13 Jun 2024 13:49:23 -0700 Subject: [PATCH 4/4] test test_redact_msgs_from_logs --- litellm/litellm_core_utils/redact_messages.py | 2 +- litellm/tests/test_utils.py | 50 +++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index b76cd4a9e4f..d6dcb489f5f 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -41,7 +41,7 @@ def redact_message_input_output_from_logging( # response cleaning # ChatCompletion Responses if ( - litellm_logging_obj.stream + litellm_logging_obj.stream is True and "complete_streaming_response" in litellm_logging_obj.model_call_details ): _streaming_response = litellm_logging_obj.model_call_details[ diff --git a/litellm/tests/test_utils.py b/litellm/tests/test_utils.py index 491283eeda8..2e32e32df7c 100644 --- a/litellm/tests/test_utils.py +++ b/litellm/tests/test_utils.py @@ -3,6 +3,7 @@ from unittest import mock from dotenv import load_dotenv import copy +from datetime import datetime load_dotenv() import os @@ -395,3 +396,52 @@ def test_get_supported_openai_params() -> None: # Unmapped provider assert get_supported_openai_params("nonexistent") is None + + +def test_redact_msgs_from_logs(): + """ + Tests that turn_off_message_logging does not modify the response_obj + + On the proxy some users were seeing the redaction impact client side responses + """ + from litellm.litellm_core_utils.redact_messages import ( + redact_message_input_output_from_logging, + ) + from litellm.utils import Logging + + litellm.turn_off_message_logging = True + + response_obj = litellm.ModelResponse( + choices=[ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "I'm LLaMA, an AI assistant developed by Meta AI that can understand and respond to human input in a conversational manner.", + "role": "assistant", + }, + } + ] + ) + + _redacted_response_obj = redact_message_input_output_from_logging( + result=response_obj, + litellm_logging_obj=Logging( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="acompletion", + litellm_call_id="1234", + start_time=datetime.now(), + function_id="1234", + ), + ) + + # Assert the response_obj content is NOT modified + assert ( + response_obj.choices[0].message.content + == "I'm LLaMA, an AI assistant developed by Meta AI that can understand and respond to human input in a conversational manner." + ) + + litellm.turn_off_message_logging = False + print("Test passed")