diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 62ca6b0254e..e164f042818 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -822,46 +822,37 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac def truncate_standard_logging_payload_content( self, standard_logging_object: StandardLoggingPayload, - ): + ) -> StandardLoggingPayload: """ - Truncate error strings and message content in logging payload + Return a copy of the logging payload with error_str, messages, and response truncated Some loggers like DataDog/ GCS Bucket have a limit on the size of the payload. (1MB) - This function truncates the error string and the message content if they exceed a certain length. + Every callback of a request shares one standard logging object, so the payload passed in is left + untouched and the callbacks that run later (the prompt caching router check, spend logs) still see + the original fields. """ - MAX_STR_LENGTH: Final = 10_000 + max_str_length: Final = 10_000 + error_str, messages, response = ( + self._truncate_field(field_value=standard_logging_object.get(field), max_length=max_str_length) + for field in ("error_str", "messages", "response") + ) + return { + **standard_logging_object, + "error_str": standard_logging_object["error_str"] if error_str is None else error_str, + "messages": standard_logging_object["messages"] if messages is None else messages, + "response": standard_logging_object["response"] if response is None else response, + } - # Truncate fields that might exceed max length - fields_to_truncate: Final = ["error_str", "messages", "response"] - for field in fields_to_truncate: - self._truncate_field( - standard_logging_object=standard_logging_object, - field_name=field, - max_length=MAX_STR_LENGTH, - ) - - def _truncate_field( - self, - standard_logging_object: StandardLoggingPayload, - field_name: str, - max_length: int, - ) -> None: + def _truncate_field(self, field_value: object, max_length: int) -> str | None: """ - Helper function to truncate a field in the logging payload + Return the truncated text of a field that exceeds max_length, or None when the field fits - This converts the field to a string and then truncates it if it exceeds the max length. - - Why convert to string ? - 1. User was sending a poorly formatted list for `messages` field, we could not predict where they would send content - - Converting to string and then truncating the logged content catches this - 2. We want to avoid modifying the original `messages`, `response`, and `error_str` in the logging payload since these are in kwargs and could be returned to the user + The field is measured as a string because users send poorly formatted lists for `messages`, so there is + no fixed place the content would be. """ - field_value: Final[object] = standard_logging_object.get(field_name) - if field_value: - str_value: Final = str(field_value) - if len(str_value) > max_length: - standard_logging_object[field_name] = self._truncate_text(text=str_value, max_length=max_length) + text: Final = str(field_value or "") + return self._truncate_text(text=text, max_length=max_length) if len(text) > max_length else None def _truncate_text(self, text: str, max_length: int) -> str: """Truncate text if it exceeds max_length""" diff --git a/litellm/integrations/datadog/datadog.py b/litellm/integrations/datadog/datadog.py index 77b12d1e3fa..2ca8b0ed236 100644 --- a/litellm/integrations/datadog/datadog.py +++ b/litellm/integrations/datadog/datadog.py @@ -563,11 +563,10 @@ class DataDogLogger( if standard_logging_object.get("status") == "failure": status = DataDogStatus.ERROR - # Build the initial payload - self.truncate_standard_logging_payload_content(standard_logging_object) + truncated_payload: Final = self.truncate_standard_logging_payload_content(standard_logging_object) dd_payload: Final = self._create_datadog_logging_payload_helper( - standard_logging_object=standard_logging_object, + standard_logging_object=truncated_payload, status=status, ) return dd_payload diff --git a/tests/logging_callback_tests/test_datadog.py b/tests/logging_callback_tests/test_datadog.py index 7ac9ac0b5ad..0ce20ce6ace 100644 --- a/tests/logging_callback_tests/test_datadog.py +++ b/tests/logging_callback_tests/test_datadog.py @@ -578,6 +578,32 @@ async def test_datadog_payload_content_truncation(): ), "response not truncated correctly" +@pytest.mark.asyncio +async def test_datadog_payload_truncation_leaves_shared_payload_intact(monkeypatch): + """ + Every callback of a request shares one standard logging object, so the datadog truncation + must not turn its messages into a string for the callbacks that run after it (the prompt + caching router check reads `messages` as a list to pin the deployment holding the cache) + """ + monkeypatch.setenv("DD_SITE", "https://fake.datadoghq.com") + monkeypatch.setenv("DD_API_KEY", "anything") + dd_logger = DataDogLogger() + standard_payload = create_standard_logging_payload() + original_messages = [{"role": "user", "content": "x" * 80_000}] + standard_payload["messages"] = original_messages + kwargs = {"standard_logging_object": standard_payload} + + dd_payload = dd_logger.create_datadog_logging_payload( + kwargs=kwargs, + response_obj=None, + start_time=datetime.now(), + end_time=datetime.now(), + ) + + assert kwargs["standard_logging_object"]["messages"] is original_messages + assert len(json.loads(dd_payload["message"])["messages"]) < 10_100 + + def test_datadog_static_methods(): """Test the static helper methods in DataDogLogger class""" diff --git a/tests/logging_callback_tests/test_standard_logging_payload.py b/tests/logging_callback_tests/test_standard_logging_payload.py index da1fbbaa04f..c125bd53904 100644 --- a/tests/logging_callback_tests/test_standard_logging_payload.py +++ b/tests/logging_callback_tests/test_standard_logging_payload.py @@ -607,42 +607,31 @@ def testget_standard_logging_payload_session_id_empty_when_flag_off(monkeypatch) def test_truncate_standard_logging_payload(): """ - 1. original messages, response, and error_str should NOT BE MODIFIED, since these are from kwargs - 2. the `messages`, `response`, and `error_str` in new standard_logging_payload should be truncated + 1. the payload passed in is never modified, since every callback of the request shares it + 2. the `messages`, `response`, and `error_str` in the returned payload are truncated """ _custom_logger = CustomLogger() standard_logging_payload: StandardLoggingPayload = ( create_standard_logging_payload_with_long_content() ) original_messages = standard_logging_payload["messages"] - len_original_messages = len(str(original_messages)) original_response = standard_logging_payload["response"] - len_original_response = len(str(original_response)) original_error_str = standard_logging_payload["error_str"] - len_original_error_str = len(str(original_error_str)) - _custom_logger.truncate_standard_logging_payload_content(standard_logging_payload) - - # Original messages, response, and error_str should NOT BE MODIFIED - assert standard_logging_payload["messages"] != original_messages - assert standard_logging_payload["response"] != original_response - assert standard_logging_payload["error_str"] != original_error_str - assert len_original_messages == len(str(original_messages)) - assert len_original_response == len(str(original_response)) - assert len_original_error_str == len(str(original_error_str)) - - print( - "logged standard_logging_payload", - json.dumps(standard_logging_payload, indent=2), + truncated = _custom_logger.truncate_standard_logging_payload_content( + standard_logging_payload ) - # Logged messages, response, and error_str should be truncated - # assert len of messages is less than 10_500 - assert len(str(standard_logging_payload["messages"])) < 10_500 - # assert len of response is less than 10_500 - assert len(str(standard_logging_payload["response"])) < 10_500 - # assert len of error_str is less than 10_500 - assert len(str(standard_logging_payload["error_str"])) < 10_500 + assert standard_logging_payload["messages"] is original_messages + assert standard_logging_payload["response"] is original_response + assert standard_logging_payload["error_str"] is original_error_str + + assert truncated["messages"] != original_messages + assert truncated["response"] != original_response + assert truncated["error_str"] != original_error_str + assert len(str(truncated["messages"])) < 10_500 + assert len(str(truncated["response"])) < 10_500 + assert len(str(truncated["error_str"])) < 10_500 def test_strip_trailing_slash():