fix(logging): datadog truncation no longer rewrites the shared standard logging payload

litellm/integrations/custom_logger.py

@@ -822,46 +822,37 @@ class CustomLogger:  # https://docs.litellm.ai/docs/observability/custom_callbac
     def truncate_standard_logging_payload_content(
         self,
         standard_logging_object: StandardLoggingPayload,
-    ):
+    ) -> StandardLoggingPayload:
         """
-        Truncate error strings and message content in logging payload
+        Return a copy of the logging payload with error_str, messages, and response truncated
 
         Some loggers like DataDog/ GCS Bucket have a limit on the size of the payload. (1MB)
 
-        This function truncates the error string and the message content if they exceed a certain length.
+        Every callback of a request shares one standard logging object, so the payload passed in is left
+        untouched and the callbacks that run later (the prompt caching router check, spend logs) still see
+        the original fields.
         """
-        MAX_STR_LENGTH: Final = 10_000
+        max_str_length: Final = 10_000
+        error_str, messages, response = (
+            self._truncate_field(field_value=standard_logging_object.get(field), max_length=max_str_length)
+            for field in ("error_str", "messages", "response")
+        )
+        return {
+            **standard_logging_object,
+            "error_str": standard_logging_object["error_str"] if error_str is None else error_str,
+            "messages": standard_logging_object["messages"] if messages is None else messages,
+            "response": standard_logging_object["response"] if response is None else response,
+        }
 
-        # Truncate fields that might exceed max length
-        fields_to_truncate: Final = ["error_str", "messages", "response"]
-        for field in fields_to_truncate:
-            self._truncate_field(
-                standard_logging_object=standard_logging_object,
-                field_name=field,
-                max_length=MAX_STR_LENGTH,
-            )
-
-    def _truncate_field(
-        self,
-        standard_logging_object: StandardLoggingPayload,
-        field_name: str,
-        max_length: int,
-    ) -> None:
+    def _truncate_field(self, field_value: object, max_length: int) -> str | None:
         """
-        Helper function to truncate a field in the logging payload
+        Return the truncated text of a field that exceeds max_length, or None when the field fits
 
-        This converts the field to a string and then truncates it if it exceeds the max length.
-
-        Why convert to string ?
-        1. User was sending a poorly formatted list for `messages` field, we could not predict where they would send content
-            - Converting to string and then truncating the logged content catches this
-        2. We want to avoid modifying the original `messages`, `response`, and `error_str` in the logging payload since these are in kwargs and could be returned to the user
+        The field is measured as a string because users send poorly formatted lists for `messages`, so there is
+        no fixed place the content would be.
         """
-        field_value: Final[object] = standard_logging_object.get(field_name)
-        if field_value:
-            str_value: Final = str(field_value)
-            if len(str_value) > max_length:
-                standard_logging_object[field_name] = self._truncate_text(text=str_value, max_length=max_length)
+        text: Final = str(field_value or "")
+        return self._truncate_text(text=text, max_length=max_length) if len(text) > max_length else None
 
     def _truncate_text(self, text: str, max_length: int) -> str:
         """Truncate text if it exceeds max_length"""

litellm/integrations/datadog/datadog.py

@@ -563,11 +563,10 @@ class DataDogLogger(
         if standard_logging_object.get("status") == "failure":
             status = DataDogStatus.ERROR
 
-        # Build the initial payload
-        self.truncate_standard_logging_payload_content(standard_logging_object)
+        truncated_payload: Final = self.truncate_standard_logging_payload_content(standard_logging_object)
 
         dd_payload: Final = self._create_datadog_logging_payload_helper(
-            standard_logging_object=standard_logging_object,
+            standard_logging_object=truncated_payload,
             status=status,
         )
         return dd_payload

tests/logging_callback_tests/test_datadog.py

@@ -578,6 +578,32 @@ async def test_datadog_payload_content_truncation():
     ), "response not truncated correctly"
 
 
+@pytest.mark.asyncio
+async def test_datadog_payload_truncation_leaves_shared_payload_intact(monkeypatch):
+    """
+    Every callback of a request shares one standard logging object, so the datadog truncation
+    must not turn its messages into a string for the callbacks that run after it (the prompt
+    caching router check reads `messages` as a list to pin the deployment holding the cache)
+    """
+    monkeypatch.setenv("DD_SITE", "https://fake.datadoghq.com")
+    monkeypatch.setenv("DD_API_KEY", "anything")
+    dd_logger = DataDogLogger()
+    standard_payload = create_standard_logging_payload()
+    original_messages = [{"role": "user", "content": "x" * 80_000}]
+    standard_payload["messages"] = original_messages
+    kwargs = {"standard_logging_object": standard_payload}
+
+    dd_payload = dd_logger.create_datadog_logging_payload(
+        kwargs=kwargs,
+        response_obj=None,
+        start_time=datetime.now(),
+        end_time=datetime.now(),
+    )
+
+    assert kwargs["standard_logging_object"]["messages"] is original_messages
+    assert len(json.loads(dd_payload["message"])["messages"]) < 10_100
+
+
 def test_datadog_static_methods():
     """Test the static helper methods in DataDogLogger class"""
 

tests/logging_callback_tests/test_standard_logging_payload.py

@@ -607,42 +607,31 @@ def testget_standard_logging_payload_session_id_empty_when_flag_off(monkeypatch)
 
 def test_truncate_standard_logging_payload():
     """
-    1. original messages, response, and error_str should NOT BE MODIFIED, since these are from kwargs
-    2. the `messages`, `response`, and `error_str` in new standard_logging_payload should be truncated
+    1. the payload passed in is never modified, since every callback of the request shares it
+    2. the `messages`, `response`, and `error_str` in the returned payload are truncated
     """
     _custom_logger = CustomLogger()
     standard_logging_payload: StandardLoggingPayload = (
         create_standard_logging_payload_with_long_content()
     )
     original_messages = standard_logging_payload["messages"]
-    len_original_messages = len(str(original_messages))
     original_response = standard_logging_payload["response"]
-    len_original_response = len(str(original_response))
     original_error_str = standard_logging_payload["error_str"]
-    len_original_error_str = len(str(original_error_str))
 
-    _custom_logger.truncate_standard_logging_payload_content(standard_logging_payload)
-
-    # Original messages, response, and error_str should NOT BE MODIFIED
-    assert standard_logging_payload["messages"] != original_messages
-    assert standard_logging_payload["response"] != original_response
-    assert standard_logging_payload["error_str"] != original_error_str
-    assert len_original_messages == len(str(original_messages))
-    assert len_original_response == len(str(original_response))
-    assert len_original_error_str == len(str(original_error_str))
-
-    print(
-        "logged standard_logging_payload",
-        json.dumps(standard_logging_payload, indent=2),
+    truncated = _custom_logger.truncate_standard_logging_payload_content(
+        standard_logging_payload
     )
 
-    # Logged messages, response, and error_str should be truncated
-    # assert len of messages is less than 10_500
-    assert len(str(standard_logging_payload["messages"])) < 10_500
-    # assert len of response is less than 10_500
-    assert len(str(standard_logging_payload["response"])) < 10_500
-    # assert len of error_str is less than 10_500
-    assert len(str(standard_logging_payload["error_str"])) < 10_500
+    assert standard_logging_payload["messages"] is original_messages
+    assert standard_logging_payload["response"] is original_response
+    assert standard_logging_payload["error_str"] is original_error_str
+
+    assert truncated["messages"] != original_messages
+    assert truncated["response"] != original_response
+    assert truncated["error_str"] != original_error_str
+    assert len(str(truncated["messages"])) < 10_500
+    assert len(str(truncated["response"])) < 10_500
+    assert len(str(truncated["error_str"])) < 10_500
 
 
 def test_strip_trailing_slash():