perf(logging): keep huge plain-text payloads on the worker thread

litellm/constants.py

@@ -173,6 +173,7 @@ MID_STREAM_FAILURE_LOG_WAIT_SECONDS: Final = 1.0
 MAX_BASE64_LENGTH_FOR_LOGGING: Final = get_env_int("MAX_BASE64_LENGTH_FOR_LOGGING", 64)
 BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS: Final = 256 * 1024
 BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS: Final = 256
+BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS: Final = 8 * 1024 * 1024
 REDACTED_BY_LITELLM: Final = "redacted-by-litellm"
 # in-memory stand-in handed to provider converters for redacted arguments; never stored
 REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER: Final = "{}"

litellm/litellm_core_utils/logging_utils.py

@@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Any, Final, TypeAlias, cast
 
 from litellm._logging import format_base64_size, verbose_logger
 from litellm.constants import (
+    BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS,
     BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS,
     BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS,
     MAX_BASE64_LENGTH_FOR_LOGGING,
@@ -179,15 +180,17 @@ async def truncate_base64_in_messages_async(
     Same result as truncate_base64_in_messages, but payloads whose string content
     reaches BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS are copied in a worker thread,
     so neither the regex pass over multi-MB base64 images nor the copy of a long
-    history blocks the event loop. A few large strings with no data URI are copied
-    inline, since that copy costs less than the thread hop.
+    history blocks the event loop. A few strings with no data URI, under
+    BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS in total, are copied inline, since
+    checking them for data URIs and copying them costs less than the thread hop.
     """
     if messages is None or MAX_BASE64_LENGTH_FOR_LOGGING <= 0:
         return messages
     leaves: Final = tuple(_iter_string_leaves(messages))
-    if sum(len(leaf) for leaf in leaves) < BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS:
+    total_chars: Final = sum(len(leaf) for leaf in leaves)
+    if total_chars < BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS:
         return truncate_base64_in_messages(messages)
-    if _is_few_plain_strings(leaves):
+    if total_chars < BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS and _is_few_plain_strings(leaves):
         return _copy_messages(messages, lambda text: text)
     return await asyncio.to_thread(truncate_base64_in_messages, messages)
 

tests/unit/litellm_core_utils/test_logging_utils.py

@@ -376,6 +376,22 @@ class TestTruncateBase64InMessagesAsync:
         assert offload_threads
         assert threading.get_ident() not in offload_threads
 
+    @pytest.mark.asyncio
+    async def test_plain_text_over_the_inline_limit_is_copied_off_the_event_loop(
+        self, monkeypatch: pytest.MonkeyPatch, scan_threads: Sequence[int]
+    ):
+        monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS", 1_000)
+        monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS", 10_000)
+        messages: Final = _tool_result_messages('data: {"choices": [{"delta": {"content": "a; b"}}]}\n\n' * 400)
+
+        result: Final = await truncate_base64_in_messages_async(messages)
+        offload_threads: Final = tuple(scan_threads)
+
+        assert result == messages
+        assert result is not messages
+        assert offload_threads
+        assert threading.get_ident() not in offload_threads
+
     @pytest.mark.asyncio
     async def test_none_and_disabled_truncation_short_circuit(self, monkeypatch, scan_threads):
         assert await truncate_base64_in_messages_async(None) is None