diff --git a/litellm/constants.py b/litellm/constants.py index 6caf6e5a1f6..6f766965921 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -173,6 +173,7 @@ MID_STREAM_FAILURE_LOG_WAIT_SECONDS: Final = 1.0 MAX_BASE64_LENGTH_FOR_LOGGING: Final = get_env_int("MAX_BASE64_LENGTH_FOR_LOGGING", 64) BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS: Final = 256 * 1024 BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS: Final = 256 +BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS: Final = 8 * 1024 * 1024 REDACTED_BY_LITELLM: Final = "redacted-by-litellm" # in-memory stand-in handed to provider converters for redacted arguments; never stored REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER: Final = "{}" diff --git a/litellm/litellm_core_utils/logging_utils.py b/litellm/litellm_core_utils/logging_utils.py index c74b9bc7b73..88d84201beb 100644 --- a/litellm/litellm_core_utils/logging_utils.py +++ b/litellm/litellm_core_utils/logging_utils.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Any, Final, TypeAlias, cast from litellm._logging import format_base64_size, verbose_logger from litellm.constants import ( + BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS, BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS, BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS, MAX_BASE64_LENGTH_FOR_LOGGING, @@ -179,15 +180,17 @@ async def truncate_base64_in_messages_async( Same result as truncate_base64_in_messages, but payloads whose string content reaches BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS are copied in a worker thread, so neither the regex pass over multi-MB base64 images nor the copy of a long - history blocks the event loop. A few large strings with no data URI are copied - inline, since that copy costs less than the thread hop. + history blocks the event loop. A few strings with no data URI, under + BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS in total, are copied inline, since + checking them for data URIs and copying them costs less than the thread hop. """ if messages is None or MAX_BASE64_LENGTH_FOR_LOGGING <= 0: return messages leaves: Final = tuple(_iter_string_leaves(messages)) - if sum(len(leaf) for leaf in leaves) < BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS: + total_chars: Final = sum(len(leaf) for leaf in leaves) + if total_chars < BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS: return truncate_base64_in_messages(messages) - if _is_few_plain_strings(leaves): + if total_chars < BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS and _is_few_plain_strings(leaves): return _copy_messages(messages, lambda text: text) return await asyncio.to_thread(truncate_base64_in_messages, messages) diff --git a/tests/unit/litellm_core_utils/test_logging_utils.py b/tests/unit/litellm_core_utils/test_logging_utils.py index 83c1f6e2f6c..a367c02fd6f 100644 --- a/tests/unit/litellm_core_utils/test_logging_utils.py +++ b/tests/unit/litellm_core_utils/test_logging_utils.py @@ -376,6 +376,22 @@ class TestTruncateBase64InMessagesAsync: assert offload_threads assert threading.get_ident() not in offload_threads + @pytest.mark.asyncio + async def test_plain_text_over_the_inline_limit_is_copied_off_the_event_loop( + self, monkeypatch: pytest.MonkeyPatch, scan_threads: Sequence[int] + ): + monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS", 1_000) + monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS", 10_000) + messages: Final = _tool_result_messages('data: {"choices": [{"delta": {"content": "a; b"}}]}\n\n' * 400) + + result: Final = await truncate_base64_in_messages_async(messages) + offload_threads: Final = tuple(scan_threads) + + assert result == messages + assert result is not messages + assert offload_threads + assert threading.get_ident() not in offload_threads + @pytest.mark.asyncio async def test_none_and_disabled_truncation_short_circuit(self, monkeypatch, scan_threads): assert await truncate_base64_in_messages_async(None) is None