scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-03 07:23:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 166

perf(logging): keep main's copy and offload, skip the regex for strings without data URI markers

Browse source
This commit is contained in:
mateo-berri 2026-10-30 11:37:16 -07:00
parent d5cae91fdc
commit d6471f84ce
2 changed files with 11 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
litellm/litellm_core_utils/logging_utils.py
View file

@ -161,16 +161,14 @@ async def truncate_base64_in_messages_async(
messages: str | list | dict | None, # mutable-ok: same contract as truncate_base64_in_messages
) -> str | list | dict | None: # mutable-ok: same contract as truncate_base64_in_messages
"""
Same result as truncate_base64_in_messages, but payloads with a data URI whose
string content reaches BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS are scanned in a
worker thread so the regex pass over multi-MB base64 images does not block the event loop.
Same result as truncate_base64_in_messages, but payloads whose string content
reaches BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS are scanned in a worker
thread so the regex pass over multi-MB base64 images does not block the event loop.
"""
if messages is None or MAX_BASE64_LENGTH_FOR_LOGGING <= 0:
return messages
leaves: Final = tuple(_iter_string_leaves(messages))
if sum(len(leaf) for leaf in leaves) < BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS or not any(
_may_contain_base64_data_uri(leaf) for leaf in leaves
):
total_chars: Final = sum(len(leaf) for leaf in _iter_string_leaves(messages))
if total_chars < BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS:
return truncate_base64_in_messages(messages)
return await asyncio.to_thread(truncate_base64_in_messages, messages)

17
tests/unit/litellm_core_utils/test_logging_utils.py
View file

@ -323,16 +323,11 @@ class TestTruncateBase64InMessagesAsync:
],
ids=["json_rows", "sse_lines", "code_with_semicolons", "sse_lines_with_semicolons", "base64_marker_without_data"],
)
@pytest.mark.parametrize(
"truncate",
[lambda messages: _run_without_yielding(truncate_base64_in_messages_async(messages)), truncate_base64_in_messages],
ids=["async", "sync"],
)
def test_tool_message_without_data_uri_is_copied_inline_without_a_regex_scan(self, row, truncate):
def test_tool_message_without_data_uri_is_copied_without_a_regex_scan(self, row):
tool_output = (row * (_PLAIN_TOOL_OUTPUT_CHARS // len(row) + 1))[:_PLAIN_TOOL_OUTPUT_CHARS]
messages = _tool_result_messages(tool_output)
result, regex_calls = _regex_calls_while(lambda: truncate(messages))
result, regex_calls = _regex_calls_while(lambda: truncate_base64_in_messages(messages))
assert result == messages
assert result is not messages
@ -351,15 +346,15 @@ class TestTruncateBase64InMessagesAsync:
assert "base64_data truncated" in result[0]["content"][1]["image_url"]["url"]
@pytest.mark.asyncio
async def test_small_data_uri_beside_large_text_is_scanned_off_the_event_loop(self, monkeypatch, scan_threads):
async def test_large_plain_text_is_copied_off_the_event_loop(self, monkeypatch, scan_threads):
monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS", 1_000)
payload = "M" * 200
messages = _image_messages(payload) + _tool_result_messages("plain tool output\n" * 100)
messages = _tool_result_messages("plain tool output\n" * 100)
result = await truncate_base64_in_messages_async(messages)
offload_threads = tuple(scan_threads)
assert payload not in result[0]["content"][1]["image_url"]["url"]
assert result == messages
assert result is not messages
assert offload_threads
assert threading.get_ident() not in offload_threads