diff --git a/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py index dcbc05b5379..42ddeac5ebc 100644 --- a/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py @@ -345,6 +345,9 @@ class _CombinedChunkSplitter: # Fewer than two distinct calls, or a fragment with no call to attach to: leave it alone. if len(groups) < 2 or any(first is None for first in opened): return (chunk,) + # An opening chunk over the bound is not held at all. + if sum(len(self._call_arguments(c)) for c in calls) > _MAX_PENDING_ARGUMENT_CHARS: + return (chunk,) if all(self._is_complete_json("".join(self._call_arguments(c) for c in group)) for group in groups): return tuple(self._chunk_with_calls(chunk, group, keep_finish=True) for group in groups) self._pending = { diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_combined_chunk.py b/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_combined_chunk.py index afae0472875..9aae5fb70ab 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_combined_chunk.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_combined_chunk.py @@ -424,3 +424,31 @@ def test_two_entries_for_the_same_index_stay_one_call(): ] assert [(name, args) for name, _, args in _tool_blocks(chunks)] == [("Read", {"file_path": "a.md"})] + + +def test_an_opening_chunk_over_the_bound_is_not_held(monkeypatch): + from litellm.llms.anthropic.pass_through.adapters import streaming_iterator + + monkeypatch.setattr(streaming_iterator, "_MAX_PENDING_ARGUMENT_CHARS", 10) + splitter = _CombinedChunkSplitter(iter(())) + oversized = _tool_chunk( + [_tool_call(0, "call_a", "Read", '{"file_path": "' + "a" * 50), _tool_call(1, "call_b", "Glob", "")] + ) + + assert splitter._expand(oversized) == (oversized,) + assert not splitter._pending + + +def test_fragments_that_push_held_calls_over_the_bound_flush_them(monkeypatch): + from litellm.llms.anthropic.pass_through.adapters import streaming_iterator + + monkeypatch.setattr(streaming_iterator, "_MAX_PENDING_ARGUMENT_CHARS", 10) + splitter = _CombinedChunkSplitter(iter(())) + opening = _tool_chunk([_tool_call(0, "call_a", "Read", ""), _tool_call(1, "call_b", "Glob", "")]) + fragment = _tool_chunk([_tool_call(0, None, None, '{"file_path": "' + "a" * 50)]) + + assert splitter._expand(opening) == () + flushed = splitter._expand(fragment) + + assert [c.choices[0].delta.tool_calls[0].function.name for c in flushed] == ["Read", "Glob"] + assert not splitter._pending