scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-10 22:41:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 82

fix(adapter): preserve tool_use arguments in /v1/messages streaming block transitions

Browse source 
When _should_start_new_content_block() triggers a transition (e.g. text → tool_use),
the adapter queues content_block_stop + content_block_start but discards the
processed_chunk. For providers like Gemini that send tool arguments in the same
streaming chunk as the function name, the input_json_delta is silently lost and
every tool_use arrives with empty input: {}.

Queue the processed delta alongside the content_block_start whenever it carries
non-empty content (partial_json, text, or thinking). Empty deltas — such as the
first chunk of OpenAI-style split tool calls — are still skipped.

Fixes #25321
This commit is contained in:
lawrence3699 2026-04-09 19:14:13 +10:00
parent a59dbc5908
commit f6e393811b
No known key found for this signature in database
2 changed files with 167 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py
View file

@ -129,8 +129,6 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
if should_start_new_block and not self.sent_content_block_finish:
# Queue the sequence: content_block_stop -> content_block_start
# The trigger chunk itself is not emitted as a delta since the
# content_block_start already carries the relevant information.
self.chunk_queue.append(
{
"type": "content_block_stop",
@ -144,6 +142,13 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
"content_block": self.current_content_block_start,
}
)
# Also queue the delta if it carries content — some providers
# (e.g. Gemini) send tool arguments in the same chunk as the
# function name, so dropping the delta loses those arguments.
if processed_chunk.get("type") == "content_block_delta":
delta = processed_chunk.get("delta", {})
if delta.get("partial_json") or delta.get("text") or delta.get("thinking"):
self.chunk_queue.append(processed_chunk)
self.sent_content_block_finish = False
return self.chunk_queue.popleft()
@ -305,8 +310,6 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
if not self.queued_usage_chunk:
if should_start_new_block and not self.sent_content_block_finish:
# Queue the sequence: content_block_stop -> content_block_start
# The trigger chunk itself is not emitted as a delta since the
# content_block_start already carries the relevant information.
# 1. Stop current content block
self.chunk_queue.append(
@ -325,6 +328,15 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
}
)
# 3. Queue the delta if it carries content — some providers
# (e.g. Gemini) send tool arguments in the same chunk as
# the function name, so dropping the delta loses those
# arguments.
if processed_chunk.get("type") == "content_block_delta":
delta = processed_chunk.get("delta", {})
if delta.get("partial_json") or delta.get("text") or delta.get("thinking"):
self.chunk_queue.append(processed_chunk)
# Reset state for new block
self.sent_content_block_finish = False

153
tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_parallel_tool_calls.py
View file

@ -2,6 +2,7 @@ import os
import sys
from typing import List
import pytest
sys.path.insert(0, os.path.abspath("../../../../.."))
@ -278,7 +279,8 @@ def test_anthropic_stream_wrapper_interleaved_tool_calls_and_text():
"content_block_delta", # {"city":
"content_block_delta", # "NY"}
"content_block_stop", # End of first tool_use content block
"content_block_start", # "The weather is nice today"
"content_block_start", # Text block start
"content_block_delta", # "The weather is nice today."
"content_block_stop",
"content_block_start", # Start of second tool_use content block
"content_block_delta", # {"city":
@ -288,7 +290,8 @@ def test_anthropic_stream_wrapper_interleaved_tool_calls_and_text():
"content_block_delta", # {"city":
"content_block_delta", # " CHI"}
"content_block_stop", # End of third tool_use content block
"content_block_start", # "The weather is not so nice today"
"content_block_start", # Text block start
"content_block_delta", # "The weather is not so nice today."
"content_block_stop",
"message_delta", # Stop reason with merged usage
"message_stop", # Final message stop
@ -307,3 +310,149 @@ def test_anthropic_stream_wrapper_interleaved_tool_calls_and_text():
get_weather_calls += 1
assert get_weather_calls == 3
def test_anthropic_stream_wrapper_tool_args_in_first_chunk():
"""
Regression test for #25321: non-OpenAI models (e.g. Gemini via litellm)
may send the tool arguments in the same streaming chunk as the function
name. The adapter must not discard those arguments when transitioning
to a new content block.
"""
responses = [
# Single chunk containing both function name AND arguments
ModelResponseStream(
choices=[
StreamingChoices(
delta=Delta(
tool_calls=[
ChatCompletionDeltaToolCall(
id="call_abc123",
function=Function(
arguments='{"command": "ls -la"}',
name="Bash",
),
index=0,
type="function",
)
]
),
index=0,
finish_reason=None,
)
],
),
ModelResponseStream(
choices=[
StreamingChoices(
delta=Delta(content="", stop_reason="tool_calls"),
index=0,
finish_reason="stop",
)
],
usage=Usage(prompt_tokens=50, completion_tokens=20, total_tokens=70),
),
]
wrapper = AnthropicStreamWrapper(
completion_stream=MockCompletionStream(responses),
model="gemini-2.5-flash",
)
chunks = []
chunk_types = []
for chunk in wrapper:
chunks.append(chunk)
chunk_types.append(chunk.get("type"))
expected_types = [
"message_start",
"content_block_start", # Initial empty text block
"content_block_stop",
"content_block_start", # tool_use block
"content_block_delta", # Tool arguments from the transition chunk
"content_block_stop",
"message_delta",
"message_stop",
]
assert expected_types == chunk_types
# Verify the tool arguments are preserved (not dropped)
tool_deltas = [
c for c in chunks if c.get("type") == "content_block_delta"
]
assert len(tool_deltas) == 1
assert tool_deltas[0]["delta"]["type"] == "input_json_delta"
assert tool_deltas[0]["delta"]["partial_json"] == '{"command": "ls -la"}'
@pytest.mark.asyncio
async def test_async_anthropic_stream_wrapper_tool_args_in_first_chunk():
"""Async version of the regression test for #25321."""
responses = [
ModelResponseStream(
choices=[
StreamingChoices(
delta=Delta(
tool_calls=[
ChatCompletionDeltaToolCall(
id="call_abc123",
function=Function(
arguments='{"command": "ls -la"}',
name="Bash",
),
index=0,
type="function",
)
]
),
index=0,
finish_reason=None,
)
],
),
ModelResponseStream(
choices=[
StreamingChoices(
delta=Delta(content="", stop_reason="tool_calls"),
index=0,
finish_reason="stop",
)
],
usage=Usage(prompt_tokens=50, completion_tokens=20, total_tokens=70),
),
]
wrapper = AnthropicStreamWrapper(
completion_stream=MockCompletionStream(responses),
model="gemini-2.5-flash",
)
chunks = []
chunk_types = []
async for chunk in wrapper:
chunks.append(chunk)
chunk_types.append(chunk.get("type"))
expected_types = [
"message_start",
"content_block_start", # Initial empty text block
"content_block_stop",
"content_block_start", # tool_use block
"content_block_delta", # Tool arguments from the transition chunk
"content_block_stop",
"message_delta",
"message_stop",
]
assert expected_types == chunk_types
tool_deltas = [
c for c in chunks if c.get("type") == "content_block_delta"
]
assert len(tool_deltas) == 1
assert tool_deltas[0]["delta"]["type"] == "input_json_delta"
assert tool_deltas[0]["delta"]["partial_json"] == '{"command": "ls -la"}'