scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-22 00:31:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 96

fix(anthropic pass-through): emit trigger chunk delta during content block transitions

Browse source 
When streaming switches content block types (e.g. thinking→text), the
first chunk of the new type (the trigger chunk) was being dropped. Only
content_block_stop and content_block_start were queued; processed_chunk
was never emitted.

This caused two symptoms with Bedrock Converse + interleaved thinking:
1. Text responses were truncated (e.g. 'Hello from LiteLLM!' lost,
   only a trailing '!' from a later chunk survived)
2. Binary 'Content block is not a text block' errors on retry

Fix: queue processed_chunk after content_block_start in both __next__
and __anext__ block transition paths.
This commit is contained in:
Ishaan Jaffer 2026-04-13 19:16:20 -07:00
parent 5abd5851e5
commit 0c20e5bcb2
No known key found for this signature in database
1 changed files with 12 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py
View file

@ -128,9 +128,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
)
if should_start_new_block and not self.sent_content_block_finish:
# Queue the sequence: content_block_stop -> content_block_start
# The trigger chunk itself is not emitted as a delta since the
# content_block_start already carries the relevant information.
# Queue the sequence: content_block_stop -> content_block_start -> trigger delta
self.chunk_queue.append(
{
"type": "content_block_stop",
@ -144,6 +142,8 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
"content_block": self.current_content_block_start,
}
)
# Emit the trigger chunk's delta so its content is not lost
self.chunk_queue.append(processed_chunk)
self.sent_content_block_finish = False
return self.chunk_queue.popleft()
@ -282,16 +282,16 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
hasattr(chunk.usage, "_cache_creation_input_tokens")
and chunk.usage._cache_creation_input_tokens > 0
):
usage_dict[
"cache_creation_input_tokens"
] = chunk.usage._cache_creation_input_tokens
usage_dict["cache_creation_input_tokens"] = (
chunk.usage._cache_creation_input_tokens
)
if (
hasattr(chunk.usage, "_cache_read_input_tokens")
and chunk.usage._cache_read_input_tokens > 0
):
usage_dict[
"cache_read_input_tokens"
] = chunk.usage._cache_read_input_tokens
usage_dict["cache_read_input_tokens"] = (
chunk.usage._cache_read_input_tokens
)
merged_chunk["usage"] = usage_dict
# Queue the merged chunk and reset
@ -304,10 +304,6 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
if not self.queued_usage_chunk:
if should_start_new_block and not self.sent_content_block_finish:
# Queue the sequence: content_block_stop -> content_block_start
# The trigger chunk itself is not emitted as a delta since the
# content_block_start already carries the relevant information.
# 1. Stop current content block
self.chunk_queue.append(
{
@ -325,6 +321,9 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
}
)
# 3. Emit the trigger chunk's delta so its content is not lost
self.chunk_queue.append(processed_chunk)
# Reset state for new block
self.sent_content_block_finish = False