mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-13 08:52:02 +00:00
fix(anthropic/messages): keep reasoning in the thinking block on streaming block transitions
This commit is contained in:
parent
93b5ca9612
commit
d8a1791b13
2 changed files with 97 additions and 4 deletions
|
|
@ -1389,9 +1389,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
"signature": thought_sig,
|
||||
}
|
||||
return "tool_use", cast("ContentBlockContentBlockDict", tool_block)
|
||||
elif choice.delta.content is not None and len(choice.delta.content) > 0:
|
||||
return "text", TextBlock(type="text", text="")
|
||||
elif isinstance(choice, StreamingChoices) and hasattr(choice.delta, "thinking_blocks"):
|
||||
if isinstance(choice, StreamingChoices) and hasattr(choice.delta, "thinking_blocks"):
|
||||
thinking_blocks = choice.delta.thinking_blocks or []
|
||||
if len(thinking_blocks) > 0:
|
||||
thinking_block = thinking_blocks[0]
|
||||
|
|
@ -1415,8 +1413,10 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
# ``Delta`` deletes the ``thinking_blocks`` attribute when unset, so the
|
||||
# branch above is skipped entirely; open a ``thinking`` block here so the
|
||||
# matching ``thinking_delta`` stream is not emitted into a text block.
|
||||
elif isinstance(choice, StreamingChoices) and getattr(choice.delta, "reasoning_content", None):
|
||||
if isinstance(choice, StreamingChoices) and getattr(choice.delta, "reasoning_content", None):
|
||||
return "thinking", ChatCompletionThinkingBlock(type="thinking", thinking="", signature="")
|
||||
if choice.delta.content is not None and len(choice.delta.content) > 0:
|
||||
return "text", TextBlock(type="text", text="")
|
||||
|
||||
return "text", TextBlock(type="text", text="")
|
||||
|
||||
|
|
|
|||
|
|
@ -340,3 +340,96 @@ def test_trigger_delta_has_content_branches(processed_chunk, expected):
|
|||
assert (
|
||||
AnthropicStreamWrapper._trigger_delta_has_content(processed_chunk) is expected
|
||||
)
|
||||
|
||||
|
||||
def _block_type_by_index(events: List[dict]) -> dict:
|
||||
return {
|
||||
e["index"]: e["content_block"]["type"]
|
||||
for e in events
|
||||
if e.get("type") == "content_block_start"
|
||||
}
|
||||
|
||||
|
||||
def _delta_types_by_index(events: List[dict]) -> List[tuple]:
|
||||
return [
|
||||
(e["index"], e["delta"]["type"])
|
||||
for e in events
|
||||
if e.get("type") == "content_block_delta"
|
||||
]
|
||||
|
||||
|
||||
def _assert_deltas_match_block_types(events: List[dict]) -> None:
|
||||
"""Every ``*_delta`` must land in a block whose declared type accepts it:
|
||||
``thinking_delta``/``signature_delta`` only in ``thinking`` blocks,
|
||||
``text_delta`` only in ``text`` blocks. This is exactly the invariant
|
||||
Claude Code enforces ("Content block is not a thinking block").
|
||||
"""
|
||||
allowed = {
|
||||
"thinking": {"thinking_delta", "signature_delta"},
|
||||
"text": {"text_delta"},
|
||||
"tool_use": {"input_json_delta"},
|
||||
}
|
||||
block_types = _block_type_by_index(events)
|
||||
for index, delta_type in _delta_types_by_index(events):
|
||||
block_type = block_types[index]
|
||||
assert delta_type in allowed[block_type], (
|
||||
f"block index {index} opened as {block_type!r} received {delta_type!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_reasoning_bundled_with_first_text_does_not_leak_into_text_block_sync():
|
||||
"""Regression for issue #33224.
|
||||
|
||||
NVIDIA NIM Nemotron 3 Ultra bundles the tail of its reasoning together with
|
||||
the first bit of visible text in a single streaming chunk. Before the fix,
|
||||
block-type detection prioritized ``content`` while the delta translator
|
||||
prioritized ``reasoning_content``, so a ``text`` block was opened and then a
|
||||
``thinking_delta`` was streamed into it — SSE that Claude Code rejects.
|
||||
"""
|
||||
chunks = [
|
||||
_make_chunk(Delta(reasoning_content="The", content=None)),
|
||||
_make_chunk(Delta(reasoning_content=" assistant.", content="! I'm an")),
|
||||
_make_chunk(Delta(reasoning_content=None, content=" AI assistant.")),
|
||||
_make_chunk(Delta(content=None), finish_reason="stop"),
|
||||
]
|
||||
wrapper = AnthropicStreamWrapper(completion_stream=iter(chunks), model="claude-x")
|
||||
events = _drain_sync(wrapper)
|
||||
|
||||
_assert_deltas_match_block_types(events)
|
||||
|
||||
thinking = "".join(
|
||||
e["delta"]["thinking"]
|
||||
for e in events
|
||||
if e.get("type") == "content_block_delta"
|
||||
and e["delta"].get("type") == "thinking_delta"
|
||||
)
|
||||
assert thinking == "The assistant."
|
||||
assert "".join(_text_deltas(events)) == " AI assistant."
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reasoning_bundled_with_first_text_does_not_leak_into_text_block_async():
|
||||
"""Async path mirrors the sync regression for issue #33224 — the proxy
|
||||
serves the async iterator to Claude Code.
|
||||
"""
|
||||
chunks = [
|
||||
_make_chunk(Delta(reasoning_content="The", content=None)),
|
||||
_make_chunk(Delta(reasoning_content=" assistant.", content="! I'm an")),
|
||||
_make_chunk(Delta(reasoning_content=None, content=" AI assistant.")),
|
||||
_make_chunk(Delta(content=None), finish_reason="stop"),
|
||||
]
|
||||
wrapper = AnthropicStreamWrapper(
|
||||
completion_stream=_AsyncStream(chunks), model="claude-x"
|
||||
)
|
||||
events = await _drain_async(wrapper)
|
||||
|
||||
_assert_deltas_match_block_types(events)
|
||||
|
||||
thinking = "".join(
|
||||
e["delta"]["thinking"]
|
||||
for e in events
|
||||
if e.get("type") == "content_block_delta"
|
||||
and e["delta"].get("type") == "thinking_delta"
|
||||
)
|
||||
assert thinking == "The assistant."
|
||||
assert "".join(_text_deltas(events)) == " AI assistant."
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue