mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-03 02:22:24 +00:00
fix(responses): emit the reasoning item on streaming /v1/responses for signature-only thinking
Anthropic models return thinking blocks with empty text and the reasoning carried in the signature: Claude Fable 5.1 and Claude Opus 5.5 by default, and Bedrock adaptive thinking with or without an effort. On streaming /v1/responses the chat->Responses bridge opened a reasoning output item only on reasoning_content text (LiteLLMCompletionStreamingIterator._ensure_output_item_for_chunk), and ChunkProcessor.get_combined_thinking_content kept an assembled thinking block only when it had thinking text. Such a response emitted no reasoning item mid-stream and none in response.completed, so a streaming Responses client could not replay the reasoning even though the reasoning tokens were billed. Non-streaming /v1/responses was unaffected. Open the reasoning item when the delta carries a signed or redacted thinking block, and keep a signed block through stream assembly even when its thinking text is empty. Unsigned text-only fragments are still dropped. The reasoning-text path is unchanged.
This commit is contained in:
parent
6dbd65b230
commit
bc9b6f8a5c
4 changed files with 72 additions and 2 deletions
|
|
@ -685,7 +685,7 @@ class ChunkProcessor:
|
|||
|
||||
def _flush_thinking_block() -> None:
|
||||
nonlocal current_thinking_text_parts, current_signature
|
||||
if len(current_thinking_text_parts) > 0 and current_signature:
|
||||
if current_signature:
|
||||
thinking_blocks.append(
|
||||
ChatCompletionThinkingBlock(
|
||||
type="thinking",
|
||||
|
|
|
|||
|
|
@ -73,6 +73,11 @@ def _output_items_with_id(items: tuple[Any, ...], item_type: str, item_id: str |
|
|||
)
|
||||
|
||||
|
||||
def _delta_has_signed_thinking_block(delta: object) -> bool:
|
||||
blocks: Final = getattr(delta, "thinking_blocks", None) or ()
|
||||
return any(isinstance(b, dict) and (b.get("signature") or b.get("data")) for b in blocks)
|
||||
|
||||
|
||||
class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
||||
"""
|
||||
Async iterator for processing streaming responses from the Responses API.
|
||||
|
|
@ -936,7 +941,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
self.sent_output_item_added_event = True
|
||||
|
||||
# Reasoning-first
|
||||
if hasattr(delta, "reasoning_content") and delta.reasoning_content:
|
||||
if (hasattr(delta, "reasoning_content") and delta.reasoning_content) or _delta_has_signed_thinking_block(delta):
|
||||
self._reasoning_active = True
|
||||
if self._cached_reasoning_item_id is None:
|
||||
self._cached_reasoning_item_id = f"rs_{uuid.uuid4()}"
|
||||
|
|
|
|||
|
|
@ -236,6 +236,31 @@ def test_get_combined_thinking_content_preserves_interleaved_blocks():
|
|||
assert result[2]["signature"] == "sig_block2"
|
||||
|
||||
|
||||
def test_get_combined_thinking_content_keeps_signed_block_without_thinking_text():
|
||||
chunks: Final = [
|
||||
ModelResponseStream(
|
||||
id="chatcmpl-123",
|
||||
object="chat.completion.chunk",
|
||||
created=1234567890,
|
||||
model="claude-sonnet-4-20250514",
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
index=0,
|
||||
delta=Delta(thinking_blocks=[{"type": "thinking", "thinking": "", "signature": "sig_only"}]),
|
||||
finish_reason=None,
|
||||
)
|
||||
],
|
||||
)
|
||||
]
|
||||
|
||||
result: Final = ChunkProcessor(chunks=chunks).get_combined_thinking_content(chunks)
|
||||
|
||||
assert result is not None
|
||||
assert [(block["type"], block["thinking"], block["signature"]) for block in result] == [
|
||||
("thinking", "", "sig_only")
|
||||
]
|
||||
|
||||
|
||||
def test_cache_read_input_tokens_retained():
|
||||
chunk1 = ModelResponseStream(
|
||||
id="chatcmpl-95aabb85-c39f-443d-ae96-0370c404d70c",
|
||||
|
|
|
|||
|
|
@ -978,6 +978,25 @@ def _reasoning_chunk(reasoning: str, finish_reason: str | None = None) -> ModelR
|
|||
)
|
||||
|
||||
|
||||
def _signature_only_thinking_chunk(signature: str) -> ModelResponseStream:
|
||||
return ModelResponseStream(
|
||||
id=CHAT_COMPLETION_ID,
|
||||
created=1748575031,
|
||||
model="claude-haiku-4-5",
|
||||
object="chat.completion.chunk",
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
index=0,
|
||||
delta=Delta(
|
||||
role="assistant",
|
||||
thinking_blocks=[{"type": "thinking", "thinking": "", "signature": signature}],
|
||||
),
|
||||
finish_reason=None,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
async def _collect_events(
|
||||
iterator: LiteLLMCompletionStreamingIterator, sync_mode: bool
|
||||
) -> list[BaseLiteLLMOpenAIResponseObject]:
|
||||
|
|
@ -1015,6 +1034,27 @@ async def test_tool_only_stream_emits_no_message_item_events(sync_mode: bool):
|
|||
assert any(getattr(event, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED for event in events)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sync_mode", [True, False])
|
||||
@pytest.mark.asyncio
|
||||
async def test_signature_only_thinking_streams_a_replayable_reasoning_item(sync_mode: bool):
|
||||
iterator: Final = _build_iterator([_signature_only_thinking_chunk("sig_only"), _chunk("4", finish_reason="stop")])
|
||||
|
||||
events: Final = await _collect_events(iterator, sync_mode)
|
||||
|
||||
added_item_types: Final = [
|
||||
event.item.type
|
||||
for event in events
|
||||
if getattr(event, "type", None) == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED
|
||||
]
|
||||
completed: Final = next(
|
||||
event for event in events if getattr(event, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED
|
||||
)
|
||||
reasoning_items: Final = [item for item in completed.response.output if getattr(item, "type", None) == "reasoning"]
|
||||
assert added_item_types[0] == "reasoning"
|
||||
assert len(reasoning_items) == 1
|
||||
assert json.loads(reasoning_items[0].encrypted_content)[0]["signature"] == "sig_only"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sync_mode", [True, False])
|
||||
@pytest.mark.asyncio
|
||||
async def test_reasoning_then_text_announces_message_item_before_text_events(sync_mode: bool):
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue