scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-06 07:48:06 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 112

Merge 964f8aa309 into 3f5e42d032

Browse source
This commit is contained in:
ryan-crabbe-berri 2026-10-29 12:04:21 +05:30 • committed by GitHub
commit a2c2f62fa5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 154 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

67
litellm/responses/litellm_completion_transformation/streaming_iterator.py
View file

@ -48,6 +48,7 @@ from litellm.types.llms.openai import (
ResponseIncompleteEvent,
ResponseInProgressEvent,
ResponseInputParam,
ResponsePartAddedEvent,
ResponsesAPIOptionalRequestParams,
ResponsesAPIResponse,
ResponsesAPIStreamEvents,
@ -177,6 +178,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
self._cached_reasoning_item_id: str | None = None
self._sent_reasoning_summary_text_done_event: bool = False
self._sent_reasoning_summary_part_done_event: bool = False
self._sent_reasoning_summary_part_added_event: bool = False
self._reasoning_summary_text: str = ""
# -- GENERIC RESPONSE-EVENTS PENDING QUEUE as required by fix --
self._pending_response_events: list[BaseLiteLLMOpenAIResponseObject] = []
@ -852,6 +854,28 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
chunk_dict["usage"] = hidden_params["usage"]
return chunk_dict
def create_reasoning_summary_part_added_event(self, reasoning_item_id: str) -> ResponsePartAddedEvent:
"""
Create response.reasoning_summary_part.added event, which opens the summary part
that the following response.reasoning_summary_text.delta events append to.
Example:
{
"type": "response.reasoning_summary_part.added",
"item_id": "rs_0c5dae30e53172980069708ba2f59c8197b71ca9820edad07c",
"output_index": 0,
"summary_index": 0,
"part": {"type": "summary_text", "text": ""}
}
"""
return ResponsePartAddedEvent(
type=ResponsesAPIStreamEvents.RESPONSE_PART_ADDED,
item_id=reasoning_item_id,
output_index=self._reasoning_index(),
summary_index=0,
part={"type": "summary_text", "text": ""},
)
def create_reasoning_summary_text_done_event(
self,
reasoning_item_id: str,
@ -1022,12 +1046,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
**{
"id": reasoning_item_id,
"type": "reasoning",
"summary": [
{
"type": "summary_text",
"text": reasoning_content,
}
],
"summary": [{"type": "summary_text", "text": reasoning_content}] if reasoning_content else [],
}
),
)
@ -1139,7 +1158,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
"id": self._cached_reasoning_item_id,
"type": "reasoning",
"status": "in_progress",
"summary": None,
"summary": [],
}
),
)
@ -1171,25 +1190,27 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
)
reasoning_item_id: Final = self._cached_reasoning_item_id
self._queue_compaction_item_done()
self._sequence_number += 1
text_done_event: Final = self.create_reasoning_summary_text_done_event(
reasoning_item_id=reasoning_item_id,
reasoning_content=reasoning_content,
sequence_number=self._sequence_number,
)
self._sequence_number += 1
part_done_event: Final = self.create_reasoning_summary_part_done_event(
reasoning_item_id=reasoning_item_id,
reasoning_content=reasoning_content,
sequence_number=self._sequence_number,
)
if reasoning_content:
self._sequence_number += 1
text_done_event: Final = self.create_reasoning_summary_text_done_event(
reasoning_item_id=reasoning_item_id,
reasoning_content=reasoning_content,
sequence_number=self._sequence_number,
)
self._sequence_number += 1
part_done_event: Final = self.create_reasoning_summary_part_done_event(
reasoning_item_id=reasoning_item_id,
reasoning_content=reasoning_content,
sequence_number=self._sequence_number,
)
self._pending_response_events.extend([text_done_event, part_done_event])
self._sequence_number += 1
reasoning_output_item_done_event: Final = self.create_reasoning_output_item_done_event(
reasoning_item_id=reasoning_item_id,
reasoning_content=reasoning_content,
sequence_number=self._sequence_number,
)
self._pending_response_events.extend([text_done_event, part_done_event, reasoning_output_item_done_event])
self._pending_response_events.append(reasoning_output_item_done_event)
self._reasoning_done_emitted = True
self._reasoning_active = False
@ -1362,11 +1383,17 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
if self._cached_reasoning_item_id is None:
self._cached_reasoning_item_id = mint_reasoning_item_id()
if not self._sent_reasoning_summary_part_added_event:
self._sent_reasoning_summary_part_added_event = True
self._pending_response_events.append(
self.create_reasoning_summary_part_added_event(self._cached_reasoning_item_id)
)
return ReasoningSummaryTextDeltaEvent(
type=ResponsesAPIStreamEvents.REASONING_SUMMARY_TEXT_DELTA,
item_id=self._cached_reasoning_item_id,
output_index=self._reasoning_index(),
summary_index=0,
delta=reasoning_content,
)

42
litellm/responses/litellm_completion_transformation/transformation.py
View file

@ -113,6 +113,7 @@ from litellm.types.responses.main import (
OutputImageGenerationCall,
OutputText,
OutputToolSearchCall,
ReasoningSummaryText,
)
from litellm.types.utils import (
ChatCompletionAnnotation,
@ -3427,23 +3428,30 @@ class LiteLLMCompletionResponsesConfig:
if reasoning_content or encrypted_content:
# Only check the first choice for reasoning content
return [
GenericResponseOutputItem(
type="reasoning",
id=mint_reasoning_item_id(),
status=LiteLLMCompletionResponsesConfig._map_chat_completion_finish_reason_to_responses_status(
choice.finish_reason
),
role="assistant",
content=[
OutputText(
type="output_text",
text=text,
annotations=[],
)
for text in (reasoning_content,)
if text
],
encrypted_content=encrypted_content,
GenericResponseOutputItem.model_validate(
{
"type": "reasoning",
"id": mint_reasoning_item_id(),
"status": LiteLLMCompletionResponsesConfig._map_chat_completion_finish_reason_to_responses_status(
choice.finish_reason
),
"role": "assistant",
"content": [
OutputText(
type="output_text",
text=text,
annotations=[],
)
for text in (reasoning_content,)
if text
],
"summary": [
ReasoningSummaryText(type="summary_text", text=text)
for text in (reasoning_content,)
if text
],
"encrypted_content": encrypted_content,
}
)
]
return []

7
litellm/types/responses/main.py
View file

@ -158,6 +158,13 @@ class GenericResponseOutputItem(BaseLiteLLMOpenAIResponseObject):
phase: Phase = None
class ReasoningSummaryText(BaseLiteLLMOpenAIResponseObject):
model_config = ConfigDict(frozen=True)
type: Literal["summary_text"]
text: str
class DeleteResponseResult(BaseLiteLLMOpenAIResponseObject):
"""
Result of a delete response request

79
tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py
View file

@ -12,11 +12,13 @@ spend tracking stores, so a follow-up previous_response_id still finds the conve
import itertools
import json
from collections.abc import Callable
from collections.abc import Callable, Mapping, Sequence
from typing import Final
from unittest.mock import AsyncMock, MagicMock
import pytest
from openai.types.responses.response_reasoning_item import Summary as OpenAIReasoningSummaryPart
from pydantic import TypeAdapter
from litellm.responses.litellm_completion_transformation.streaming_iterator import (
LiteLLMCompletionStreamingIterator,
@ -1241,6 +1243,77 @@ async def test_reasoning_item_closes_before_message_item_opens(stream_chunks: St
assert _event_text(events, ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA) == "Hello!"
def _split_reasoning_then_text_chunks() -> tuple[ModelResponseStream, ...]:
return (_reasoning_chunk("let me "), _reasoning_chunk("think"), _chunk("Hello!", finish_reason="stop"))
def _signature_only_thinking_then_text_chunks() -> tuple[ModelResponseStream, ...]:
return (_signature_only_thinking_chunk("sig_only"), _chunk("4", finish_reason="stop"))
def _streamed_reasoning_text(chunks: Sequence[ModelResponseStream]) -> str:
return "".join(getattr(chunk.choices[0].delta, "reasoning_content", None) or "" for chunk in chunks)
def _proxy_sse_payload(event: BaseLiteLLMOpenAIResponseObject) -> Mapping[str, object]:
"""The same dump litellm.proxy.proxy_server._serialize_streaming_chunk puts on the SSE wire."""
return json.loads(event.model_dump_json(exclude_none=True, exclude_unset=True))
def _reasoning_summary_as_the_openai_sdk_accumulates_it(
payloads: Sequence[Mapping[str, object]],
) -> Sequence[Mapping[str, str]]:
"""Replays openai-node ResponseAccumulator's lookups, so a key or index missing on the wire raises as the SDK throws."""
items: Final = {}
for payload in payloads:
match payload["type"]:
case "response.output_item.added":
items[payload["output_index"]] = payload["item"]
case "response.reasoning_summary_part.added":
items[payload["output_index"]]["summary"].append(payload["part"])
case "response.reasoning_summary_text.delta":
items[payload["output_index"]]["summary"][payload["summary_index"]]["text"] += payload["delta"]
reasoning_item: Final = next(item for item in items.values() if item["type"] == "reasoning")
return reasoning_item["summary"]
def _reasoning_item(items: Sequence[Mapping[str, object]]) -> Mapping[str, object]:
return next(item for item in items if item["type"] == "reasoning")
@pytest.mark.parametrize(
"stream_chunks",
[
_split_reasoning_then_text_chunks,
_reasoning_and_text_in_one_chunk,
_signed_thinking_and_text_in_one_chunk,
_signature_only_thinking_then_text_chunks,
],
ids=[
"split_reasoning_then_text_chunks",
"reasoning_and_text_in_one_chunk",
"signed_thinking_and_text_in_one_chunk",
"signature_only_thinking_then_text_chunks",
],
)
@pytest.mark.parametrize("sync_mode", [True, False])
@pytest.mark.asyncio
async def test_sdk_stream_readers_can_place_every_reasoning_summary_delta(stream_chunks: StreamChunks, sync_mode: bool):
chunks: Final = stream_chunks()
iterator: Final = _build_iterator(chunks)
payloads: Final = [_proxy_sse_payload(event) for event in await _collect_events(iterator, sync_mode)]
accumulated: Final = _reasoning_summary_as_the_openai_sdk_accumulates_it(payloads)
done_items: Final = [payload["item"] for payload in payloads if payload["type"] == "response.output_item.done"]
completed_payload: Final = next(payload for payload in payloads if payload["type"] == "response.completed")
completed_summary: Final = _reasoning_item(completed_payload["response"]["output"])["summary"]
assert "".join(part["text"] for part in accumulated) == _streamed_reasoning_text(chunks)
assert _reasoning_item(done_items)["summary"] == accumulated
assert completed_summary == accumulated
TypeAdapter(list[OpenAIReasoningSummaryPart]).validate_python([*accumulated, *completed_summary])
@pytest.mark.parametrize("sync_mode", [True, False])
@pytest.mark.asyncio
async def test_tool_then_reasoning_then_text_gives_message_its_own_output_index(sync_mode: bool):
@ -1523,9 +1596,7 @@ async def test_anthropic_tool_search_call_streams_as_tool_search_call_item(sync_
model="claude-sonnet-5-5",
object="chat.completion.chunk",
choices=[
StreamingChoices(
index=0, delta=delta, finish_reason="tool_calls" if index == len(deltas) - 1 else None
)
StreamingChoices(index=0, delta=delta, finish_reason="tool_calls" if index == len(deltas) - 1 else None)
],
)
for index, delta in enumerate(deltas)