mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge 3dc8570b66 into e6b4774ac6
This commit is contained in:
commit
308518841c
10 changed files with 258 additions and 100 deletions
|
|
@ -1974,7 +1974,7 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
"annotations": (),
|
||||
"logprobs": None,
|
||||
}
|
||||
return (
|
||||
events: Final = (
|
||||
*_open_item_closing_events(responses_so_far),
|
||||
OutputItemAddedEvent(
|
||||
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
|
||||
|
|
@ -2019,6 +2019,10 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
response=_blocked_response(exc, response_id=response_id, model=model, output_item=item),
|
||||
),
|
||||
)
|
||||
return tuple(
|
||||
event.model_copy(update={"sequence_number": number})
|
||||
for number, event in enumerate(events, start=_next_stream_sequence_number(responses_so_far))
|
||||
)
|
||||
|
||||
|
||||
class _BlockedContentPart(TypedDict):
|
||||
|
|
|
|||
|
|
@ -26,7 +26,11 @@ from litellm.responses.litellm_completion_transformation.tool_search import (
|
|||
from litellm.responses.litellm_completion_transformation.transformation import (
|
||||
LiteLLMCompletionResponsesConfig,
|
||||
)
|
||||
from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator, stamp_responses_usage_cost
|
||||
from litellm.responses.streaming_iterator import (
|
||||
SEQUENCE_NUMBER_STAMPED_ON_EMISSION,
|
||||
ResponsesAPIStreamingIterator,
|
||||
stamp_responses_usage_cost,
|
||||
)
|
||||
from litellm.responses.utils import ResponsesAPIRequestUtils
|
||||
from litellm.types.llms.openai import (
|
||||
PART_UNION_TYPES,
|
||||
|
|
@ -210,14 +214,11 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
return execution
|
||||
|
||||
def _next_output_item_added(self, output_index: int, item: BaseLiteLLMOpenAIResponseObject) -> OutputItemAddedEvent:
|
||||
self._sequence_number += 1
|
||||
added: Final = OutputItemAddedEvent(
|
||||
return OutputItemAddedEvent(
|
||||
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
|
||||
output_index=output_index,
|
||||
item=item,
|
||||
)
|
||||
added.__dict__["sequence_number"] = self._sequence_number
|
||||
return added
|
||||
|
||||
def _queue_output_item_added(self, output_index: int, item: BaseLiteLLMOpenAIResponseObject) -> None:
|
||||
self._pending_tool_events.append(self._next_output_item_added(output_index, item))
|
||||
|
|
@ -239,12 +240,10 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
item: Final = build_tool_search_call(
|
||||
call_id, final_args or self._tool_args_by_call_id.get(call_id, ""), "completed", execution
|
||||
)
|
||||
self._sequence_number += 1
|
||||
self._pending_tool_events.append(
|
||||
OutputItemDoneEvent(
|
||||
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
|
||||
output_index=output_index,
|
||||
sequence_number=self._sequence_number,
|
||||
item=BaseLiteLLMOpenAIResponseObject(**item.model_dump()),
|
||||
)
|
||||
)
|
||||
|
|
@ -317,12 +316,10 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
if streamed_item is None:
|
||||
return
|
||||
self._sent_compaction_item_done = True
|
||||
self._sequence_number += 1
|
||||
self._pending_response_events.append(
|
||||
OutputItemDoneEvent(
|
||||
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
|
||||
output_index=self._compaction_output_index,
|
||||
sequence_number=self._sequence_number,
|
||||
item=streamed_item,
|
||||
)
|
||||
)
|
||||
|
|
@ -465,15 +462,12 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
chunk_size = 10 # Match typical OpenAI delta size
|
||||
for i in range(0, len(fn_args_delta), chunk_size):
|
||||
delta_chunk = fn_args_delta[i : i + chunk_size]
|
||||
self._sequence_number += 1
|
||||
delta_event: BaseLiteLLMOpenAIResponseObject = FunctionCallArgumentsDeltaEvent(
|
||||
type=ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA,
|
||||
item_id=self._tool_item_id_by_call_id.get(call_id, call_id),
|
||||
output_index=output_index,
|
||||
delta=delta_chunk,
|
||||
)
|
||||
# Add sequence_number as extra field (BaseLiteLLMOpenAIResponseObject allows extra fields)
|
||||
delta_event.__dict__["sequence_number"] = self._sequence_number
|
||||
self._pending_tool_events.append(delta_event)
|
||||
|
||||
def _queue_final_tool_call_done_events(self, litellm_complete_object: ModelResponse) -> None:
|
||||
|
|
@ -541,33 +535,27 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
chunk_size = 10 # Match typical OpenAI delta size
|
||||
for i in range(0, len(remaining_args), chunk_size):
|
||||
delta_chunk = remaining_args[i : i + chunk_size]
|
||||
self._sequence_number += 1
|
||||
delta_event = FunctionCallArgumentsDeltaEvent(
|
||||
type=ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA,
|
||||
item_id=self._tool_item_id_by_call_id.get(call_id, call_id),
|
||||
output_index=output_index,
|
||||
delta=delta_chunk,
|
||||
)
|
||||
delta_event.__dict__["sequence_number"] = self._sequence_number
|
||||
self._pending_tool_events.append(delta_event)
|
||||
|
||||
self._sequence_number += 1
|
||||
done_event = FunctionCallArgumentsDoneEvent(
|
||||
type=ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DONE,
|
||||
item_id=self._tool_item_id_by_call_id.get(call_id, call_id),
|
||||
output_index=output_index,
|
||||
arguments=final_args,
|
||||
)
|
||||
done_event.__dict__["sequence_number"] = self._sequence_number
|
||||
self._pending_tool_events.append(done_event)
|
||||
|
||||
self._sequence_number += 1
|
||||
item_kwargs = self._tool_call_item_kwargs(call_id, fn_name, final_args, "completed")
|
||||
item_kwargs["id"] = self._tool_item_id_by_call_id.setdefault(call_id, item_kwargs["id"])
|
||||
item_done_event = OutputItemDoneEvent(
|
||||
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
|
||||
output_index=output_index,
|
||||
sequence_number=self._sequence_number,
|
||||
item=BaseLiteLLMOpenAIResponseObject(**item_kwargs),
|
||||
)
|
||||
self._pending_tool_events.append(item_done_event)
|
||||
|
|
@ -597,16 +585,11 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING, WebSearchCallSearchingEvent),
|
||||
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED, WebSearchCallCompletedEvent),
|
||||
):
|
||||
self._sequence_number += 1
|
||||
event = event_class(type=event_type, output_index=output_index, item_id=item.id)
|
||||
event.__dict__["sequence_number"] = self._sequence_number
|
||||
self._pending_tool_events.append(event)
|
||||
self._sequence_number += 1
|
||||
self._pending_tool_events.append(event_class(type=event_type, output_index=output_index, item_id=item.id))
|
||||
self._pending_tool_events.append(
|
||||
OutputItemDoneEvent(
|
||||
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
|
||||
output_index=output_index,
|
||||
sequence_number=self._sequence_number,
|
||||
item=BaseLiteLLMOpenAIResponseObject(**item.model_dump()),
|
||||
)
|
||||
)
|
||||
|
|
@ -742,31 +725,24 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
|
||||
"""
|
||||
response_created_event_data: Final = self._default_response_created_event_data()
|
||||
self._sequence_number += 1
|
||||
event: Final = ResponseCreatedEvent(
|
||||
return ResponseCreatedEvent(
|
||||
type=ResponsesAPIStreamEvents.RESPONSE_CREATED,
|
||||
response=self._with_encoded_response_id(ResponsesAPIResponse(**response_created_event_data)),
|
||||
)
|
||||
event.__dict__["sequence_number"] = self._sequence_number
|
||||
return event
|
||||
|
||||
def create_response_in_progress_event(self) -> ResponseInProgressEvent:
|
||||
response_in_progress_event_data: Final = self._default_response_created_event_data()
|
||||
response_in_progress_event_data["status"] = "in_progress"
|
||||
self._sequence_number += 1
|
||||
event: Final = ResponseInProgressEvent(
|
||||
return ResponseInProgressEvent(
|
||||
type=ResponsesAPIStreamEvents.RESPONSE_IN_PROGRESS,
|
||||
response=self._with_encoded_response_id(ResponsesAPIResponse(**response_in_progress_event_data)),
|
||||
)
|
||||
event.__dict__["sequence_number"] = self._sequence_number
|
||||
return event
|
||||
|
||||
def create_output_item_added_event(self) -> OutputItemAddedEvent:
|
||||
if self._cached_item_id is None:
|
||||
self._cached_item_id = f"msg_{uuid.uuid4()}"
|
||||
|
||||
self._sequence_number += 1
|
||||
event: Final = OutputItemAddedEvent(
|
||||
return OutputItemAddedEvent(
|
||||
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
|
||||
output_index=self._message_index(),
|
||||
item=BaseLiteLLMOpenAIResponseObject(
|
||||
|
|
@ -779,29 +755,23 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
}
|
||||
),
|
||||
)
|
||||
event.__dict__["sequence_number"] = self._sequence_number
|
||||
return event
|
||||
|
||||
def create_content_part_added_event(self) -> ContentPartAddedEvent:
|
||||
if self._cached_item_id is None:
|
||||
self._cached_item_id = f"msg_{uuid.uuid4()}"
|
||||
|
||||
self._sequence_number += 1
|
||||
event: Final = ContentPartAddedEvent(
|
||||
return ContentPartAddedEvent(
|
||||
type=ResponsesAPIStreamEvents.CONTENT_PART_ADDED,
|
||||
item_id=self._cached_item_id,
|
||||
output_index=self._message_index(),
|
||||
content_index=0,
|
||||
part=BaseLiteLLMOpenAIResponseObject(**{"type": "output_text", "text": "", "annotations": []}),
|
||||
)
|
||||
event.__dict__["sequence_number"] = self._sequence_number
|
||||
return event
|
||||
|
||||
def _queue_message_item_added_events(self) -> None:
|
||||
item_id: Final = self._message_item_id()
|
||||
self.sent_message_item_added_event = True
|
||||
self.sent_content_part_added_event = True
|
||||
self._sequence_number += 1
|
||||
event: Final = OutputItemAddedEvent(
|
||||
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
|
||||
output_index=self._message_index(),
|
||||
|
|
@ -815,7 +785,6 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
}
|
||||
),
|
||||
)
|
||||
event.__dict__["sequence_number"] = self._sequence_number
|
||||
self._pending_response_events.append(event)
|
||||
self._pending_response_events.append(self.create_content_part_added_event())
|
||||
|
||||
|
|
@ -871,7 +840,6 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
self,
|
||||
reasoning_item_id: str,
|
||||
reasoning_content: str,
|
||||
sequence_number: int,
|
||||
) -> ReasoningSummaryTextDoneEvent:
|
||||
"""
|
||||
Create response.reasoning_summary_text.done event.
|
||||
|
|
@ -881,7 +849,6 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
"type": "response.reasoning_summary_text.done",
|
||||
"item_id": "rs_0c5dae30e53172980069708ba2f59c8197b71ca9820edad07c",
|
||||
"output_index": 0,
|
||||
"sequence_number": 97,
|
||||
"summary_index": 0,
|
||||
"text": "**Clarifying the first humans**\n\nThe I'm addressing the user's specific interest."
|
||||
}
|
||||
|
|
@ -890,7 +857,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
type=ResponsesAPIStreamEvents.REASONING_SUMMARY_TEXT_DONE,
|
||||
item_id=reasoning_item_id,
|
||||
output_index=self._reasoning_index(),
|
||||
sequence_number=sequence_number,
|
||||
sequence_number=SEQUENCE_NUMBER_STAMPED_ON_EMISSION,
|
||||
summary_index=0,
|
||||
text=reasoning_content,
|
||||
)
|
||||
|
|
@ -899,7 +866,6 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
self,
|
||||
reasoning_item_id: str,
|
||||
reasoning_content: str,
|
||||
sequence_number: int,
|
||||
) -> ReasoningSummaryPartDoneEvent:
|
||||
"""
|
||||
Create response.reasoning_summary_part.done event.
|
||||
|
|
@ -913,7 +879,6 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
"type": "summary_text",
|
||||
"text": "**Clarifying the first humans**\n\nThe earlier hominins. It feels important to ensure I'm addressing the user's specific interest."
|
||||
},
|
||||
"sequence_number": 98,
|
||||
"summary_index": 0
|
||||
}
|
||||
"""
|
||||
|
|
@ -921,7 +886,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
type=ResponsesAPIStreamEvents.REASONING_SUMMARY_PART_DONE,
|
||||
item_id=reasoning_item_id,
|
||||
output_index=self._reasoning_index(),
|
||||
sequence_number=sequence_number,
|
||||
sequence_number=SEQUENCE_NUMBER_STAMPED_ON_EMISSION,
|
||||
summary_index=0,
|
||||
part=BaseLiteLLMOpenAIResponseObject(
|
||||
**{
|
||||
|
|
@ -985,7 +950,6 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
return OutputItemDoneEvent(
|
||||
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
|
||||
output_index=self._message_index(),
|
||||
sequence_number=1,
|
||||
item=BaseLiteLLMOpenAIResponseObject(
|
||||
**{
|
||||
"id": self._cached_item_id,
|
||||
|
|
@ -1007,7 +971,6 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
self,
|
||||
reasoning_item_id: str,
|
||||
reasoning_content: str,
|
||||
sequence_number: int,
|
||||
) -> OutputItemDoneEvent:
|
||||
"""
|
||||
Create response.output_item.done event for reasoning items.
|
||||
|
|
@ -1016,7 +979,6 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
{
|
||||
"type": "response.output_item.done",
|
||||
"output_index": 0,
|
||||
"sequence_number": 99,
|
||||
"item": {
|
||||
"id": "rs_0c5dae30e53172980069708ba2f59c8197b71ca9820edad07c",
|
||||
"type": "reasoning",
|
||||
|
|
@ -1032,7 +994,6 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
return OutputItemDoneEvent(
|
||||
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
|
||||
output_index=self._reasoning_index(),
|
||||
sequence_number=sequence_number,
|
||||
item=BaseLiteLLMOpenAIResponseObject(
|
||||
**{
|
||||
"id": reasoning_item_id,
|
||||
|
|
@ -1137,7 +1098,6 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
if chunk.choices[0].finish_reason is None and not _delta_carries_output(delta):
|
||||
return
|
||||
|
||||
self._sequence_number += 1
|
||||
self.sent_output_item_added_event = True
|
||||
|
||||
# Reasoning-first
|
||||
|
|
@ -1147,20 +1107,20 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
self._cached_reasoning_item_id = mint_reasoning_item_id()
|
||||
self._reasoning_item_id = self._cached_reasoning_item_id
|
||||
|
||||
event = OutputItemAddedEvent(
|
||||
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
|
||||
output_index=self._reasoning_index(),
|
||||
item=BaseLiteLLMOpenAIResponseObject(
|
||||
**{
|
||||
"id": self._cached_reasoning_item_id,
|
||||
"type": "reasoning",
|
||||
"status": "in_progress",
|
||||
"summary": None,
|
||||
}
|
||||
),
|
||||
self._pending_response_events.append(
|
||||
OutputItemAddedEvent(
|
||||
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
|
||||
output_index=self._reasoning_index(),
|
||||
item=BaseLiteLLMOpenAIResponseObject(
|
||||
**{
|
||||
"id": self._cached_reasoning_item_id,
|
||||
"type": "reasoning",
|
||||
"status": "in_progress",
|
||||
"summary": None,
|
||||
}
|
||||
),
|
||||
)
|
||||
)
|
||||
event.__dict__["sequence_number"] = self._sequence_number
|
||||
self._pending_response_events.append(event)
|
||||
return
|
||||
|
||||
# Tool-first
|
||||
|
|
@ -1191,30 +1151,34 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
)
|
||||
reasoning_item_id: Final = self._cached_reasoning_item_id
|
||||
self._queue_compaction_item_done()
|
||||
self._sequence_number += 1
|
||||
text_done_event: Final = self.create_reasoning_summary_text_done_event(
|
||||
reasoning_item_id=reasoning_item_id,
|
||||
reasoning_content=reasoning_content,
|
||||
sequence_number=self._sequence_number,
|
||||
)
|
||||
self._sequence_number += 1
|
||||
part_done_event: Final = self.create_reasoning_summary_part_done_event(
|
||||
reasoning_item_id=reasoning_item_id,
|
||||
reasoning_content=reasoning_content,
|
||||
sequence_number=self._sequence_number,
|
||||
)
|
||||
self._sequence_number += 1
|
||||
reasoning_output_item_done_event: Final = self.create_reasoning_output_item_done_event(
|
||||
reasoning_item_id=reasoning_item_id,
|
||||
reasoning_content=reasoning_content,
|
||||
sequence_number=self._sequence_number,
|
||||
)
|
||||
self._pending_response_events.extend([text_done_event, part_done_event, reasoning_output_item_done_event])
|
||||
self._reasoning_done_emitted = True
|
||||
self._reasoning_active = False
|
||||
|
||||
def _stamp_sequence_number(self, event: BaseLiteLLMOpenAIResponseObject) -> BaseLiteLLMOpenAIResponseObject:
|
||||
stamped: Final = event.model_copy(update={"sequence_number": self._sequence_number})
|
||||
self._sequence_number += 1
|
||||
return stamped
|
||||
|
||||
async def __anext__(
|
||||
self,
|
||||
) -> ResponsesAPIStreamingResponse | ResponseCompletedEvent | BaseLiteLLMOpenAIResponseObject:
|
||||
return self._stamp_sequence_number(await self._anext_unstamped_event())
|
||||
|
||||
async def _anext_unstamped_event(
|
||||
self,
|
||||
) -> ResponsesAPIStreamingResponse | ResponseCompletedEvent | BaseLiteLLMOpenAIResponseObject:
|
||||
try:
|
||||
while True:
|
||||
|
|
@ -1275,6 +1239,11 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
|
||||
def __next__(
|
||||
self,
|
||||
) -> ResponsesAPIStreamingResponse | ResponseCompletedEvent | BaseLiteLLMOpenAIResponseObject:
|
||||
return self._stamp_sequence_number(self._next_unstamped_event())
|
||||
|
||||
def _next_unstamped_event(
|
||||
self,
|
||||
) -> ResponsesAPIStreamingResponse | ResponseCompletedEvent | BaseLiteLLMOpenAIResponseObject:
|
||||
try:
|
||||
while True:
|
||||
|
|
@ -1416,16 +1385,13 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
if not self.sent_message_item_added_event:
|
||||
self._queue_message_item_added_events()
|
||||
self._hold_tool_calls_behind_text(getattr(chunk.choices[0].delta, "tool_calls", None))
|
||||
self._sequence_number += 1
|
||||
text_delta_event: Final = OutputTextDeltaEvent(
|
||||
return OutputTextDeltaEvent(
|
||||
type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA,
|
||||
item_id=item_id,
|
||||
output_index=self._message_index(),
|
||||
content_index=0,
|
||||
delta=delta_content,
|
||||
)
|
||||
text_delta_event.__dict__["sequence_number"] = self._sequence_number
|
||||
return text_delta_event
|
||||
|
||||
# Priority 3: Handle tool call deltas (if any) -> queue events and emit them
|
||||
# For each tool call delta, we emit events one at a time to match OpenAI's streaming behavior
|
||||
|
|
|
|||
|
|
@ -365,6 +365,7 @@ def _mid_stream_fallback_eligible(mapped_exception: Exception) -> bool:
|
|||
|
||||
|
||||
PRE_OUTPUT_LIFECYCLE_EVENT_TYPES: Final = frozenset({"response.created", "response.in_progress", "response.queued"})
|
||||
SEQUENCE_NUMBER_STAMPED_ON_EMISSION: Final = -1
|
||||
|
||||
|
||||
class BaseResponsesAPIStreamingIterator:
|
||||
|
|
@ -1911,7 +1912,6 @@ def build_synthetic_response_events(
|
|||
_build_response_status_event(openai_types.ResponsesAPIStreamEvents.RESPONSE_IN_PROGRESS, transformed),
|
||||
]
|
||||
|
||||
sequence_number = 0
|
||||
output_items: Final[Sequence[object]] = getattr(transformed, "output", []) or []
|
||||
for output_index, output_item in enumerate(output_items):
|
||||
output_item_payload = _dump_response_object(output_item)
|
||||
|
|
@ -2021,35 +2021,31 @@ def build_synthetic_response_events(
|
|||
delta=summary_text[i : i + chunk_size],
|
||||
)
|
||||
)
|
||||
sequence_number += 1
|
||||
events.append(
|
||||
openai_types.ReasoningSummaryTextDoneEvent(
|
||||
type=openai_types.ResponsesAPIStreamEvents.REASONING_SUMMARY_TEXT_DONE,
|
||||
item_id=item_id,
|
||||
output_index=output_index,
|
||||
sequence_number=sequence_number,
|
||||
sequence_number=SEQUENCE_NUMBER_STAMPED_ON_EMISSION,
|
||||
summary_index=summary_index,
|
||||
text=summary_text,
|
||||
)
|
||||
)
|
||||
sequence_number += 1
|
||||
events.append(
|
||||
openai_types.ReasoningSummaryPartDoneEvent(
|
||||
type=openai_types.ResponsesAPIStreamEvents.REASONING_SUMMARY_PART_DONE,
|
||||
item_id=item_id,
|
||||
output_index=output_index,
|
||||
sequence_number=sequence_number,
|
||||
sequence_number=SEQUENCE_NUMBER_STAMPED_ON_EMISSION,
|
||||
summary_index=summary_index,
|
||||
part=openai_types.BaseLiteLLMOpenAIResponseObject(**summary_payload),
|
||||
)
|
||||
)
|
||||
|
||||
sequence_number += 1
|
||||
events.append(
|
||||
openai_types.OutputItemDoneEvent(
|
||||
type=openai_types.ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
|
||||
output_index=output_index,
|
||||
sequence_number=sequence_number,
|
||||
item=openai_types.BaseLiteLLMOpenAIResponseObject(**output_item_payload),
|
||||
)
|
||||
)
|
||||
|
|
@ -2060,7 +2056,7 @@ def build_synthetic_response_events(
|
|||
response=transformed,
|
||||
)
|
||||
)
|
||||
return events
|
||||
return [event.model_copy(update={"sequence_number": number}) for number, event in enumerate(events)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -551,6 +551,7 @@ if TYPE_CHECKING:
|
|||
ResponseAPIUsage,
|
||||
ResponseIncompleteEvent,
|
||||
ResponsesAPIResponse,
|
||||
ResponsesAPIStreamingResponse,
|
||||
)
|
||||
|
||||
Span = _Span
|
||||
|
|
@ -4301,12 +4302,13 @@ class Router(DeprecatedRouterMembers):
|
|||
|
||||
fallback_headers_adopted: bool = False
|
||||
|
||||
def __init__(self, async_generator: AsyncGenerator):
|
||||
def __init__(self, async_generator: "AsyncGenerator[ResponsesAPIStreamingResponse, None]"):
|
||||
import time
|
||||
from datetime import datetime
|
||||
|
||||
self._async_generator = async_generator
|
||||
self._billing_source: object = source_iterator
|
||||
self._last_sequence_number = -1
|
||||
# Mirror every attribute BaseResponsesAPIStreamingIterator.__init__
|
||||
# would have set. The wrapper bypasses super().__init__ (it has no
|
||||
# httpx.Response of its own and no provider config to drive), so
|
||||
|
|
@ -4413,7 +4415,13 @@ class Router(DeprecatedRouterMembers):
|
|||
if self.completed_response is None and getattr(chunk, "type", None) in _RESPONSES_TERMINAL_EVENT_TYPES:
|
||||
self.completed_response = chunk
|
||||
self._yielded_first_chunk = True
|
||||
return chunk
|
||||
sequence_number: Final = getattr(chunk, "sequence_number", None)
|
||||
if not isinstance(sequence_number, int):
|
||||
return chunk
|
||||
self._last_sequence_number = max(self._last_sequence_number + 1, sequence_number)
|
||||
if self._last_sequence_number == sequence_number:
|
||||
return chunk
|
||||
return chunk.model_copy(update={"sequence_number": self._last_sequence_number})
|
||||
|
||||
async def aclose(self):
|
||||
# async generators always expose aclose — no defensive check needed.
|
||||
|
|
@ -4475,7 +4483,9 @@ class Router(DeprecatedRouterMembers):
|
|||
exc,
|
||||
)
|
||||
|
||||
wrapper: Final = FallbackResponsesStreamWrapper(stream_with_fallbacks())
|
||||
wrapper: Final = FallbackResponsesStreamWrapper(
|
||||
stream_with_fallbacks() # pyright: ignore[reportArgumentType] # relays response events typed as object
|
||||
)
|
||||
return wrapper
|
||||
|
||||
def _aresponses_background_stream_can_release(
|
||||
|
|
|
|||
|
|
@ -32,16 +32,16 @@ class BaseLiteLLMOpenAIResponseObject(LiteLLMBaseModel):
|
|||
model_config = ConfigDict(extra="allow", protected_namespaces=())
|
||||
|
||||
def __getitem__(self, key):
|
||||
return self.__dict__[key]
|
||||
return dict(iter(self))[key]
|
||||
|
||||
def get(self, key, default=None):
|
||||
return self.__dict__.get(key, default)
|
||||
return dict(iter(self)).get(key, default)
|
||||
|
||||
def __contains__(self, key) -> bool:
|
||||
return key in self.__dict__
|
||||
return key in dict(iter(self))
|
||||
|
||||
def items(self):
|
||||
return self.__dict__.items()
|
||||
return dict(iter(self)).items()
|
||||
|
||||
|
||||
class HiddenParams(OpenAIObject):
|
||||
|
|
|
|||
|
|
@ -4515,6 +4515,34 @@ class TestBuildBlockSseChunks:
|
|||
assert completed["output"][0]["content"][0]["text"] == "Blocked by policy."
|
||||
assert completed["usage"] == {"input_tokens": 7, "output_tokens": 21, "total_tokens": 28}
|
||||
|
||||
def test_continuation_numbers_every_event_after_the_last_one_relayed(self):
|
||||
yielded = [
|
||||
{"type": "response.created", "sequence_number": 0, "response": {"id": "resp_live", "output": []}},
|
||||
{
|
||||
"type": "response.output_item.added",
|
||||
"sequence_number": 1,
|
||||
"output_index": 0,
|
||||
"item": {"id": "msg_orig", "type": "message", "role": "assistant"},
|
||||
},
|
||||
{
|
||||
"type": "response.output_text.delta",
|
||||
"sequence_number": 2,
|
||||
"item_id": "msg_orig",
|
||||
"output_index": 0,
|
||||
"content_index": 0,
|
||||
"delta": "partial",
|
||||
},
|
||||
]
|
||||
|
||||
payloads = self._payloads(
|
||||
OpenAIResponsesHandler().build_block_sse_chunks(
|
||||
self._exc(original_response=yielded), stream_started=True, responses_so_far=yielded
|
||||
)
|
||||
)
|
||||
|
||||
assert payloads[-1]["type"] == "response.completed"
|
||||
assert [payload.get("sequence_number") for payload in payloads] == list(range(3, 3 + len(payloads)))
|
||||
|
||||
def test_continuation_reads_usage_from_typed_completed_event(self):
|
||||
from litellm.types.llms.openai import (
|
||||
ResponseCompletedEvent,
|
||||
|
|
|
|||
|
|
@ -477,7 +477,6 @@ def test_tool_call_arguments_are_chunked_to_match_openai_behavior():
|
|||
assert evt is not None
|
||||
assert evt.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED
|
||||
assert evt.output_index == 0
|
||||
assert hasattr(evt, "__dict__") and "sequence_number" in evt.__dict__
|
||||
|
||||
# Collect all remaining delta events from the pending queue by creating empty chunks
|
||||
delta_events = []
|
||||
|
|
@ -511,17 +510,11 @@ def test_tool_call_arguments_are_chunked_to_match_openai_behavior():
|
|||
assert len(evt.delta) <= 10
|
||||
assert evt.item_id == "fc_call_test"
|
||||
assert evt.output_index == 0
|
||||
assert hasattr(evt, "__dict__") and "sequence_number" in evt.__dict__
|
||||
|
||||
# Verify all deltas concatenated equal the original arguments
|
||||
concatenated = "".join(evt.delta for evt in delta_events)
|
||||
assert concatenated == large_arguments
|
||||
|
||||
# Verify sequence numbers are increasing
|
||||
sequence_numbers = [evt.__dict__["sequence_number"] for evt in delta_events]
|
||||
assert sequence_numbers == sorted(sequence_numbers)
|
||||
assert len(set(sequence_numbers)) == len(sequence_numbers) # All unique
|
||||
|
||||
|
||||
def test_tool_call_delta_without_id_uses_index_mapping():
|
||||
iterator = LiteLLMCompletionStreamingIterator(
|
||||
|
|
@ -1988,3 +1981,50 @@ def test_the_partial_response_billed_on_disconnect_carries_the_logging_objects_b
|
|||
partial.response.usage.input_tokens,
|
||||
partial.response.usage.output_tokens,
|
||||
), "the bill is priced on the partial's own usage"
|
||||
|
||||
|
||||
def _serialized_sequence_numbers(events: Iterable[BaseLiteLLMOpenAIResponseObject]) -> list[int]:
|
||||
payloads: Final = tuple(json.loads(event.model_dump_json()) for event in events)
|
||||
assert all("sequence_number" in payload for payload in payloads), [
|
||||
payload["type"] for payload in payloads if "sequence_number" not in payload
|
||||
]
|
||||
return [payload["sequence_number"] for payload in payloads]
|
||||
|
||||
|
||||
def _long_text_chunks() -> tuple[ModelResponseStream, ...]:
|
||||
return (*(_chunk(f"word{i} ") for i in range(300)), _chunk("end", finish_reason="stop"))
|
||||
|
||||
|
||||
def _text_then_tool_call_chunks() -> tuple[ModelResponseStream, ...]:
|
||||
return (_text_opening_a_tool_call_chunk("Checking. "), _tool_arguments_chunk('{"x": 1}', finish_reason="tool_calls"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"stream",
|
||||
[
|
||||
lambda: (_chunk("Hello"), _chunk("!", finish_reason="stop")),
|
||||
_long_text_chunks,
|
||||
_text_then_tool_call_chunks,
|
||||
_reasoning_then_text_chunks,
|
||||
_reasoning_and_text_in_one_chunk,
|
||||
_reasoning_citation_then_tool_call_finishing_chunks,
|
||||
lambda: (_tool_call_chunk(), _chunk("", finish_reason="tool_calls")),
|
||||
],
|
||||
ids=["text", "long_text", "text_then_tool", "reasoning_then_text", "reasoning_and_text", "citation_tool", "tool"],
|
||||
)
|
||||
@pytest.mark.parametrize("sync_mode", [True, False])
|
||||
@pytest.mark.asyncio
|
||||
async def test_every_event_serializes_its_position_in_the_stream_from_zero(stream: StreamChunks, sync_mode: bool):
|
||||
iterator: Final = _build_iterator(stream())
|
||||
events: Final = list(iterator) if sync_mode else [event async for event in iterator]
|
||||
|
||||
assert _serialized_sequence_numbers(events) == list(range(len(events)))
|
||||
|
||||
|
||||
def test_bridged_events_answer_dict_reads_of_their_number():
|
||||
events: Final = list(_build_iterator((_chunk("Hello"), _chunk("!", finish_reason="stop"))))
|
||||
positions: Final = list(range(len(events)))
|
||||
|
||||
assert [event["sequence_number"] for event in events] == positions
|
||||
assert [event.get("sequence_number") for event in events] == positions
|
||||
assert all("sequence_number" in event for event in events)
|
||||
|
|
|
|||
|
|
@ -2835,3 +2835,52 @@ async def test_websocket_turns_logging_survives_garbage_collection(terminal_even
|
|||
await _run_websocket_turn(logger, terminal_event)
|
||||
|
||||
assert await _logging_tasks_left_after_garbage_collection(logger) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"output",
|
||||
[
|
||||
[
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_seq",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "hello world", "annotations": []}],
|
||||
}
|
||||
],
|
||||
[
|
||||
{
|
||||
"type": "reasoning",
|
||||
"id": "rs_seq",
|
||||
"summary": [{"type": "summary_text", "text": "thinking about it"}],
|
||||
},
|
||||
{
|
||||
"type": "function_call",
|
||||
"id": "fc_seq",
|
||||
"call_id": "call_seq",
|
||||
"name": "lookup",
|
||||
"arguments": '{"id":1}',
|
||||
},
|
||||
],
|
||||
],
|
||||
ids=["message", "reasoning_and_tool"],
|
||||
)
|
||||
def test_synthetic_stream_serializes_every_event_position_from_zero(output: list[dict[str, object]]):
|
||||
transformed: Final = ResponsesAPIResponse(
|
||||
id="resp_seq",
|
||||
created_at=int(datetime.now().timestamp()),
|
||||
status="completed",
|
||||
model="claude-sonnet-5-5",
|
||||
object="response",
|
||||
output=output,
|
||||
)
|
||||
|
||||
events: Final = streaming_module.build_synthetic_response_events(
|
||||
transformed=transformed,
|
||||
logging_obj=None,
|
||||
chunk_size=5,
|
||||
)
|
||||
|
||||
payloads: Final = tuple(json.loads(event.model_dump_json()) for event in events)
|
||||
assert [payload.get("sequence_number") for payload in payloads] == list(range(len(payloads)))
|
||||
|
|
|
|||
|
|
@ -6776,6 +6776,44 @@ async def test_aresponses_streaming_content_policy_error_event_routes_to_content
|
|||
assert fallback_call.await_args.kwargs["model"] == "gemini/gemini-2.5-flash"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aresponses_stream_keeps_numbering_after_a_mid_stream_fallback():
|
||||
from litellm.types.llms.openai import OutputTextDeltaEvent, ResponsesAPIStreamEvents
|
||||
|
||||
def delta(sequence_number: int, text: str) -> OutputTextDeltaEvent:
|
||||
return OutputTextDeltaEvent(
|
||||
type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA,
|
||||
item_id="msg_1",
|
||||
output_index=0,
|
||||
content_index=0,
|
||||
delta=text,
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
|
||||
from litellm.exceptions import MidStreamFallbackError
|
||||
|
||||
router = _InjectedFallbackRouter(_AsyncList([delta(0, "c"), delta(1, "d")]))
|
||||
primary = _make_responses_iterator(
|
||||
chunks=[delta(0, "a"), delta(1, "b")],
|
||||
error=MidStreamFallbackError(
|
||||
message="primary died",
|
||||
model="primary",
|
||||
llm_provider="anthropic",
|
||||
is_pre_first_chunk=False,
|
||||
generated_content="ab",
|
||||
),
|
||||
model="primary",
|
||||
)
|
||||
|
||||
wrapped = await router._aresponses_streaming_iterator(
|
||||
response=primary,
|
||||
initial_kwargs={"model": "primary", "stream": True, "input": "Hi", "original_generic_function": litellm.aresponses},
|
||||
)
|
||||
events = [event async for event in wrapped]
|
||||
|
||||
assert [(event.delta, event.sequence_number) for event in events] == [("a", 0), ("b", 1), ("c", 2), ("d", 3)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aresponses_streaming_iterator_unwraps_content_policy_trigger_for_fallback_dispatch():
|
||||
"""The fallback dispatch matches on the trigger's own type, so the wrapper must hand it the
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ from typing import Final
|
|||
import pytest
|
||||
from pydantic import ConfigDict, create_model
|
||||
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject, LiteLLMBaseModel
|
||||
|
||||
_PROBE: Final = """
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
|
@ -142,3 +142,30 @@ def test_concurrent_first_use_of_a_deferred_subclass_always_builds_that_subclass
|
|||
assert _concurrent_first_use_outcomes(child) == {child.__name__}, f"round {round_id}"
|
||||
finally:
|
||||
sys.setswitchinterval(previous_switch_interval)
|
||||
|
||||
|
||||
class _ResponseEvent(BaseLiteLLMOpenAIResponseObject):
|
||||
type: str
|
||||
|
||||
|
||||
def test_response_object_answers_dict_reads_for_undeclared_fields():
|
||||
event: Final = _ResponseEvent(type="response.created", sequence_number=3)
|
||||
|
||||
assert event["type"] == "response.created"
|
||||
assert event["sequence_number"] == 3
|
||||
assert event.get("sequence_number") == 3
|
||||
assert "sequence_number" in event
|
||||
assert dict(event.items()) == {"type": "response.created", "sequence_number": 3}
|
||||
|
||||
|
||||
def test_response_object_answers_dict_reads_for_an_undeclared_field_named_keys():
|
||||
assert _ResponseEvent(type="response.created", keys="value")["keys"] == "value"
|
||||
|
||||
|
||||
def test_response_object_dict_reads_still_miss_absent_keys():
|
||||
event: Final = _ResponseEvent(type="response.created")
|
||||
|
||||
assert event.get("sequence_number", "fallback") == "fallback"
|
||||
assert "sequence_number" not in event
|
||||
with pytest.raises(KeyError):
|
||||
event["sequence_number"]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue