From 161fb93f99f73baa668e711ca64d1b26599b85b0 Mon Sep 17 00:00:00 2001 From: Kaustubh Anil Jain <78426775+kaust1495@users.noreply.github.com> Date: Wed, 7 Oct 2026 03:30:11 +0530 Subject: [PATCH 1/6] fix(responses): serialize sequence_number on every bridged streaming event --- .../streaming_iterator.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index e21f706ba60..749f29a1145 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -132,6 +132,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._next_tool_output_index: int = 1 # output_index=0 reserved for the message item self._final_tool_events_queued: bool = False self._sequence_number: int = 0 + self._emitted_sequence_number: int = 0 self._cached_reasoning_item_id: str | None = None self._sent_reasoning_summary_text_done_event: bool = False self._sent_reasoning_summary_part_done_event: bool = False @@ -975,8 +976,25 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._queue_message_item_added_events() return + def _stamp_sequence_number(self, event: Any) -> Any: + """ + Give every emitted event a serialized, strictly increasing sequence_number. + + Writing to ``event.__dict__`` skips pydantic's extra-field storage, so those + values never reach ``model_dump()``/``model_dump_json()``. Setting the + attribute stores it correctly (declared field or allowed extra). + """ + self._emitted_sequence_number += 1 + event.sequence_number = self._emitted_sequence_number + return event + async def __anext__( self, + ) -> ResponsesAPIStreamingResponse | ResponseCompletedEvent | BaseLiteLLMOpenAIResponseObject: + return self._stamp_sequence_number(await self._anext_event()) + + async def _anext_event( + self, ) -> ResponsesAPIStreamingResponse | ResponseCompletedEvent | BaseLiteLLMOpenAIResponseObject: try: while True: @@ -1087,6 +1105,11 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): def __next__( self, + ) -> ResponsesAPIStreamingResponse | ResponseCompletedEvent | BaseLiteLLMOpenAIResponseObject: + return self._stamp_sequence_number(self._next_event()) + + def _next_event( + self, ) -> ResponsesAPIStreamingResponse | ResponseCompletedEvent | BaseLiteLLMOpenAIResponseObject: try: while True: From e13fb713f780f17a16ed91e1e3be86a3b00fe067 Mon Sep 17 00:00:00 2001 From: Kaustubh Anil Jain <78426775+kaust1495@users.noreply.github.com> Date: Wed, 7 Oct 2026 03:30:35 +0530 Subject: [PATCH 2/6] test(responses): cover sequence_number on bridged streaming events --- .../test_streaming_iterator_transformation.py | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index 041bcf1b6d7..88a3d450273 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -1171,3 +1171,75 @@ async def test_plain_text_stream_announces_exactly_one_message_item(sync_mode: b ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE, ): assert event.item_id == message_item_adds[0].item.id + + +def _serialized_sequence_numbers(events) -> list[int]: + payloads = [json.loads(event.model_dump_json()) for event in events] + assert all("sequence_number" in payload for payload in payloads), [ + payload["type"] for payload in payloads if "sequence_number" not in payload + ] + return [payload["sequence_number"] for payload in payloads] + + +def _assert_consecutive(sequence_numbers: list[int]) -> None: + assert sequence_numbers == list(range(sequence_numbers[0], sequence_numbers[0] + len(sequence_numbers))) + + +def test_sync_stream_events_serialize_increasing_sequence_numbers(): + """Every event, including output_item.done, serializes a strictly increasing sequence_number.""" + events = list(_build_iterator([_chunk("Hello"), _chunk("!", finish_reason="stop")])) + + sequence_numbers = _serialized_sequence_numbers(events) + _assert_consecutive(sequence_numbers) + done = [e for e in events if getattr(e, "type", None) == ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE] + assert len(done) == 1 + assert json.loads(done[0].model_dump_json())["sequence_number"] > 1 + + +@pytest.mark.asyncio +async def test_async_stream_events_serialize_increasing_sequence_numbers(): + events = [event async for event in _build_iterator([_chunk("Hello"), _chunk("!", finish_reason="stop")])] + + _assert_consecutive(_serialized_sequence_numbers(events)) + + +def test_long_stream_sequence_numbers_stay_unique_and_ordered(): + """Stress: many chunks never produce duplicate or out-of-order sequence numbers.""" + chunks = [_chunk(f"word{i} ") for i in range(300)] + [_chunk("end", finish_reason="stop")] + events = list(_build_iterator(chunks)) + + sequence_numbers = _serialized_sequence_numbers(events) + assert len(events) > 300 + _assert_consecutive(sequence_numbers) + assert len(set(sequence_numbers)) == len(sequence_numbers) + + +def test_tool_call_stream_sequence_numbers_are_serialized_and_increasing(): + chunk = ModelResponseStream( + id="chunk-1", + created=123, + model="test-model", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta( + role="assistant", + content="", + tool_calls=[ + { + "id": "call_1", + "type": "function", + "function": {"name": "do_thing", "arguments": '{"x": "' + "a" * 55 + '"}'}, + } + ], + ), + ) + ], + ) + events = list(_build_iterator([chunk, _chunk("", finish_reason="tool_calls")])) + + sequence_numbers = _serialized_sequence_numbers(events) + _assert_consecutive(sequence_numbers) + assert any(e.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA for e in events) From 1ecde9e12576600c529e0f73d7f47d41e20d7958 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 11:20:56 -0700 Subject: [PATCH 3/6] fix(responses): read undeclared fields through dict accessors and number guardrail block continuations --- .../guardrail_translation/handler.py | 6 +++- .../streaming_iterator.py | 3 -- litellm/types/llms/base.py | 8 ++--- ...test_openai_responses_guardrail_handler.py | 28 ++++++++++++++++++ .../test_streaming_iterator_transformation.py | 9 ++++++ tests/unit/types/llms/test_types_llms_base.py | 29 ++++++++++++++++++- 6 files changed, 74 insertions(+), 9 deletions(-) diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 559daf767c2..64aca9496eb 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -1974,7 +1974,7 @@ class OpenAIResponsesHandler(BaseTranslation): "annotations": (), "logprobs": None, } - return ( + events: Final = ( *_open_item_closing_events(responses_so_far), OutputItemAddedEvent( type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, @@ -2019,6 +2019,10 @@ class OpenAIResponsesHandler(BaseTranslation): response=_blocked_response(exc, response_id=response_id, model=model, output_item=item), ), ) + return tuple( + event.model_copy(update={"sequence_number": number}) + for number, event in enumerate(events, start=_next_stream_sequence_number(responses_so_far)) + ) class _BlockedContentPart(TypedDict): diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 221aac17264..ae58fb0ce79 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -849,7 +849,6 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): "type": "response.reasoning_summary_text.done", "item_id": "rs_0c5dae30e53172980069708ba2f59c8197b71ca9820edad07c", "output_index": 0, - "sequence_number": 97, "summary_index": 0, "text": "**Clarifying the first humans**\n\nThe I'm addressing the user's specific interest." } @@ -880,7 +879,6 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): "type": "summary_text", "text": "**Clarifying the first humans**\n\nThe earlier hominins. It feels important to ensure I'm addressing the user's specific interest." }, - "sequence_number": 98, "summary_index": 0 } """ @@ -981,7 +979,6 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): { "type": "response.output_item.done", "output_index": 0, - "sequence_number": 99, "item": { "id": "rs_0c5dae30e53172980069708ba2f59c8197b71ca9820edad07c", "type": "reasoning", diff --git a/litellm/types/llms/base.py b/litellm/types/llms/base.py index 5a2da071cec..37937892e84 100644 --- a/litellm/types/llms/base.py +++ b/litellm/types/llms/base.py @@ -32,16 +32,16 @@ class BaseLiteLLMOpenAIResponseObject(LiteLLMBaseModel): model_config = ConfigDict(extra="allow", protected_namespaces=()) def __getitem__(self, key): - return self.__dict__[key] + return dict(iter(self))[key] def get(self, key, default=None): - return self.__dict__.get(key, default) + return dict(iter(self)).get(key, default) def __contains__(self, key) -> bool: - return key in self.__dict__ + return key in dict(iter(self)) def items(self): - return self.__dict__.items() + return dict(iter(self)).items() class HiddenParams(OpenAIObject): diff --git a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py index 673e880a1c0..8a7bd3b1d3f 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -4509,6 +4509,34 @@ class TestBuildBlockSseChunks: assert completed["output"][0]["content"][0]["text"] == "Blocked by policy." assert completed["usage"] == {"input_tokens": 7, "output_tokens": 21, "total_tokens": 28} + def test_continuation_numbers_every_event_after_the_last_one_relayed(self): + yielded = [ + {"type": "response.created", "sequence_number": 0, "response": {"id": "resp_live", "output": []}}, + { + "type": "response.output_item.added", + "sequence_number": 1, + "output_index": 0, + "item": {"id": "msg_orig", "type": "message", "role": "assistant"}, + }, + { + "type": "response.output_text.delta", + "sequence_number": 2, + "item_id": "msg_orig", + "output_index": 0, + "content_index": 0, + "delta": "partial", + }, + ] + + payloads = self._payloads( + OpenAIResponsesHandler().build_block_sse_chunks( + self._exc(original_response=yielded), stream_started=True, responses_so_far=yielded + ) + ) + + assert payloads[-1]["type"] == "response.completed" + assert [payload.get("sequence_number") for payload in payloads] == list(range(3, 3 + len(payloads))) + def test_continuation_reads_usage_from_typed_completed_event(self): from litellm.types.llms.openai import ( ResponseCompletedEvent, diff --git a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index c92c1ec4ee1..9c9b3fc4809 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -2019,3 +2019,12 @@ async def test_every_event_serializes_its_position_in_the_stream_from_zero(strea events: Final = list(iterator) if sync_mode else [event async for event in iterator] assert _serialized_sequence_numbers(events) == list(range(len(events))) + + +def test_bridged_events_answer_dict_reads_of_their_number(): + events: Final = list(_build_iterator((_chunk("Hello"), _chunk("!", finish_reason="stop")))) + positions: Final = list(range(len(events))) + + assert [event["sequence_number"] for event in events] == positions + assert [event.get("sequence_number") for event in events] == positions + assert all("sequence_number" in event for event in events) diff --git a/tests/unit/types/llms/test_types_llms_base.py b/tests/unit/types/llms/test_types_llms_base.py index 964689cdeeb..07b641fb392 100644 --- a/tests/unit/types/llms/test_types_llms_base.py +++ b/tests/unit/types/llms/test_types_llms_base.py @@ -9,7 +9,7 @@ from typing import Final import pytest from pydantic import ConfigDict, create_model -from litellm.types.llms.base import LiteLLMBaseModel +from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject, LiteLLMBaseModel _PROBE: Final = """ from litellm.types.llms.base import LiteLLMBaseModel @@ -142,3 +142,30 @@ def test_concurrent_first_use_of_a_deferred_subclass_always_builds_that_subclass assert _concurrent_first_use_outcomes(child) == {child.__name__}, f"round {round_id}" finally: sys.setswitchinterval(previous_switch_interval) + + +class _ResponseEvent(BaseLiteLLMOpenAIResponseObject): + type: str + + +def test_response_object_answers_dict_reads_for_undeclared_fields(): + event: Final = _ResponseEvent(type="response.created", sequence_number=3) + + assert event["type"] == "response.created" + assert event["sequence_number"] == 3 + assert event.get("sequence_number") == 3 + assert "sequence_number" in event + assert dict(event.items()) == {"type": "response.created", "sequence_number": 3} + + +def test_response_object_answers_dict_reads_for_an_undeclared_field_named_keys(): + assert _ResponseEvent(type="response.created", keys="value")["keys"] == "value" + + +def test_response_object_dict_reads_still_miss_absent_keys(): + event: Final = _ResponseEvent(type="response.created") + + assert event.get("sequence_number", "fallback") == "fallback" + assert "sequence_number" not in event + with pytest.raises(KeyError): + event["sequence_number"] From af590f01ecc21a6bedf0e57d5da430dfa7f3af57 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 23:38:37 -0700 Subject: [PATCH 4/6] fix(responses): scope dict reads that see undeclared fields to Responses stream events Moves the extras-aware __getitem__, get, __contains__ and items from the shared response base onto a new base for the 37 stream event classes, so usage, Gemini, Interactions and moderation objects keep reading __dict__ and Gemini responses keep _hidden_params out of .get --- litellm/types/llms/base.py | 8 +- litellm/types/llms/openai.py | 100 +++++++++++------- ...test_openai_responses_guardrail_handler.py | 4 +- .../unit/responses/test_streaming_iterator.py | 16 +-- tests/unit/test_router/test_router.py | 49 ++++++++- tests/unit/types/llms/test_types_llms_base.py | 29 +---- .../unit/types/llms/test_types_llms_openai.py | 31 ++++++ 7 files changed, 154 insertions(+), 83 deletions(-) diff --git a/litellm/types/llms/base.py b/litellm/types/llms/base.py index 37937892e84..5a2da071cec 100644 --- a/litellm/types/llms/base.py +++ b/litellm/types/llms/base.py @@ -32,16 +32,16 @@ class BaseLiteLLMOpenAIResponseObject(LiteLLMBaseModel): model_config = ConfigDict(extra="allow", protected_namespaces=()) def __getitem__(self, key): - return dict(iter(self))[key] + return self.__dict__[key] def get(self, key, default=None): - return dict(iter(self)).get(key, default) + return self.__dict__.get(key, default) def __contains__(self, key) -> bool: - return key in dict(iter(self)) + return key in self.__dict__ def items(self): - return dict(iter(self)).items() + return self.__dict__.items() class HiddenParams(OpenAIObject): diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index a54d66c4beb..3e6e32ec3ec 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1,8 +1,9 @@ import builtins import math -from collections.abc import Iterable, Mapping, Sequence +from collections.abc import ItemsView, Iterable, Mapping, Sequence from enum import Enum from os import PathLike +from types import MappingProxyType from typing import IO, TYPE_CHECKING, Any, Final, Generic, Literal, Optional, TypeAlias, Union import httpx @@ -1748,17 +1749,40 @@ class ResponsesAPIStreamEvents(str, Enum): ERROR = "error" -class ResponseCreatedEvent(BaseLiteLLMOpenAIResponseObject): +_NO_EXTRAS: Final[Mapping[str, object]] = MappingProxyType({}) + + +class BaseResponsesStreamEvent(BaseLiteLLMOpenAIResponseObject): # frozen-ok: guardrails rewrite events in place + def __getitem__(self, key: str) -> object: + return self._mapping_holding(key)[key] + + def get(self, key: str, default: object = None) -> object: + return self._mapping_holding(key).get(key, default) + + def __contains__(self, key: object) -> bool: + return key in self.__dict__ or key in self._extras() + + def items(self) -> ItemsView[str, object]: + return MappingProxyType({**self.__dict__, **self._extras()}).items() + + def _extras(self) -> Mapping[str, object]: + return self.__pydantic_extra__ or _NO_EXTRAS + + def _mapping_holding(self, key: str) -> Mapping[str, object]: + return self.__dict__ if key in self.__dict__ else self._extras() + + +class ResponseCreatedEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.RESPONSE_CREATED] response: ResponsesAPIResponse -class ResponseInProgressEvent(BaseLiteLLMOpenAIResponseObject): +class ResponseInProgressEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.RESPONSE_IN_PROGRESS] response: ResponsesAPIResponse -class ResponseCompletedEvent(BaseLiteLLMOpenAIResponseObject): +class ResponseCompletedEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.RESPONSE_COMPLETED] response: ResponsesAPIResponse _hidden_params: dict = PrivateAttr(default_factory=dict) @@ -1772,12 +1796,12 @@ class ResponseCompletedEvent(BaseLiteLLMOpenAIResponseObject): self._hidden_params = hidden_params -class ResponseFailedEvent(BaseLiteLLMOpenAIResponseObject): +class ResponseFailedEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.RESPONSE_FAILED] response: ResponsesAPIResponse -class ResponseIncompleteEvent(BaseLiteLLMOpenAIResponseObject): +class ResponseIncompleteEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE] response: ResponsesAPIResponse @@ -1785,7 +1809,7 @@ class ResponseIncompleteEvent(BaseLiteLLMOpenAIResponseObject): ResponsesTerminalEvent: TypeAlias = ResponseCompletedEvent | ResponseIncompleteEvent | ResponseFailedEvent -class ResponsePartAddedEvent(BaseLiteLLMOpenAIResponseObject): +class ResponsePartAddedEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.RESPONSE_PART_ADDED] item_id: str output_index: int @@ -1793,7 +1817,7 @@ class ResponsePartAddedEvent(BaseLiteLLMOpenAIResponseObject): part: dict -class ReasoningSummaryTextDeltaEvent(BaseLiteLLMOpenAIResponseObject): +class ReasoningSummaryTextDeltaEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.REASONING_SUMMARY_TEXT_DELTA] item_id: str output_index: int @@ -1801,7 +1825,7 @@ class ReasoningSummaryTextDeltaEvent(BaseLiteLLMOpenAIResponseObject): delta: str -class ReasoningSummaryTextDoneEvent(BaseLiteLLMOpenAIResponseObject): +class ReasoningSummaryTextDoneEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.REASONING_SUMMARY_TEXT_DONE] item_id: str output_index: int @@ -1810,7 +1834,7 @@ class ReasoningSummaryTextDoneEvent(BaseLiteLLMOpenAIResponseObject): text: str -class ReasoningSummaryPartDoneEvent(BaseLiteLLMOpenAIResponseObject): +class ReasoningSummaryPartDoneEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.REASONING_SUMMARY_PART_DONE] item_id: str output_index: int @@ -1819,13 +1843,13 @@ class ReasoningSummaryPartDoneEvent(BaseLiteLLMOpenAIResponseObject): part: BaseLiteLLMOpenAIResponseObject -class OutputItemAddedEvent(BaseLiteLLMOpenAIResponseObject): +class OutputItemAddedEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED] output_index: int item: BaseLiteLLMOpenAIResponseObject | None -class OutputItemDoneEvent(BaseLiteLLMOpenAIResponseObject): +class OutputItemDoneEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE] output_index: int sequence_number: int = 1 @@ -1845,7 +1869,7 @@ class OpenAIChatCompletionLogprobsContent(TypedDict, total=False): top_logprobs: list[OpenAIChatCompletionLogprobsContentTopLogprobs] -class ContentPartAddedEvent(BaseLiteLLMOpenAIResponseObject): +class ContentPartAddedEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.CONTENT_PART_ADDED] item_id: str output_index: int @@ -1873,7 +1897,7 @@ class ContentPartDonePartReasoningText(BaseLiteLLMOpenAIResponseObject): PART_UNION_TYPES = ContentPartDonePartOutputText | ContentPartDonePartRefusal | ContentPartDonePartReasoningText -class ContentPartDoneEvent(BaseLiteLLMOpenAIResponseObject): +class ContentPartDoneEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.CONTENT_PART_DONE] item_id: str output_index: int @@ -1881,7 +1905,7 @@ class ContentPartDoneEvent(BaseLiteLLMOpenAIResponseObject): part: PART_UNION_TYPES -class OutputTextDeltaEvent(BaseLiteLLMOpenAIResponseObject): +class OutputTextDeltaEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA] item_id: str output_index: int @@ -1889,7 +1913,7 @@ class OutputTextDeltaEvent(BaseLiteLLMOpenAIResponseObject): delta: str -class OutputTextAnnotationAddedEvent(BaseLiteLLMOpenAIResponseObject): +class OutputTextAnnotationAddedEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED] item_id: str output_index: int @@ -1898,7 +1922,7 @@ class OutputTextAnnotationAddedEvent(BaseLiteLLMOpenAIResponseObject): annotation: dict -class OutputTextDoneEvent(BaseLiteLLMOpenAIResponseObject): +class OutputTextDoneEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE] item_id: str output_index: int @@ -1906,7 +1930,7 @@ class OutputTextDoneEvent(BaseLiteLLMOpenAIResponseObject): text: str -class RefusalDeltaEvent(BaseLiteLLMOpenAIResponseObject): +class RefusalDeltaEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.REFUSAL_DELTA] item_id: str output_index: int @@ -1914,7 +1938,7 @@ class RefusalDeltaEvent(BaseLiteLLMOpenAIResponseObject): delta: str -class RefusalDoneEvent(BaseLiteLLMOpenAIResponseObject): +class RefusalDoneEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.REFUSAL_DONE] item_id: str output_index: int @@ -1922,72 +1946,72 @@ class RefusalDoneEvent(BaseLiteLLMOpenAIResponseObject): refusal: str -class FunctionCallArgumentsDeltaEvent(BaseLiteLLMOpenAIResponseObject): +class FunctionCallArgumentsDeltaEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA] item_id: str output_index: int delta: str -class FunctionCallArgumentsDoneEvent(BaseLiteLLMOpenAIResponseObject): +class FunctionCallArgumentsDoneEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DONE] item_id: str output_index: int arguments: str -class FileSearchCallInProgressEvent(BaseLiteLLMOpenAIResponseObject): +class FileSearchCallInProgressEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.FILE_SEARCH_CALL_IN_PROGRESS] output_index: int item_id: str -class FileSearchCallSearchingEvent(BaseLiteLLMOpenAIResponseObject): +class FileSearchCallSearchingEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.FILE_SEARCH_CALL_SEARCHING] output_index: int item_id: str -class FileSearchCallCompletedEvent(BaseLiteLLMOpenAIResponseObject): +class FileSearchCallCompletedEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.FILE_SEARCH_CALL_COMPLETED] output_index: int item_id: str -class WebSearchCallInProgressEvent(BaseLiteLLMOpenAIResponseObject): +class WebSearchCallInProgressEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS] output_index: int item_id: str -class WebSearchCallSearchingEvent(BaseLiteLLMOpenAIResponseObject): +class WebSearchCallSearchingEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING] output_index: int item_id: str -class WebSearchCallCompletedEvent(BaseLiteLLMOpenAIResponseObject): +class WebSearchCallCompletedEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED] output_index: int item_id: str # MCP List Tools Events -class MCPListToolsInProgressEvent(BaseLiteLLMOpenAIResponseObject): +class MCPListToolsInProgressEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.MCP_LIST_TOOLS_IN_PROGRESS] sequence_number: int output_index: int item_id: str -class MCPListToolsCompletedEvent(BaseLiteLLMOpenAIResponseObject): +class MCPListToolsCompletedEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.MCP_LIST_TOOLS_COMPLETED] sequence_number: int output_index: int item_id: str -class MCPListToolsFailedEvent(BaseLiteLLMOpenAIResponseObject): +class MCPListToolsFailedEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.MCP_LIST_TOOLS_FAILED] sequence_number: int output_index: int @@ -1995,14 +2019,14 @@ class MCPListToolsFailedEvent(BaseLiteLLMOpenAIResponseObject): # MCP Call Events -class MCPCallInProgressEvent(BaseLiteLLMOpenAIResponseObject): +class MCPCallInProgressEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.MCP_CALL_IN_PROGRESS] sequence_number: int output_index: int item_id: str -class MCPCallArgumentsDeltaEvent(BaseLiteLLMOpenAIResponseObject): +class MCPCallArgumentsDeltaEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.MCP_CALL_ARGUMENTS_DELTA] output_index: int item_id: str @@ -2010,7 +2034,7 @@ class MCPCallArgumentsDeltaEvent(BaseLiteLLMOpenAIResponseObject): sequence_number: int -class MCPCallArgumentsDoneEvent(BaseLiteLLMOpenAIResponseObject): +class MCPCallArgumentsDoneEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.MCP_CALL_ARGUMENTS_DONE] output_index: int item_id: str @@ -2018,21 +2042,21 @@ class MCPCallArgumentsDoneEvent(BaseLiteLLMOpenAIResponseObject): sequence_number: int -class MCPCallCompletedEvent(BaseLiteLLMOpenAIResponseObject): +class MCPCallCompletedEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.MCP_CALL_COMPLETED] sequence_number: int item_id: str output_index: int -class MCPCallFailedEvent(BaseLiteLLMOpenAIResponseObject): +class MCPCallFailedEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.MCP_CALL_FAILED] sequence_number: int item_id: str output_index: int -class ImageGenerationPartialImageEvent(BaseLiteLLMOpenAIResponseObject): +class ImageGenerationPartialImageEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.IMAGE_GENERATION_PARTIAL_IMAGE] partial_image_index: int b64_json: str @@ -2047,7 +2071,7 @@ class ErrorEventError(BaseLiteLLMOpenAIResponseObject): param: str | dict[str, object] | None = None -class ErrorEvent(BaseLiteLLMOpenAIResponseObject): +class ErrorEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.ERROR] sequence_number: int error: ErrorEventError | None = None @@ -2056,7 +2080,7 @@ class ErrorEvent(BaseLiteLLMOpenAIResponseObject): param: str | Mapping[str, object] | None = None -class GenericEvent(BaseLiteLLMOpenAIResponseObject): +class GenericEvent(BaseResponsesStreamEvent): type: str model_config = ConfigDict(extra="allow", protected_namespaces=()) diff --git a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py index c0277b59c16..40510a3d35c 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -4516,7 +4516,7 @@ class TestBuildBlockSseChunks: assert completed["usage"] == {"input_tokens": 7, "output_tokens": 21, "total_tokens": 28} def test_continuation_numbers_every_event_after_the_last_one_relayed(self): - yielded = [ + yielded: Final = [ {"type": "response.created", "sequence_number": 0, "response": {"id": "resp_live", "output": []}}, { "type": "response.output_item.added", @@ -4534,7 +4534,7 @@ class TestBuildBlockSseChunks: }, ] - payloads = self._payloads( + payloads: Final = self._payloads( OpenAIResponsesHandler().build_block_sse_chunks( self._exc(original_response=yielded), stream_started=True, responses_so_far=yielded ) diff --git a/tests/unit/responses/test_streaming_iterator.py b/tests/unit/responses/test_streaming_iterator.py index 46389878423..8b73c69033a 100644 --- a/tests/unit/responses/test_streaming_iterator.py +++ b/tests/unit/responses/test_streaming_iterator.py @@ -2867,13 +2867,15 @@ async def test_websocket_turns_logging_survives_garbage_collection(terminal_even ids=["message", "reasoning_and_tool"], ) def test_synthetic_stream_serializes_every_event_position_from_zero(output: list[dict[str, object]]): - transformed: Final = ResponsesAPIResponse( - id="resp_seq", - created_at=int(datetime.now().timestamp()), - status="completed", - model="claude-sonnet-5-5", - object="response", - output=output, + transformed: Final = ResponsesAPIResponse.model_validate( + { + "id": "resp_seq", + "created_at": int(datetime.now().timestamp()), + "status": "completed", + "model": "claude-sonnet-5-5", + "object": "response", + "output": output, + } ) events: Final = streaming_module.build_synthetic_response_events( diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index ead4898950b..b8ff0e8cd3f 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -6791,8 +6791,8 @@ async def test_aresponses_stream_keeps_numbering_after_a_mid_stream_fallback(): from litellm.exceptions import MidStreamFallbackError - router = _InjectedFallbackRouter(_AsyncList([delta(0, "c"), delta(1, "d")])) - primary = _make_responses_iterator( + router: Final = _InjectedFallbackRouter(_AsyncList([delta(0, "c"), delta(1, "d")])) + primary: Final = _make_responses_iterator( chunks=[delta(0, "a"), delta(1, "b")], error=MidStreamFallbackError( message="primary died", @@ -6804,15 +6804,56 @@ async def test_aresponses_stream_keeps_numbering_after_a_mid_stream_fallback(): model="primary", ) - wrapped = await router._aresponses_streaming_iterator( + wrapped: Final = await router._aresponses_streaming_iterator( response=primary, initial_kwargs={"model": "primary", "stream": True, "input": "Hi", "original_generic_function": litellm.aresponses}, ) - events = [event async for event in wrapped] + events: Final = [event async for event in wrapped] assert [(event.delta, event.sequence_number) for event in events] == [("a", 0), ("b", 1), ("c", 2), ("d", 3)] + +@pytest.mark.asyncio +async def test_aresponses_stream_fallback_keeps_higher_numbers_and_passes_unnumbered_events(): + from litellm.exceptions import MidStreamFallbackError + from litellm.types.llms.openai import OutputTextDeltaEvent, ResponsesAPIStreamEvents + + def delta(text: str) -> OutputTextDeltaEvent: + return OutputTextDeltaEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, item_id="msg_1", output_index=0, content_index=0, delta=text + ) + + def numbered(sequence_number: int, text: str) -> OutputTextDeltaEvent: + return delta(text).model_copy(update={"sequence_number": sequence_number}) + + router: Final = _InjectedFallbackRouter(_AsyncList([numbered(7, "c"), delta("d"), numbered(8, "e")])) + primary: Final = _make_responses_iterator( + chunks=[numbered(0, "a"), numbered(1, "b")], + error=MidStreamFallbackError( + message="primary died", + model="primary", + llm_provider="anthropic", + is_pre_first_chunk=False, + generated_content="ab", + ), + model="primary", + ) + + wrapped: Final = await router._aresponses_streaming_iterator( + response=primary, + initial_kwargs={"model": "primary", "stream": True, "input": "Hi", "original_generic_function": litellm.aresponses}, + ) + events: Final = [event async for event in wrapped] + + assert [(event.delta, event.get("sequence_number")) for event in events] == [ + ("a", 0), + ("b", 1), + ("c", 7), + ("d", None), + ("e", 8), + ] + @pytest.mark.asyncio async def test_aresponses_streaming_iterator_unwraps_content_policy_trigger_for_fallback_dispatch(): """The fallback dispatch matches on the trigger's own type, so the wrapper must hand it the diff --git a/tests/unit/types/llms/test_types_llms_base.py b/tests/unit/types/llms/test_types_llms_base.py index 07b641fb392..964689cdeeb 100644 --- a/tests/unit/types/llms/test_types_llms_base.py +++ b/tests/unit/types/llms/test_types_llms_base.py @@ -9,7 +9,7 @@ from typing import Final import pytest from pydantic import ConfigDict, create_model -from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject, LiteLLMBaseModel +from litellm.types.llms.base import LiteLLMBaseModel _PROBE: Final = """ from litellm.types.llms.base import LiteLLMBaseModel @@ -142,30 +142,3 @@ def test_concurrent_first_use_of_a_deferred_subclass_always_builds_that_subclass assert _concurrent_first_use_outcomes(child) == {child.__name__}, f"round {round_id}" finally: sys.setswitchinterval(previous_switch_interval) - - -class _ResponseEvent(BaseLiteLLMOpenAIResponseObject): - type: str - - -def test_response_object_answers_dict_reads_for_undeclared_fields(): - event: Final = _ResponseEvent(type="response.created", sequence_number=3) - - assert event["type"] == "response.created" - assert event["sequence_number"] == 3 - assert event.get("sequence_number") == 3 - assert "sequence_number" in event - assert dict(event.items()) == {"type": "response.created", "sequence_number": 3} - - -def test_response_object_answers_dict_reads_for_an_undeclared_field_named_keys(): - assert _ResponseEvent(type="response.created", keys="value")["keys"] == "value" - - -def test_response_object_dict_reads_still_miss_absent_keys(): - event: Final = _ResponseEvent(type="response.created") - - assert event.get("sequence_number", "fallback") == "fallback" - assert "sequence_number" not in event - with pytest.raises(KeyError): - event["sequence_number"] diff --git a/tests/unit/types/llms/test_types_llms_openai.py b/tests/unit/types/llms/test_types_llms_openai.py index 1b7a9693f7b..1ce916ce070 100644 --- a/tests/unit/types/llms/test_types_llms_openai.py +++ b/tests/unit/types/llms/test_types_llms_openai.py @@ -16,8 +16,10 @@ from litellm.types.llms.openai import ( OpenAIChatCompletionLogprobsContent, OpenAIModerationResponse, OpenAIVideoObject, + OutputTextDeltaEvent, ResponseCompletedEvent, ResponsesAPIResponse, + ResponsesAPIStreamEvents, ) @@ -799,3 +801,32 @@ def test_rebuilding_a_dumped_responses_answer_keeps_the_tool_search_call_keys(ex rebuilt: Final = ResponsesAPIResponse(**answer.model_dump()) assert rebuilt.model_dump()["output"] == [search_call] + + +def _text_delta() -> OutputTextDeltaEvent: + return OutputTextDeltaEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, item_id="msg_1", output_index=0, content_index=0, delta="hi" + ) + + +def test_stream_event_dict_reads_see_a_field_set_after_construction(): + event: Final = _text_delta().model_copy(update={"sequence_number": 3}) + + assert event["sequence_number"] == 3 + assert event.get("sequence_number") == 3 + assert "sequence_number" in event + assert dict(event.items())["sequence_number"] == 3 + assert event["delta"] == "hi" + + +def test_stream_event_dict_reads_see_an_undeclared_field_named_keys(): + assert _text_delta().model_copy(update={"keys": "value"})["keys"] == "value" + + +def test_stream_event_dict_reads_still_miss_absent_keys(): + event: Final = _text_delta() + + assert event.get("sequence_number", "fallback") == "fallback" + assert "sequence_number" not in event + with pytest.raises(KeyError): + event["sequence_number"] From b3fe12ecbfc20cc4c9702e30a12f345be6611a7f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:42:37 -0700 Subject: [PATCH 5/6] fix(router): renumber Responses stream events only after a fallback takes over A stream that never falls back now keeps its provider's own sequence numbers, repeats included, so a starting_after resume still matches them. After a fallback the wrapper continues above the highest number the client saw --- litellm/router.py | 3 ++ tests/unit/test_router/test_router.py | 66 +++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/litellm/router.py b/litellm/router.py index 9a2dcb4aa8f..7c3f0314f10 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -4416,6 +4416,9 @@ class Router(DeprecatedRouterMembers): sequence_number: Final = getattr(chunk, "sequence_number", None) if not isinstance(sequence_number, int): return chunk + if not self.fallback_headers_adopted: + self._last_sequence_number = max(self._last_sequence_number, sequence_number) + return chunk self._last_sequence_number = max(self._last_sequence_number + 1, sequence_number) if self._last_sequence_number == sequence_number: return chunk diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index b8ff0e8cd3f..1f0bf2b419e 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -6854,6 +6854,72 @@ async def test_aresponses_stream_fallback_keeps_higher_numbers_and_passes_unnumb ("e", 8), ] + +@pytest.mark.asyncio +async def test_aresponses_stream_without_fallback_passes_upstream_numbers_unchanged(): + from litellm.types.llms.openai import OutputTextDeltaEvent, ResponsesAPIStreamEvents + + def delta(sequence_number: int, text: str) -> OutputTextDeltaEvent: + return OutputTextDeltaEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, + item_id="msg_1", + output_index=0, + content_index=0, + delta=text, + sequence_number=sequence_number, + ) + + router: Final = _InjectedFallbackRouter(_AsyncList([])) + primary: Final = _make_responses_iterator( + chunks=[delta(0, "a"), delta(1, "b"), delta(1, "c"), delta(0, "d")], model="primary" + ) + + wrapped: Final = await router._aresponses_streaming_iterator( + response=primary, + initial_kwargs={"model": "primary", "stream": True, "input": "Hi", "original_generic_function": litellm.aresponses}, + ) + events: Final = [event async for event in wrapped] + + assert [(event.delta, event.sequence_number) for event in events] == [("a", 0), ("b", 1), ("c", 1), ("d", 0)] + + +@pytest.mark.asyncio +async def test_aresponses_stream_fallback_continues_above_the_highest_number_the_primary_sent(): + from litellm.exceptions import MidStreamFallbackError + from litellm.types.llms.openai import OutputTextDeltaEvent, ResponsesAPIStreamEvents + + def delta(sequence_number: int, text: str) -> OutputTextDeltaEvent: + return OutputTextDeltaEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, + item_id="msg_1", + output_index=0, + content_index=0, + delta=text, + sequence_number=sequence_number, + ) + + router: Final = _InjectedFallbackRouter(_AsyncList([delta(0, "d")])) + primary: Final = _make_responses_iterator( + chunks=[delta(0, "a"), delta(3, "b"), delta(1, "c")], + error=MidStreamFallbackError( + message="primary died", + model="primary", + llm_provider="anthropic", + is_pre_first_chunk=False, + generated_content="abc", + ), + model="primary", + ) + + wrapped: Final = await router._aresponses_streaming_iterator( + response=primary, + initial_kwargs={"model": "primary", "stream": True, "input": "Hi", "original_generic_function": litellm.aresponses}, + ) + events: Final = [event async for event in wrapped] + + assert [(event.delta, event.sequence_number) for event in events] == [("a", 0), ("b", 3), ("c", 1), ("d", 4)] + + @pytest.mark.asyncio async def test_aresponses_streaming_iterator_unwraps_content_policy_trigger_for_fallback_dispatch(): """The fallback dispatch matches on the trigger's own type, so the wrapper must hand it the From b4a1f09b902c40dfcbd5f36c5eccc1dbfd935ffa Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 03:55:12 -0700 Subject: [PATCH 6/6] fix(guardrails): keep Responses stream numbers unique when LLM Shield flushes held text The restorer's flush copied the last delta and kept its sequence_number, so a native stream repeated a number and a bridged stream sent the flush unnumbered. A flush now takes the done event's slot and later events move up by one, and a stream the provider never closed numbers its flush after the last event. Also pins the created_at in the streaming iterator test fixture instead of reading the clock. --- .../llm_shield_proxy/stream_restorers.py | 30 ++++++++++--- .../guardrail_hooks/test_llm_shield_proxy.py | 42 +++++++++++++++++++ .../unit/responses/test_streaming_iterator.py | 2 +- 3 files changed, 68 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/stream_restorers.py b/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/stream_restorers.py index ef007155c46..66a2d427944 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/stream_restorers.py +++ b/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/stream_restorers.py @@ -261,7 +261,8 @@ class ResponsesStreamRestorer: own window, keyed by the family, the item id and the part index. - Any `*.done` event closes the stream of the same family. Whatever its window still holds goes out first, as a copy of that stream's last delta event -- so it carries - the stream's own ids, and repeats that event's `sequence_number`. Then every text + the stream's own ids. It takes the done event's `sequence_number`, and every later + event's number moves up by one, so the numbers stay unique and in order. Then every text field on the done event is restored in full: its string fields other than identifiers, plus any `part` or `item` it repeats. - `response.completed` / `response.incomplete` repeat the whole reply, and are @@ -273,6 +274,8 @@ class ResponsesStreamRestorer: self._rehydrate: Final = rehydrate self._carries: Final[dict[ResponsesStreamKey, str]] = {} # mutable-ok: per-stream windows advanced in place. self._last_deltas: Final[dict[ResponsesStreamKey, object]] = {} # mutable-ok: newest delta per stream. + self._inserted: int = 0 + self._last_sequence_number: int | None = None async def restore(self, event: object) -> tuple[object, ...]: """The events to emit in place of `event`: any flush, then the event itself.""" @@ -281,7 +284,7 @@ class ResponsesStreamRestorer: return (event,) if kind.endswith(".delta") and kind not in RESPONSES_BINARY_DELTAS: await self._restore_delta(event, kind) - return (event,) + return self._numbered((), event) slots: Final[SlotSink] = [] flushed: Final = await self._flush(responses_stream_key(event, kind)) if kind.endswith(".done") else () if kind.endswith(".done"): @@ -294,12 +297,29 @@ class ResponsesStreamRestorer: for item in read_list(read_field(event, "response"), "output"): collect_response_item(item, slots) await rehydrate_slots(slots, self._rehydrate) - return (*flushed, event) + return self._numbered(flushed, event) async def finish(self) -> tuple[object, ...]: """Flushes every stream the provider never closed, e.g. a truncated reply.""" - flushed: Final = tuple([await self._flush(key) for key in tuple(self._carries)]) - return tuple(itertools.chain.from_iterable(flushed)) + flushed: Final = tuple(itertools.chain.from_iterable([await self._flush(key) for key in tuple(self._carries)])) + last: Final = self._last_sequence_number + if last is not None: + for offset, flush in enumerate(flushed, start=1): + write_field(flush, "sequence_number", last + offset) + return flushed + + def _numbered(self, flushed: tuple[object, ...], event: object) -> tuple[object, ...]: + """Gives each flush the slot `event` arrived in and moves `event` and later events up.""" + number: Final = read_field(event, "sequence_number") + if not isinstance(number, int): + return (*flushed, event) + for offset, flush in enumerate(flushed): + write_field(flush, "sequence_number", number + self._inserted + offset) + self._inserted += len(flushed) + self._last_sequence_number = number + self._inserted + if self._inserted: + write_field(event, "sequence_number", self._last_sequence_number) + return (*flushed, event) async def _restore_delta(self, event: object, kind: str) -> None: text: Final = read_field(event, "delta") diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_llm_shield_proxy.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_llm_shield_proxy.py index 3da77a8707e..2603cf92f30 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_llm_shield_proxy.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_llm_shield_proxy.py @@ -1694,6 +1694,48 @@ class TestResponsesStreamRestoration: assert "".join(e.delta for e in out) == "see a@example.com a[" + @pytest.mark.asyncio + async def test_a_flushed_delta_takes_the_done_events_number_and_later_events_move_up(self): + guardrail, _ = _shielded(self.VAULT) + done = OutputTextDoneEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE, + item_id="msg_1", + output_index=0, + content_index=0, + text="Mail [EMAIL_1] x[", + sequence_number=3, + ) + + out = await _restore_stream( + guardrail, + [ + self._text_delta("Mail [EMA", 1), + self._text_delta("IL_1] x[", 2), + done, + self._text_delta("next", 4, content_index=1), + ], + ) + + assert [(e.type, e.sequence_number) for e in out] == [ + (ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, 1), + (ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, 2), + (ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, 3), + (ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE, 4), + (ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, 5), + ] + assert out[2].delta == "[" + + @pytest.mark.asyncio + async def test_a_truncated_stream_flush_continues_after_the_last_number(self): + guardrail, _ = _shielded(self.VAULT) + + out = await _restore_stream( + guardrail, [self._text_delta("see [EMAIL_1] a[", 7), self._text_delta(" b", 8, content_index=1)] + ) + + assert [e.sequence_number for e in out] == [7, 8, 9] + assert out[-1].delta == "[" + @pytest.mark.asyncio async def test_completed_response_is_restored(self): """The terminal event repeats the whole reply, and clients read it as the answer.""" diff --git a/tests/unit/responses/test_streaming_iterator.py b/tests/unit/responses/test_streaming_iterator.py index 8b73c69033a..11c1ddb8cb1 100644 --- a/tests/unit/responses/test_streaming_iterator.py +++ b/tests/unit/responses/test_streaming_iterator.py @@ -2870,7 +2870,7 @@ def test_synthetic_stream_serializes_every_event_position_from_zero(output: list transformed: Final = ResponsesAPIResponse.model_validate( { "id": "resp_seq", - "created_at": int(datetime.now().timestamp()), + "created_at": 1741476542, "status": "completed", "model": "claude-sonnet-5-5", "object": "response",