From 6a0fab449844265a871992d5e25ac4c93d6db506 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 26 May 2026 14:59:46 -0700 Subject: [PATCH 01/10] fix(responses): emit sequence_number, text.format, and usage details on Responses API stream MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The /v1/responses streaming surface had three OpenAI-spec violations that strict deserializers (Grok Build CLI, OpenAI SDK) reject: 1. sequence_number was dropped from the wire. The completion-transformation streaming iterator set event.__dict__["sequence_number"] = N after constructing the Pydantic model, but that mutates the instance dict without updating __pydantic_extra__, so model_dump() silently omits the field. 13 mutation sites converted to constructor kwargs; an additional 5 event constructors that never set sequence_number at all (OutputTextDone, ContentPartDone, OutputTextAnnotationAdded, ReasoningSummaryTextDelta, ResponseCompleted) now do. 2. response.completed.response.text was hardcoded to {}. The spec requires text.format to be present. Default to {"format": {"type": "text"}} when the request didn't supply a text config; honor the request value otherwise. 3. usage.input_tokens_details and usage.output_tokens_details were dropped when upstream didn't provide prompt_tokens_details / completion_tokens_details. The spec requires both fields on every response.completed.usage. Both now always emit, defaulting to zero-valued details (cached_tokens=0, reasoning_tokens=0). This unblocks Anthropic chat-completion routes, which omit completion_tokens_details on tool-only function-call responses. Verified end-to-end with Grok Build CLI 0.2.2 → LiteLLM → claude-haiku-4-5: grok -p 'hi' --model litellm-haiku --output-format json returns clean JSON with exit=0 and zero stderr errors. Confirmed across single-word, arithmetic, multi-line, and 'grok models' catalog probes. Tests: - 4 contract-pinning usage tests updated to assert the new always-emit shape. - 1 streaming-tool-call test that asserted the buggy __dict__ pattern updated to read sequence_number from model_dump() (the actual wire payload). - 1 new regression test pins the Grok contract: every streaming event must carry sequence_number in model_dump() output, monotonic and unique. --- .../streaming_iterator.py | 51 ++++++++++------- .../transformation.py | 49 +++++++---------- .../test_litellm_completion_responses.py | 16 ++++-- ...test_tool_call_streaming_transformation.py | 55 +++++++++++++++++-- .../test_responses_api_bridge_non_stream.py | 26 ++++++--- 5 files changed, 130 insertions(+), 67 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 767281d43ab..4637cea0817 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -219,8 +219,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): "status": "in_progress", } ), + sequence_number=self._sequence_number, ) - event.__dict__["sequence_number"] = self._sequence_number self._pending_tool_events.append(event) if fn_args_delta: @@ -232,16 +232,19 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): for i in range(0, len(fn_args_delta), chunk_size): delta_chunk = fn_args_delta[i : i + chunk_size] self._sequence_number += 1 + # sequence_number is passed via constructor so it lands in + # __pydantic_extra__ (BaseLiteLLMOpenAIResponseObject allows extra + # fields). Setting it via __dict__ afterward is silently dropped + # by model_dump() and breaks strict OpenAI Responses-API clients. delta_event: BaseLiteLLMOpenAIResponseObject = ( FunctionCallArgumentsDeltaEvent( type=ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA, item_id=call_id, output_index=output_index, delta=delta_chunk, + sequence_number=self._sequence_number, ) ) - # Add sequence_number as extra field (BaseLiteLLMOpenAIResponseObject allows extra fields) - delta_event.__dict__["sequence_number"] = self._sequence_number self._pending_tool_events.append(delta_event) def _queue_final_tool_call_done_events( @@ -306,8 +309,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): "status": "in_progress", } ), + sequence_number=self._sequence_number, ) - event.__dict__["sequence_number"] = self._sequence_number self._pending_tool_events.append(event) final_args = fn_args or self._tool_args_by_call_id.get(call_id, "") @@ -328,8 +331,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): item_id=call_id, output_index=output_index, delta=delta_chunk, + sequence_number=self._sequence_number, ) - delta_event.__dict__["sequence_number"] = self._sequence_number self._pending_tool_events.append(delta_event) self._sequence_number += 1 @@ -338,8 +341,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): item_id=call_id, output_index=output_index, arguments=final_args, + sequence_number=self._sequence_number, ) - done_event.__dict__["sequence_number"] = self._sequence_number self._pending_tool_events.append(done_event) self._sequence_number += 1 @@ -424,30 +427,28 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): """ response_created_event_data = self._default_response_created_event_data() self._sequence_number += 1 - event = ResponseCreatedEvent( + return ResponseCreatedEvent( type=ResponsesAPIStreamEvents.RESPONSE_CREATED, response=ResponsesAPIResponse(**response_created_event_data), + sequence_number=self._sequence_number, ) - event.__dict__["sequence_number"] = self._sequence_number - return event def create_response_in_progress_event(self) -> ResponseInProgressEvent: response_in_progress_event_data = self._default_response_created_event_data() response_in_progress_event_data["status"] = "in_progress" self._sequence_number += 1 - event = ResponseInProgressEvent( + return ResponseInProgressEvent( type=ResponsesAPIStreamEvents.RESPONSE_IN_PROGRESS, response=ResponsesAPIResponse(**response_in_progress_event_data), + sequence_number=self._sequence_number, ) - event.__dict__["sequence_number"] = self._sequence_number - return event def create_output_item_added_event(self) -> OutputItemAddedEvent: if self._cached_item_id is None: self._cached_item_id = f"msg_{str(uuid.uuid4())}" self._sequence_number += 1 - event = OutputItemAddedEvent( + return OutputItemAddedEvent( type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, output_index=0, item=BaseLiteLLMOpenAIResponseObject( @@ -459,16 +460,15 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): "content": [], } ), + sequence_number=self._sequence_number, ) - event.__dict__["sequence_number"] = self._sequence_number - return event def create_content_part_added_event(self) -> ContentPartAddedEvent: if self._cached_item_id is None: self._cached_item_id = f"msg_{str(uuid.uuid4())}" self._sequence_number += 1 - event = ContentPartAddedEvent( + return ContentPartAddedEvent( type=ResponsesAPIStreamEvents.CONTENT_PART_ADDED, item_id=self._cached_item_id, output_index=0, @@ -476,9 +476,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): part=BaseLiteLLMOpenAIResponseObject( **{"type": "output_text", "text": "", "annotations": []} ), + sequence_number=self._sequence_number, ) - event.__dict__["sequence_number"] = self._sequence_number - return event def _merge_provider_specific_fields(self, src: dict) -> None: """Merge provider_specific_fields using last-value-wins for lists. @@ -599,6 +598,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if self._cached_item_id is None: self._cached_item_id = f"msg_{str(uuid.uuid4())}" + self._sequence_number += 1 return OutputTextDoneEvent( type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE, item_id=self._cached_item_id, @@ -606,6 +606,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): content_index=0, text=getattr(litellm_complete_object.choices[0].message, "content", "") # type: ignore or "", + sequence_number=self._sequence_number, ) def create_output_content_part_done_event( @@ -636,12 +637,14 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): logprobs=None, ) + self._sequence_number += 1 return ContentPartDoneEvent( type=ResponsesAPIStreamEvents.CONTENT_PART_DONE, item_id=self._cached_item_id, output_index=0, content_index=0, part=part, + sequence_number=self._sequence_number, ) def create_output_item_done_event( @@ -817,8 +820,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): "summary": None, } ), + sequence_number=self._sequence_number, ) - event.__dict__["sequence_number"] = self._sequence_number self._pending_response_events.append(event) return @@ -842,8 +845,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): "content": [], } ), + sequence_number=self._sequence_number, ) - event.__dict__["sequence_number"] = self._sequence_number self._pending_response_events.append(event) # Emit content_part.added immediately after output_item.added for message @@ -1075,6 +1078,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if hasattr(annotation, "model_dump") else dict(annotation) ) + self._sequence_number += 1 event = OutputTextAnnotationAddedEvent( type=ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED, item_id=item_id, @@ -1082,6 +1086,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): content_index=0, annotation_index=idx, annotation=annotation_dict, + sequence_number=self._sequence_number, ) self._pending_annotation_events.append(event) # Priority 1: Handle reasoning content (highest priority) @@ -1092,11 +1097,13 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): ): reasoning_content = chunk.choices[0].delta.reasoning_content + self._sequence_number += 1 return ReasoningSummaryTextDeltaEvent( type=ResponsesAPIStreamEvents.REASONING_SUMMARY_TEXT_DELTA, item_id=f"rs_{hash(str(reasoning_content))}", output_index=0, delta=reasoning_content, + sequence_number=self._sequence_number, ) # Priority 2: Handle text deltas @@ -1109,8 +1116,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): output_index=0, content_index=0, delta=delta_content, + sequence_number=self._sequence_number, ) - text_delta_event.__dict__["sequence_number"] = self._sequence_number return text_delta_event # Priority 3: Handle tool call deltas (if any) -> queue events and emit them @@ -1191,9 +1198,11 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): litellm_metadata=self.litellm_metadata, ) + self._sequence_number += 1 return ResponseCompletedEvent( type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, response=encoded_response, + sequence_number=self._sequence_number, ) else: return None diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index e2ba8353591..a305f04db72 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -1692,7 +1692,10 @@ class LiteLLMCompletionResponsesConfig: status=LiteLLMCompletionResponsesConfig._map_chat_completion_finish_reason_to_responses_status( finish_reason ), - text={}, + # OpenAI Responses spec requires `text.format` to be present. Honor the + # request's `text` config when supplied; otherwise default to plain text. + # Strict deserializers (Grok Build CLI, OpenAI SDK) reject `text: {}`. + text=responses_api_request.get("text") or {"format": {"type": "text"}}, truncation=getattr(chat_completion_response, "truncation", None), usage=LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage( chat_completion_response=chat_completion_response @@ -2091,21 +2094,18 @@ class LiteLLMCompletionResponsesConfig: if hasattr(usage, "cost") and usage.cost is not None: setattr(response_usage, "cost", usage.cost) - # Translate prompt_tokens_details to input_tokens_details - if ( - hasattr(usage, "prompt_tokens_details") - and usage.prompt_tokens_details is not None - ): - prompt_details = usage.prompt_tokens_details - input_details_dict: Dict[str, int] = {} - + # Translate prompt_tokens_details -> input_tokens_details. + # OpenAI's Responses spec requires this field to always be present on + # response.completed.usage, so we emit it with defaults if upstream + # omitted prompt_tokens_details entirely. + prompt_details = getattr(usage, "prompt_tokens_details", None) + input_details_dict: Dict[str, int] = {"cached_tokens": 0} + if prompt_details is not None: if ( hasattr(prompt_details, "cached_tokens") and prompt_details.cached_tokens is not None ): input_details_dict["cached_tokens"] = prompt_details.cached_tokens - else: - input_details_dict["cached_tokens"] = 0 if ( hasattr(prompt_details, "text_tokens") @@ -2119,18 +2119,14 @@ class LiteLLMCompletionResponsesConfig: ): input_details_dict["audio_tokens"] = prompt_details.audio_tokens - if input_details_dict: - response_usage.input_tokens_details = InputTokensDetails( - **input_details_dict - ) + response_usage.input_tokens_details = InputTokensDetails(**input_details_dict) - # Translate completion_tokens_details to output_tokens_details - if ( - hasattr(usage, "completion_tokens_details") - and usage.completion_tokens_details is not None - ): - completion_details = usage.completion_tokens_details - output_details_dict: Dict[str, int] = {} + # Translate completion_tokens_details -> output_tokens_details. Same + # always-present requirement; strict deserializers (Grok Build CLI, the + # OpenAI SDK) reject a usage object without it. + completion_details = getattr(usage, "completion_tokens_details", None) + output_details_dict: Dict[str, int] = {"reasoning_tokens": 0} + if completion_details is not None: if ( hasattr(completion_details, "reasoning_tokens") and completion_details.reasoning_tokens is not None @@ -2138,8 +2134,6 @@ class LiteLLMCompletionResponsesConfig: output_details_dict["reasoning_tokens"] = ( completion_details.reasoning_tokens ) - else: - output_details_dict["reasoning_tokens"] = 0 if ( hasattr(completion_details, "text_tokens") @@ -2153,10 +2147,9 @@ class LiteLLMCompletionResponsesConfig: ): output_details_dict["image_tokens"] = completion_details.image_tokens - if output_details_dict: - response_usage.output_tokens_details = OutputTokensDetails( - **output_details_dict - ) + response_usage.output_tokens_details = OutputTokensDetails( + **output_details_dict + ) return response_usage diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 503a610e016..d132644d0b9 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -1601,7 +1601,13 @@ class TestUsageTransformation: assert response_usage.input_tokens_details.text_tokens == 9 def test_transform_usage_without_details(self): - """Test transformation when prompt_tokens_details and completion_tokens_details are None""" + """Test transformation when prompt_tokens_details and completion_tokens_details are None. + + Per the OpenAI Responses spec, `input_tokens_details` and + `output_tokens_details` must always be present on the response.completed + usage object; strict deserializers (Grok Build CLI, OpenAI SDK) reject + the event when either is missing. We emit them with zero defaults. + """ # Setup: Usage without details (basic usage only) usage = Usage( prompt_tokens=9, @@ -1629,12 +1635,14 @@ class TestUsageTransformation: chat_completion_response=chat_completion_response ) - # Assert: Basic usage should still be transformed, but details should be None + # Assert: basic counts transformed, and details always present with zero defaults assert response_usage.input_tokens == 9 assert response_usage.output_tokens == 27 assert response_usage.total_tokens == 36 - assert response_usage.input_tokens_details is None - assert response_usage.output_tokens_details is None + assert response_usage.input_tokens_details is not None + assert response_usage.input_tokens_details.cached_tokens == 0 + assert response_usage.output_tokens_details is not None + assert response_usage.output_tokens_details.reasoning_tokens == 0 def test_transform_usage_with_image_tokens(self): """Test that image_tokens from Vertex AI/Gemini are properly transformed to output_tokens_details""" diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_tool_call_streaming_transformation.py b/tests/test_litellm/responses/litellm_completion_transformation/test_tool_call_streaming_transformation.py index fa6f42609ca..8872f62ab7c 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_tool_call_streaming_transformation.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_tool_call_streaming_transformation.py @@ -187,11 +187,15 @@ def test_tool_call_arguments_are_chunked_to_match_openai_behavior(): # Process the chunk once - it queues all events internally evt = iterator._transform_chat_completion_chunk_to_response_api_chunk(chunk) - # First event should be OUTPUT_ITEM_ADDED + # First event should be OUTPUT_ITEM_ADDED, carrying a serialized sequence_number. + # `sequence_number` is passed through the constructor so it lands in + # __pydantic_extra__ (BaseLiteLLMOpenAIResponseObject allows extras); a raw + # __dict__ assignment would be silently dropped by model_dump(). We assert on + # the dump because that is what hits the wire as SSE. assert evt is not None assert evt.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED assert evt.output_index == 1 - assert hasattr(evt, "__dict__") and "sequence_number" in evt.__dict__ + assert "sequence_number" in evt.model_dump() # Collect all remaining delta events from the pending queue by creating empty chunks delta_events = [] @@ -220,19 +224,21 @@ def test_tool_call_arguments_are_chunked_to_match_openai_behavior(): # Verify multiple delta events were created (at least 6 chunks for 67 chars) assert len(delta_events) >= 6 # 67 chars split into chunks of max 10 chars each - # Verify each delta is at most 10 characters + # Verify each delta is at most 10 characters and carries a serializable + # sequence_number (the value lives in __pydantic_extra__, not __dict__). for evt in delta_events: assert len(evt.delta) <= 10 assert evt.item_id == "call_test" assert evt.output_index == 1 - assert hasattr(evt, "__dict__") and "sequence_number" in evt.__dict__ + assert "sequence_number" in evt.model_dump() # Verify all deltas concatenated equal the original arguments concatenated = "".join(evt.delta for evt in delta_events) assert concatenated == large_arguments - # Verify sequence numbers are increasing - sequence_numbers = [evt.__dict__["sequence_number"] for evt in delta_events] + # Verify sequence numbers are increasing and unique (read from the dumped + # payload, which is what actually hits the wire). + sequence_numbers = [evt.model_dump()["sequence_number"] for evt in delta_events] assert sequence_numbers == sorted(sequence_numbers) assert len(set(sequence_numbers)) == len(sequence_numbers) # All unique @@ -397,3 +403,40 @@ def test_reused_index_with_new_call_id_marks_fallback_ambiguous(): assert arguments_by_call_id["call_b"] == '{"b":' assert arguments_by_call_id["call_a"] != '{"a":1}' assert arguments_by_call_id["call_b"] != '{"b":1}' + + +def test_streaming_events_serialize_sequence_number_for_strict_clients(): + """Pin the Grok Build CLI / OpenAI SDK contract: every emitted streaming + event must carry `sequence_number` in its on-wire JSON payload. + + Earlier code set `event.__dict__["sequence_number"] = N` after construction, + which silently dropped the field during Pydantic's `model_dump()` and broke + strict deserializers. The fix routes the value through the constructor so it + lands in `__pydantic_extra__` and survives serialization. + """ + iterator = LiteLLMCompletionStreamingIterator( + model="test-model", + litellm_custom_stream_wrapper=AsyncMock(), + request_input="hi", + responses_api_request={}, + ) + + created = iterator.create_response_created_event() + in_progress = iterator.create_response_in_progress_event() + item_added = iterator.create_output_item_added_event() + part_added = iterator.create_content_part_added_event() + + for evt in (created, in_progress, item_added, part_added): + dumped = evt.model_dump() + assert ( + "sequence_number" in dumped + ), f"{type(evt).__name__} dropped sequence_number" + assert isinstance(dumped["sequence_number"], int) + + # And the values must be monotonic — strict clients enforce ordering. + seqs = [ + evt.model_dump()["sequence_number"] + for evt in (created, in_progress, item_added, part_added) + ] + assert seqs == sorted(seqs), seqs + assert len(set(seqs)) == len(seqs), seqs diff --git a/tests/test_litellm/test_responses_api_bridge_non_stream.py b/tests/test_litellm/test_responses_api_bridge_non_stream.py index 8905293d6b6..d539470bcf7 100644 --- a/tests/test_litellm/test_responses_api_bridge_non_stream.py +++ b/tests/test_litellm/test_responses_api_bridge_non_stream.py @@ -131,7 +131,11 @@ def test_transform_usage_no_token_details(): """ Test that transformation works when completion response has NO token details. - This simulates providers that don't return detailed token breakdowns. + This simulates providers that don't return detailed token breakdowns. Per the + OpenAI Responses spec, `input_tokens_details` and `output_tokens_details` must + always be present on `response.completed.usage`; strict deserializers (Grok + Build CLI, OpenAI SDK) reject the event when either is missing. Defaults are + zeros. """ completion_response = create_mock_completion_response( model="gpt-4", @@ -150,9 +154,11 @@ def test_transform_usage_no_token_details(): assert responses_usage.output_tokens == 20 assert responses_usage.total_tokens == 30 - # Token details should not be present when not provided - assert responses_usage.input_tokens_details is None - assert responses_usage.output_tokens_details is None + # Token details are always emitted, defaulting to zeros when upstream omits them. + assert isinstance(responses_usage.input_tokens_details, InputTokensDetails) + assert responses_usage.input_tokens_details.cached_tokens == 0 + assert isinstance(responses_usage.output_tokens_details, OutputTokensDetails) + assert responses_usage.output_tokens_details.reasoning_tokens == 0 print("✓ Transformation works with no token details") @@ -186,8 +192,10 @@ def test_transform_usage_with_cached_tokens_only(): assert isinstance(responses_usage.input_tokens_details, InputTokensDetails) assert responses_usage.input_tokens_details.cached_tokens == 80 - # Output details should not be present (no reasoning_tokens provided) - assert responses_usage.output_tokens_details is None + # Output details are always emitted (defaulting reasoning_tokens to 0) to match + # the OpenAI Responses spec — strict deserializers require the field. + assert isinstance(responses_usage.output_tokens_details, OutputTokensDetails) + assert responses_usage.output_tokens_details.reasoning_tokens == 0 print("✓ Transformation works with cached_tokens only") @@ -216,8 +224,10 @@ def test_transform_usage_with_reasoning_tokens_only(): assert responses_usage.output_tokens == 100 assert responses_usage.total_tokens == 150 - # Input details should not be present (no cached_tokens provided) - assert responses_usage.input_tokens_details is None + # Input details are always emitted (defaulting cached_tokens to 0) to match + # the OpenAI Responses spec. + assert isinstance(responses_usage.input_tokens_details, InputTokensDetails) + assert responses_usage.input_tokens_details.cached_tokens == 0 # Output details should be present with reasoning_tokens assert responses_usage.output_tokens_details is not None From 283da839a3a3032d66e7cf661d5a96f90ef87089 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 26 May 2026 22:06:51 +0000 Subject: [PATCH 02/10] fix: increment sequence number in create_output_item_done_event Co-authored-by: Yassin Kortam --- .../litellm_completion_transformation/streaming_iterator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 4637cea0817..fb28468bf8c 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -659,10 +659,11 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): response_annotations = LiteLLMCompletionResponsesConfig._transform_chat_completion_annotations_to_response_output_annotations( annotations=annotations ) + self._sequence_number += 1 return OutputItemDoneEvent( type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, output_index=0, - sequence_number=1, + sequence_number=self._sequence_number, item=BaseLiteLLMOpenAIResponseObject( **{ "id": self._cached_item_id, From d5aa5337fa0ac5abb68bdaef5cfec923e95a4d5c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 26 May 2026 22:18:13 +0000 Subject: [PATCH 03/10] fix(responses): assign annotation event sequence_number at emit time Annotation events were assigned a sequence_number when queued, but emitted later (after text/reasoning/tool deltas from subsequent chunks). This caused the queued annotation events to have lower sequence numbers than events emitted before them, breaking the monotonic ordering guarantee for strict clients. Defer assignment to emit time so sequence_number reflects actual emission order. Co-authored-by: Yassin Kortam --- .../streaming_iterator.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index fb28468bf8c..e73436400b9 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -1079,7 +1079,9 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if hasattr(annotation, "model_dump") else dict(annotation) ) - self._sequence_number += 1 + # Sequence number is assigned at emit time (see Priority 4 + # below) to preserve monotonic ordering relative to + # higher-priority events from later chunks. event = OutputTextAnnotationAddedEvent( type=ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED, item_id=item_id, @@ -1087,7 +1089,6 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): content_index=0, annotation_index=idx, annotation=annotation_dict, - sequence_number=self._sequence_number, ) self._pending_annotation_events.append(event) # Priority 1: Handle reasoning content (highest priority) @@ -1134,12 +1135,16 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): return self._pending_tool_events.pop(0) # Priority 4: If we have pending annotation events, emit the next one - # This happens when the current chunk has no text/reasoning content + # This happens when the current chunk has no text/reasoning content. + # Assign the sequence number here (at emit time) so it stays monotonic + # relative to other events emitted from intervening chunks. if ( hasattr(self, "_pending_annotation_events") and self._pending_annotation_events ): event = self._pending_annotation_events.pop(0) + self._sequence_number += 1 + event.sequence_number = self._sequence_number return event # Priority 5: If we have pending tool events (from earlier chunk), emit the next one From a8c00601e7d07ef206553863151f3ca95ea2b898 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 26 May 2026 22:26:57 +0000 Subject: [PATCH 04/10] fix(responses): populate usage details on early-return when usage is None Co-authored-by: Yassin Kortam --- .../litellm_completion_transformation/transformation.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index a305f04db72..3fc1daff14b 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -2082,6 +2082,8 @@ class LiteLLMCompletionResponsesConfig: input_tokens=0, output_tokens=0, total_tokens=0, + input_tokens_details=InputTokensDetails(cached_tokens=0), + output_tokens_details=OutputTokensDetails(reasoning_tokens=0), ) response_usage = ResponseAPIUsage( From 115a081420f27babd77333d78df99bf82d657993 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 27 May 2026 00:21:10 +0000 Subject: [PATCH 05/10] fix(types): declare sequence_number on Responses API streaming events Pydantic-extra storage works at runtime but mypy (with pydantic.mypy) treats undeclared constructor kwargs as call-arg errors. Declare sequence_number explicitly on the event classes used by the Responses API streaming iterator so the type checker accepts the kwarg and allows setattr on OutputTextAnnotationAddedEvent. --- litellm/types/llms/openai.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index abe58199dfd..b770eb6b47d 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1448,16 +1448,19 @@ class ResponsesAPIStreamEvents(str, Enum): class ResponseCreatedEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.RESPONSE_CREATED] response: ResponsesAPIResponse + sequence_number: int = 0 class ResponseInProgressEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.RESPONSE_IN_PROGRESS] response: ResponsesAPIResponse + sequence_number: int = 0 class ResponseCompletedEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.RESPONSE_COMPLETED] response: ResponsesAPIResponse + sequence_number: int = 0 _hidden_params: dict = PrivateAttr(default_factory=dict) @@ -1484,6 +1487,7 @@ class ReasoningSummaryTextDeltaEvent(BaseLiteLLMOpenAIResponseObject): output_index: int summary_index: int = 0 delta: str + sequence_number: int = 0 class ReasoningSummaryTextDoneEvent(BaseLiteLLMOpenAIResponseObject): @@ -1508,6 +1512,7 @@ class OutputItemAddedEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED] output_index: int item: Optional[BaseLiteLLMOpenAIResponseObject] + sequence_number: int = 0 class OutputItemDoneEvent(BaseLiteLLMOpenAIResponseObject): @@ -1536,6 +1541,7 @@ class ContentPartAddedEvent(BaseLiteLLMOpenAIResponseObject): output_index: int content_index: int part: BaseLiteLLMOpenAIResponseObject + sequence_number: int = 0 class ContentPartDonePartOutputText(BaseLiteLLMOpenAIResponseObject): @@ -1568,6 +1574,7 @@ class ContentPartDoneEvent(BaseLiteLLMOpenAIResponseObject): output_index: int content_index: int part: PART_UNION_TYPES + sequence_number: int = 0 class OutputTextDeltaEvent(BaseLiteLLMOpenAIResponseObject): @@ -1576,6 +1583,7 @@ class OutputTextDeltaEvent(BaseLiteLLMOpenAIResponseObject): output_index: int content_index: int delta: str + sequence_number: int = 0 class OutputTextAnnotationAddedEvent(BaseLiteLLMOpenAIResponseObject): @@ -1585,6 +1593,7 @@ class OutputTextAnnotationAddedEvent(BaseLiteLLMOpenAIResponseObject): content_index: int annotation_index: int annotation: dict + sequence_number: int = 0 class OutputTextDoneEvent(BaseLiteLLMOpenAIResponseObject): @@ -1616,6 +1625,7 @@ class FunctionCallArgumentsDeltaEvent(BaseLiteLLMOpenAIResponseObject): item_id: str output_index: int delta: str + sequence_number: int = 0 class FunctionCallArgumentsDoneEvent(BaseLiteLLMOpenAIResponseObject): @@ -1623,6 +1633,7 @@ class FunctionCallArgumentsDoneEvent(BaseLiteLLMOpenAIResponseObject): item_id: str output_index: int arguments: str + sequence_number: int = 0 class FileSearchCallInProgressEvent(BaseLiteLLMOpenAIResponseObject): From 1a6a908a72fa739379eb108c1d723b4ad2ebde12 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 27 May 2026 00:29:11 +0000 Subject: [PATCH 06/10] fix: add sequence_number default field to OutputTextDoneEvent for consistency Co-authored-by: Yassin Kortam --- litellm/types/llms/openai.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index b770eb6b47d..e89fe94f955 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1602,6 +1602,7 @@ class OutputTextDoneEvent(BaseLiteLLMOpenAIResponseObject): output_index: int content_index: int text: str + sequence_number: int = 0 class RefusalDeltaEvent(BaseLiteLLMOpenAIResponseObject): From 0460593466e298e6e489bb9bd47bb5dcc1d59f83 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 27 May 2026 00:36:42 +0000 Subject: [PATCH 07/10] fix(types): align OutputItemDoneEvent sequence_number default to 0 --- litellm/types/llms/openai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index e89fe94f955..535510c2a0a 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1518,7 +1518,7 @@ class OutputItemAddedEvent(BaseLiteLLMOpenAIResponseObject): class OutputItemDoneEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE] output_index: int - sequence_number: int = 1 + sequence_number: int = 0 item: BaseLiteLLMOpenAIResponseObject From f1c33c4d1c2c9253b1e2b7172e411f25357887c8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 27 May 2026 00:45:29 +0000 Subject: [PATCH 08/10] fix(mcp): set sequence_number on mcp_call output_item.done event create_mcp_call_events relied on OutputItemDoneEvent's default sequence_number, which was previously 1 but is now 0. Pass an explicit value so the event matches the streaming sequence order emitted by the surrounding MCP call events. --- litellm/responses/mcp/mcp_streaming_iterator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/responses/mcp/mcp_streaming_iterator.py b/litellm/responses/mcp/mcp_streaming_iterator.py index 42c46dff47c..246caa584cd 100644 --- a/litellm/responses/mcp/mcp_streaming_iterator.py +++ b/litellm/responses/mcp/mcp_streaming_iterator.py @@ -218,6 +218,7 @@ def create_mcp_call_events( output_item_done_event = OutputItemDoneEvent( type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, output_index=0, + sequence_number=sequence_start + 4, item=BaseLiteLLMOpenAIResponseObject( **{ "id": item_id, From 3b3da793daebd6e52e3b3122fe2f018b26b818b4 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 27 May 2026 00:48:23 +0000 Subject: [PATCH 09/10] fix(responses): assign monotonic sequence_number to all synthetic events Several Responses API event types now declare sequence_number with a default of 0. The native synthetic streaming path constructed many events without explicitly passing sequence_number, which caused them to serialize as 0 and break the strict monotonic ordering guarantee expected by some clients (e.g. Grok Build CLI). Assign sequence_number to every event after the list is built so the stream is guaranteed monotonic regardless of which helper produced each event. Co-authored-by: Yassin Kortam --- litellm/responses/streaming_iterator.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index c4e72cb7dc5..a7eecf53dfe 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -1212,6 +1212,18 @@ def _build_synthetic_response_events( response=transformed, ) ) + + # Assign monotonic sequence_number to every event. The helpers above + # build events without consistently passing sequence_number, and several + # event types now declare `sequence_number: int = 0` (default), which + # would otherwise serialize as 0 for most events and break the strict + # monotonic ordering guarantee expected by some Responses API clients. + for idx, event in enumerate(events): + try: + event.sequence_number = idx + except (AttributeError, ValueError): + pass + return events From 7f52f18118027cc12d5bb6289de79cc5c1dd157f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 27 May 2026 00:57:26 +0000 Subject: [PATCH 10/10] fix(responses): start synthetic stream sequence numbers at 1 Co-authored-by: Yassin Kortam --- litellm/responses/streaming_iterator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index a7eecf53dfe..d86230be910 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -1218,7 +1218,7 @@ def _build_synthetic_response_events( # event types now declare `sequence_number: int = 0` (default), which # would otherwise serialize as 0 for most events and break the strict # monotonic ordering guarantee expected by some Responses API clients. - for idx, event in enumerate(events): + for idx, event in enumerate(events, start=1): try: event.sequence_number = idx except (AttributeError, ValueError):