diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 767281d43ab..4637cea0817 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -219,8 +219,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): "status": "in_progress", } ), + sequence_number=self._sequence_number, ) - event.__dict__["sequence_number"] = self._sequence_number self._pending_tool_events.append(event) if fn_args_delta: @@ -232,16 +232,19 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): for i in range(0, len(fn_args_delta), chunk_size): delta_chunk = fn_args_delta[i : i + chunk_size] self._sequence_number += 1 + # sequence_number is passed via constructor so it lands in + # __pydantic_extra__ (BaseLiteLLMOpenAIResponseObject allows extra + # fields). Setting it via __dict__ afterward is silently dropped + # by model_dump() and breaks strict OpenAI Responses-API clients. delta_event: BaseLiteLLMOpenAIResponseObject = ( FunctionCallArgumentsDeltaEvent( type=ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA, item_id=call_id, output_index=output_index, delta=delta_chunk, + sequence_number=self._sequence_number, ) ) - # Add sequence_number as extra field (BaseLiteLLMOpenAIResponseObject allows extra fields) - delta_event.__dict__["sequence_number"] = self._sequence_number self._pending_tool_events.append(delta_event) def _queue_final_tool_call_done_events( @@ -306,8 +309,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): "status": "in_progress", } ), + sequence_number=self._sequence_number, ) - event.__dict__["sequence_number"] = self._sequence_number self._pending_tool_events.append(event) final_args = fn_args or self._tool_args_by_call_id.get(call_id, "") @@ -328,8 +331,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): item_id=call_id, output_index=output_index, delta=delta_chunk, + sequence_number=self._sequence_number, ) - delta_event.__dict__["sequence_number"] = self._sequence_number self._pending_tool_events.append(delta_event) self._sequence_number += 1 @@ -338,8 +341,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): item_id=call_id, output_index=output_index, arguments=final_args, + sequence_number=self._sequence_number, ) - done_event.__dict__["sequence_number"] = self._sequence_number self._pending_tool_events.append(done_event) self._sequence_number += 1 @@ -424,30 +427,28 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): """ response_created_event_data = self._default_response_created_event_data() self._sequence_number += 1 - event = ResponseCreatedEvent( + return ResponseCreatedEvent( type=ResponsesAPIStreamEvents.RESPONSE_CREATED, response=ResponsesAPIResponse(**response_created_event_data), + sequence_number=self._sequence_number, ) - event.__dict__["sequence_number"] = self._sequence_number - return event def create_response_in_progress_event(self) -> ResponseInProgressEvent: response_in_progress_event_data = self._default_response_created_event_data() response_in_progress_event_data["status"] = "in_progress" self._sequence_number += 1 - event = ResponseInProgressEvent( + return ResponseInProgressEvent( type=ResponsesAPIStreamEvents.RESPONSE_IN_PROGRESS, response=ResponsesAPIResponse(**response_in_progress_event_data), + sequence_number=self._sequence_number, ) - event.__dict__["sequence_number"] = self._sequence_number - return event def create_output_item_added_event(self) -> OutputItemAddedEvent: if self._cached_item_id is None: self._cached_item_id = f"msg_{str(uuid.uuid4())}" self._sequence_number += 1 - event = OutputItemAddedEvent( + return OutputItemAddedEvent( type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, output_index=0, item=BaseLiteLLMOpenAIResponseObject( @@ -459,16 +460,15 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): "content": [], } ), + sequence_number=self._sequence_number, ) - event.__dict__["sequence_number"] = self._sequence_number - return event def create_content_part_added_event(self) -> ContentPartAddedEvent: if self._cached_item_id is None: self._cached_item_id = f"msg_{str(uuid.uuid4())}" self._sequence_number += 1 - event = ContentPartAddedEvent( + return ContentPartAddedEvent( type=ResponsesAPIStreamEvents.CONTENT_PART_ADDED, item_id=self._cached_item_id, output_index=0, @@ -476,9 +476,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): part=BaseLiteLLMOpenAIResponseObject( **{"type": "output_text", "text": "", "annotations": []} ), + sequence_number=self._sequence_number, ) - event.__dict__["sequence_number"] = self._sequence_number - return event def _merge_provider_specific_fields(self, src: dict) -> None: """Merge provider_specific_fields using last-value-wins for lists. @@ -599,6 +598,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if self._cached_item_id is None: self._cached_item_id = f"msg_{str(uuid.uuid4())}" + self._sequence_number += 1 return OutputTextDoneEvent( type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE, item_id=self._cached_item_id, @@ -606,6 +606,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): content_index=0, text=getattr(litellm_complete_object.choices[0].message, "content", "") # type: ignore or "", + sequence_number=self._sequence_number, ) def create_output_content_part_done_event( @@ -636,12 +637,14 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): logprobs=None, ) + self._sequence_number += 1 return ContentPartDoneEvent( type=ResponsesAPIStreamEvents.CONTENT_PART_DONE, item_id=self._cached_item_id, output_index=0, content_index=0, part=part, + sequence_number=self._sequence_number, ) def create_output_item_done_event( @@ -817,8 +820,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): "summary": None, } ), + sequence_number=self._sequence_number, ) - event.__dict__["sequence_number"] = self._sequence_number self._pending_response_events.append(event) return @@ -842,8 +845,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): "content": [], } ), + sequence_number=self._sequence_number, ) - event.__dict__["sequence_number"] = self._sequence_number self._pending_response_events.append(event) # Emit content_part.added immediately after output_item.added for message @@ -1075,6 +1078,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if hasattr(annotation, "model_dump") else dict(annotation) ) + self._sequence_number += 1 event = OutputTextAnnotationAddedEvent( type=ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED, item_id=item_id, @@ -1082,6 +1086,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): content_index=0, annotation_index=idx, annotation=annotation_dict, + sequence_number=self._sequence_number, ) self._pending_annotation_events.append(event) # Priority 1: Handle reasoning content (highest priority) @@ -1092,11 +1097,13 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): ): reasoning_content = chunk.choices[0].delta.reasoning_content + self._sequence_number += 1 return ReasoningSummaryTextDeltaEvent( type=ResponsesAPIStreamEvents.REASONING_SUMMARY_TEXT_DELTA, item_id=f"rs_{hash(str(reasoning_content))}", output_index=0, delta=reasoning_content, + sequence_number=self._sequence_number, ) # Priority 2: Handle text deltas @@ -1109,8 +1116,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): output_index=0, content_index=0, delta=delta_content, + sequence_number=self._sequence_number, ) - text_delta_event.__dict__["sequence_number"] = self._sequence_number return text_delta_event # Priority 3: Handle tool call deltas (if any) -> queue events and emit them @@ -1191,9 +1198,11 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): litellm_metadata=self.litellm_metadata, ) + self._sequence_number += 1 return ResponseCompletedEvent( type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, response=encoded_response, + sequence_number=self._sequence_number, ) else: return None diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index e2ba8353591..a305f04db72 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -1692,7 +1692,10 @@ class LiteLLMCompletionResponsesConfig: status=LiteLLMCompletionResponsesConfig._map_chat_completion_finish_reason_to_responses_status( finish_reason ), - text={}, + # OpenAI Responses spec requires `text.format` to be present. Honor the + # request's `text` config when supplied; otherwise default to plain text. + # Strict deserializers (Grok Build CLI, OpenAI SDK) reject `text: {}`. + text=responses_api_request.get("text") or {"format": {"type": "text"}}, truncation=getattr(chat_completion_response, "truncation", None), usage=LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage( chat_completion_response=chat_completion_response @@ -2091,21 +2094,18 @@ class LiteLLMCompletionResponsesConfig: if hasattr(usage, "cost") and usage.cost is not None: setattr(response_usage, "cost", usage.cost) - # Translate prompt_tokens_details to input_tokens_details - if ( - hasattr(usage, "prompt_tokens_details") - and usage.prompt_tokens_details is not None - ): - prompt_details = usage.prompt_tokens_details - input_details_dict: Dict[str, int] = {} - + # Translate prompt_tokens_details -> input_tokens_details. + # OpenAI's Responses spec requires this field to always be present on + # response.completed.usage, so we emit it with defaults if upstream + # omitted prompt_tokens_details entirely. + prompt_details = getattr(usage, "prompt_tokens_details", None) + input_details_dict: Dict[str, int] = {"cached_tokens": 0} + if prompt_details is not None: if ( hasattr(prompt_details, "cached_tokens") and prompt_details.cached_tokens is not None ): input_details_dict["cached_tokens"] = prompt_details.cached_tokens - else: - input_details_dict["cached_tokens"] = 0 if ( hasattr(prompt_details, "text_tokens") @@ -2119,18 +2119,14 @@ class LiteLLMCompletionResponsesConfig: ): input_details_dict["audio_tokens"] = prompt_details.audio_tokens - if input_details_dict: - response_usage.input_tokens_details = InputTokensDetails( - **input_details_dict - ) + response_usage.input_tokens_details = InputTokensDetails(**input_details_dict) - # Translate completion_tokens_details to output_tokens_details - if ( - hasattr(usage, "completion_tokens_details") - and usage.completion_tokens_details is not None - ): - completion_details = usage.completion_tokens_details - output_details_dict: Dict[str, int] = {} + # Translate completion_tokens_details -> output_tokens_details. Same + # always-present requirement; strict deserializers (Grok Build CLI, the + # OpenAI SDK) reject a usage object without it. + completion_details = getattr(usage, "completion_tokens_details", None) + output_details_dict: Dict[str, int] = {"reasoning_tokens": 0} + if completion_details is not None: if ( hasattr(completion_details, "reasoning_tokens") and completion_details.reasoning_tokens is not None @@ -2138,8 +2134,6 @@ class LiteLLMCompletionResponsesConfig: output_details_dict["reasoning_tokens"] = ( completion_details.reasoning_tokens ) - else: - output_details_dict["reasoning_tokens"] = 0 if ( hasattr(completion_details, "text_tokens") @@ -2153,10 +2147,9 @@ class LiteLLMCompletionResponsesConfig: ): output_details_dict["image_tokens"] = completion_details.image_tokens - if output_details_dict: - response_usage.output_tokens_details = OutputTokensDetails( - **output_details_dict - ) + response_usage.output_tokens_details = OutputTokensDetails( + **output_details_dict + ) return response_usage diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 503a610e016..d132644d0b9 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -1601,7 +1601,13 @@ class TestUsageTransformation: assert response_usage.input_tokens_details.text_tokens == 9 def test_transform_usage_without_details(self): - """Test transformation when prompt_tokens_details and completion_tokens_details are None""" + """Test transformation when prompt_tokens_details and completion_tokens_details are None. + + Per the OpenAI Responses spec, `input_tokens_details` and + `output_tokens_details` must always be present on the response.completed + usage object; strict deserializers (Grok Build CLI, OpenAI SDK) reject + the event when either is missing. We emit them with zero defaults. + """ # Setup: Usage without details (basic usage only) usage = Usage( prompt_tokens=9, @@ -1629,12 +1635,14 @@ class TestUsageTransformation: chat_completion_response=chat_completion_response ) - # Assert: Basic usage should still be transformed, but details should be None + # Assert: basic counts transformed, and details always present with zero defaults assert response_usage.input_tokens == 9 assert response_usage.output_tokens == 27 assert response_usage.total_tokens == 36 - assert response_usage.input_tokens_details is None - assert response_usage.output_tokens_details is None + assert response_usage.input_tokens_details is not None + assert response_usage.input_tokens_details.cached_tokens == 0 + assert response_usage.output_tokens_details is not None + assert response_usage.output_tokens_details.reasoning_tokens == 0 def test_transform_usage_with_image_tokens(self): """Test that image_tokens from Vertex AI/Gemini are properly transformed to output_tokens_details""" diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_tool_call_streaming_transformation.py b/tests/test_litellm/responses/litellm_completion_transformation/test_tool_call_streaming_transformation.py index fa6f42609ca..8872f62ab7c 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_tool_call_streaming_transformation.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_tool_call_streaming_transformation.py @@ -187,11 +187,15 @@ def test_tool_call_arguments_are_chunked_to_match_openai_behavior(): # Process the chunk once - it queues all events internally evt = iterator._transform_chat_completion_chunk_to_response_api_chunk(chunk) - # First event should be OUTPUT_ITEM_ADDED + # First event should be OUTPUT_ITEM_ADDED, carrying a serialized sequence_number. + # `sequence_number` is passed through the constructor so it lands in + # __pydantic_extra__ (BaseLiteLLMOpenAIResponseObject allows extras); a raw + # __dict__ assignment would be silently dropped by model_dump(). We assert on + # the dump because that is what hits the wire as SSE. assert evt is not None assert evt.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED assert evt.output_index == 1 - assert hasattr(evt, "__dict__") and "sequence_number" in evt.__dict__ + assert "sequence_number" in evt.model_dump() # Collect all remaining delta events from the pending queue by creating empty chunks delta_events = [] @@ -220,19 +224,21 @@ def test_tool_call_arguments_are_chunked_to_match_openai_behavior(): # Verify multiple delta events were created (at least 6 chunks for 67 chars) assert len(delta_events) >= 6 # 67 chars split into chunks of max 10 chars each - # Verify each delta is at most 10 characters + # Verify each delta is at most 10 characters and carries a serializable + # sequence_number (the value lives in __pydantic_extra__, not __dict__). for evt in delta_events: assert len(evt.delta) <= 10 assert evt.item_id == "call_test" assert evt.output_index == 1 - assert hasattr(evt, "__dict__") and "sequence_number" in evt.__dict__ + assert "sequence_number" in evt.model_dump() # Verify all deltas concatenated equal the original arguments concatenated = "".join(evt.delta for evt in delta_events) assert concatenated == large_arguments - # Verify sequence numbers are increasing - sequence_numbers = [evt.__dict__["sequence_number"] for evt in delta_events] + # Verify sequence numbers are increasing and unique (read from the dumped + # payload, which is what actually hits the wire). + sequence_numbers = [evt.model_dump()["sequence_number"] for evt in delta_events] assert sequence_numbers == sorted(sequence_numbers) assert len(set(sequence_numbers)) == len(sequence_numbers) # All unique @@ -397,3 +403,40 @@ def test_reused_index_with_new_call_id_marks_fallback_ambiguous(): assert arguments_by_call_id["call_b"] == '{"b":' assert arguments_by_call_id["call_a"] != '{"a":1}' assert arguments_by_call_id["call_b"] != '{"b":1}' + + +def test_streaming_events_serialize_sequence_number_for_strict_clients(): + """Pin the Grok Build CLI / OpenAI SDK contract: every emitted streaming + event must carry `sequence_number` in its on-wire JSON payload. + + Earlier code set `event.__dict__["sequence_number"] = N` after construction, + which silently dropped the field during Pydantic's `model_dump()` and broke + strict deserializers. The fix routes the value through the constructor so it + lands in `__pydantic_extra__` and survives serialization. + """ + iterator = LiteLLMCompletionStreamingIterator( + model="test-model", + litellm_custom_stream_wrapper=AsyncMock(), + request_input="hi", + responses_api_request={}, + ) + + created = iterator.create_response_created_event() + in_progress = iterator.create_response_in_progress_event() + item_added = iterator.create_output_item_added_event() + part_added = iterator.create_content_part_added_event() + + for evt in (created, in_progress, item_added, part_added): + dumped = evt.model_dump() + assert ( + "sequence_number" in dumped + ), f"{type(evt).__name__} dropped sequence_number" + assert isinstance(dumped["sequence_number"], int) + + # And the values must be monotonic — strict clients enforce ordering. + seqs = [ + evt.model_dump()["sequence_number"] + for evt in (created, in_progress, item_added, part_added) + ] + assert seqs == sorted(seqs), seqs + assert len(set(seqs)) == len(seqs), seqs diff --git a/tests/test_litellm/test_responses_api_bridge_non_stream.py b/tests/test_litellm/test_responses_api_bridge_non_stream.py index 8905293d6b6..d539470bcf7 100644 --- a/tests/test_litellm/test_responses_api_bridge_non_stream.py +++ b/tests/test_litellm/test_responses_api_bridge_non_stream.py @@ -131,7 +131,11 @@ def test_transform_usage_no_token_details(): """ Test that transformation works when completion response has NO token details. - This simulates providers that don't return detailed token breakdowns. + This simulates providers that don't return detailed token breakdowns. Per the + OpenAI Responses spec, `input_tokens_details` and `output_tokens_details` must + always be present on `response.completed.usage`; strict deserializers (Grok + Build CLI, OpenAI SDK) reject the event when either is missing. Defaults are + zeros. """ completion_response = create_mock_completion_response( model="gpt-4", @@ -150,9 +154,11 @@ def test_transform_usage_no_token_details(): assert responses_usage.output_tokens == 20 assert responses_usage.total_tokens == 30 - # Token details should not be present when not provided - assert responses_usage.input_tokens_details is None - assert responses_usage.output_tokens_details is None + # Token details are always emitted, defaulting to zeros when upstream omits them. + assert isinstance(responses_usage.input_tokens_details, InputTokensDetails) + assert responses_usage.input_tokens_details.cached_tokens == 0 + assert isinstance(responses_usage.output_tokens_details, OutputTokensDetails) + assert responses_usage.output_tokens_details.reasoning_tokens == 0 print("✓ Transformation works with no token details") @@ -186,8 +192,10 @@ def test_transform_usage_with_cached_tokens_only(): assert isinstance(responses_usage.input_tokens_details, InputTokensDetails) assert responses_usage.input_tokens_details.cached_tokens == 80 - # Output details should not be present (no reasoning_tokens provided) - assert responses_usage.output_tokens_details is None + # Output details are always emitted (defaulting reasoning_tokens to 0) to match + # the OpenAI Responses spec — strict deserializers require the field. + assert isinstance(responses_usage.output_tokens_details, OutputTokensDetails) + assert responses_usage.output_tokens_details.reasoning_tokens == 0 print("✓ Transformation works with cached_tokens only") @@ -216,8 +224,10 @@ def test_transform_usage_with_reasoning_tokens_only(): assert responses_usage.output_tokens == 100 assert responses_usage.total_tokens == 150 - # Input details should not be present (no cached_tokens provided) - assert responses_usage.input_tokens_details is None + # Input details are always emitted (defaulting cached_tokens to 0) to match + # the OpenAI Responses spec. + assert isinstance(responses_usage.input_tokens_details, InputTokensDetails) + assert responses_usage.input_tokens_details.cached_tokens == 0 # Output details should be present with reasoning_tokens assert responses_usage.output_tokens_details is not None