scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-12 23:01:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 89

fix(responses): emit sequence_number, text.format, and usage details on Responses API stream

Browse source 
The /v1/responses streaming surface had three OpenAI-spec violations that
strict deserializers (Grok Build CLI, OpenAI SDK) reject:

1. sequence_number was dropped from the wire. The completion-transformation
   streaming iterator set event.__dict__["sequence_number"] = N after
   constructing the Pydantic model, but that mutates the instance dict
   without updating __pydantic_extra__, so model_dump() silently omits the
   field. 13 mutation sites converted to constructor kwargs; an additional
   5 event constructors that never set sequence_number at all (OutputTextDone,
   ContentPartDone, OutputTextAnnotationAdded, ReasoningSummaryTextDelta,
   ResponseCompleted) now do.

2. response.completed.response.text was hardcoded to {}. The spec requires
   text.format to be present. Default to {"format": {"type": "text"}}
   when the request didn't supply a text config; honor the request value
   otherwise.

3. usage.input_tokens_details and usage.output_tokens_details were dropped
   when upstream didn't provide prompt_tokens_details / completion_tokens_details.
   The spec requires both fields on every response.completed.usage. Both now
   always emit, defaulting to zero-valued details (cached_tokens=0,
   reasoning_tokens=0). This unblocks Anthropic chat-completion routes,
   which omit completion_tokens_details on tool-only function-call responses.

Verified end-to-end with Grok Build CLI 0.2.2 → LiteLLM → claude-haiku-4-5:
grok -p 'hi' --model litellm-haiku --output-format json returns clean JSON
with exit=0 and zero stderr errors. Confirmed across single-word, arithmetic,
multi-line, and 'grok models' catalog probes.

Tests:
- 4 contract-pinning usage tests updated to assert the new always-emit shape.
- 1 streaming-tool-call test that asserted the buggy __dict__ pattern updated
  to read sequence_number from model_dump() (the actual wire payload).
- 1 new regression test pins the Grok contract: every streaming event must
  carry sequence_number in model_dump() output, monotonic and unique.
This commit is contained in:
mateo-berri 2026-05-26 14:59:46 -07:00
parent 123a9ce487
commit 6a0fab4498
5 changed files with 130 additions and 67 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
litellm/responses/litellm_completion_transformation/streaming_iterator.py
View file

@ -219,8 +219,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
"status": "in_progress",
}
),
sequence_number=self._sequence_number,
)
event.__dict__["sequence_number"] = self._sequence_number
self._pending_tool_events.append(event)
if fn_args_delta:
@ -232,16 +232,19 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
for i in range(0, len(fn_args_delta), chunk_size):
delta_chunk = fn_args_delta[i : i + chunk_size]
self._sequence_number += 1
# sequence_number is passed via constructor so it lands in
# __pydantic_extra__ (BaseLiteLLMOpenAIResponseObject allows extra
# fields). Setting it via __dict__ afterward is silently dropped
# by model_dump() and breaks strict OpenAI Responses-API clients.
delta_event: BaseLiteLLMOpenAIResponseObject = (
FunctionCallArgumentsDeltaEvent(
type=ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA,
item_id=call_id,
output_index=output_index,
delta=delta_chunk,
sequence_number=self._sequence_number,
)
)
# Add sequence_number as extra field (BaseLiteLLMOpenAIResponseObject allows extra fields)
delta_event.__dict__["sequence_number"] = self._sequence_number
self._pending_tool_events.append(delta_event)
def _queue_final_tool_call_done_events(
@ -306,8 +309,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
"status": "in_progress",
}
),
sequence_number=self._sequence_number,
)
event.__dict__["sequence_number"] = self._sequence_number
self._pending_tool_events.append(event)
final_args = fn_args or self._tool_args_by_call_id.get(call_id, "")
@ -328,8 +331,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
item_id=call_id,
output_index=output_index,
delta=delta_chunk,
sequence_number=self._sequence_number,
)
delta_event.__dict__["sequence_number"] = self._sequence_number
self._pending_tool_events.append(delta_event)
self._sequence_number += 1
@ -338,8 +341,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
item_id=call_id,
output_index=output_index,
arguments=final_args,
sequence_number=self._sequence_number,
)
done_event.__dict__["sequence_number"] = self._sequence_number
self._pending_tool_events.append(done_event)
self._sequence_number += 1
@ -424,30 +427,28 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
"""
response_created_event_data = self._default_response_created_event_data()
self._sequence_number += 1
event = ResponseCreatedEvent(
return ResponseCreatedEvent(
type=ResponsesAPIStreamEvents.RESPONSE_CREATED,
response=ResponsesAPIResponse(**response_created_event_data),
sequence_number=self._sequence_number,
)
event.__dict__["sequence_number"] = self._sequence_number
return event
def create_response_in_progress_event(self) -> ResponseInProgressEvent:
response_in_progress_event_data = self._default_response_created_event_data()
response_in_progress_event_data["status"] = "in_progress"
self._sequence_number += 1
event = ResponseInProgressEvent(
return ResponseInProgressEvent(
type=ResponsesAPIStreamEvents.RESPONSE_IN_PROGRESS,
response=ResponsesAPIResponse(**response_in_progress_event_data),
sequence_number=self._sequence_number,
)
event.__dict__["sequence_number"] = self._sequence_number
return event
def create_output_item_added_event(self) -> OutputItemAddedEvent:
if self._cached_item_id is None:
self._cached_item_id = f"msg_{str(uuid.uuid4())}"
self._sequence_number += 1
event = OutputItemAddedEvent(
return OutputItemAddedEvent(
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
output_index=0,
item=BaseLiteLLMOpenAIResponseObject(
@ -459,16 +460,15 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
"content": [],
}
),
sequence_number=self._sequence_number,
)
event.__dict__["sequence_number"] = self._sequence_number
return event
def create_content_part_added_event(self) -> ContentPartAddedEvent:
if self._cached_item_id is None:
self._cached_item_id = f"msg_{str(uuid.uuid4())}"
self._sequence_number += 1
event = ContentPartAddedEvent(
return ContentPartAddedEvent(
type=ResponsesAPIStreamEvents.CONTENT_PART_ADDED,
item_id=self._cached_item_id,
output_index=0,
@ -476,9 +476,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
part=BaseLiteLLMOpenAIResponseObject(
**{"type": "output_text", "text": "", "annotations": []}
),
sequence_number=self._sequence_number,
)
event.__dict__["sequence_number"] = self._sequence_number
return event
def _merge_provider_specific_fields(self, src: dict) -> None:
"""Merge provider_specific_fields using last-value-wins for lists.
@ -599,6 +598,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
if self._cached_item_id is None:
self._cached_item_id = f"msg_{str(uuid.uuid4())}"
self._sequence_number += 1
return OutputTextDoneEvent(
type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE,
item_id=self._cached_item_id,
@ -606,6 +606,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
content_index=0,
text=getattr(litellm_complete_object.choices[0].message, "content", "") # type: ignore
or "",
sequence_number=self._sequence_number,
)
def create_output_content_part_done_event(
@ -636,12 +637,14 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
logprobs=None,
)
self._sequence_number += 1
return ContentPartDoneEvent(
type=ResponsesAPIStreamEvents.CONTENT_PART_DONE,
item_id=self._cached_item_id,
output_index=0,
content_index=0,
part=part,
sequence_number=self._sequence_number,
)
def create_output_item_done_event(
@ -817,8 +820,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
"summary": None,
}
),
sequence_number=self._sequence_number,
)
event.__dict__["sequence_number"] = self._sequence_number
self._pending_response_events.append(event)
return
@ -842,8 +845,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
"content": [],
}
),
sequence_number=self._sequence_number,
)
event.__dict__["sequence_number"] = self._sequence_number
self._pending_response_events.append(event)
# Emit content_part.added immediately after output_item.added for message
@ -1075,6 +1078,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
if hasattr(annotation, "model_dump")
else dict(annotation)
)
self._sequence_number += 1
event = OutputTextAnnotationAddedEvent(
type=ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED,
item_id=item_id,
@ -1082,6 +1086,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
content_index=0,
annotation_index=idx,
annotation=annotation_dict,
sequence_number=self._sequence_number,
)
self._pending_annotation_events.append(event)
# Priority 1: Handle reasoning content (highest priority)
@ -1092,11 +1097,13 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
):
reasoning_content = chunk.choices[0].delta.reasoning_content
self._sequence_number += 1
return ReasoningSummaryTextDeltaEvent(
type=ResponsesAPIStreamEvents.REASONING_SUMMARY_TEXT_DELTA,
item_id=f"rs_{hash(str(reasoning_content))}",
output_index=0,
delta=reasoning_content,
sequence_number=self._sequence_number,
)
# Priority 2: Handle text deltas
@ -1109,8 +1116,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
output_index=0,
content_index=0,
delta=delta_content,
sequence_number=self._sequence_number,
)
text_delta_event.__dict__["sequence_number"] = self._sequence_number
return text_delta_event
# Priority 3: Handle tool call deltas (if any) -> queue events and emit them
@ -1191,9 +1198,11 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
litellm_metadata=self.litellm_metadata,
)
self._sequence_number += 1
return ResponseCompletedEvent(
type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED,
response=encoded_response,
sequence_number=self._sequence_number,
)
else:
return None

49
litellm/responses/litellm_completion_transformation/transformation.py
View file

@ -1692,7 +1692,10 @@ class LiteLLMCompletionResponsesConfig:
status=LiteLLMCompletionResponsesConfig._map_chat_completion_finish_reason_to_responses_status(
finish_reason
),
text={},
# OpenAI Responses spec requires `text.format` to be present. Honor the
# request's `text` config when supplied; otherwise default to plain text.
# Strict deserializers (Grok Build CLI, OpenAI SDK) reject `text: {}`.
text=responses_api_request.get("text") or {"format": {"type": "text"}},
truncation=getattr(chat_completion_response, "truncation", None),
usage=LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage(
chat_completion_response=chat_completion_response
@ -2091,21 +2094,18 @@ class LiteLLMCompletionResponsesConfig:
if hasattr(usage, "cost") and usage.cost is not None:
setattr(response_usage, "cost", usage.cost)
# Translate prompt_tokens_details to input_tokens_details
if (
hasattr(usage, "prompt_tokens_details")
and usage.prompt_tokens_details is not None
):
prompt_details = usage.prompt_tokens_details
input_details_dict: Dict[str, int] = {}
# Translate prompt_tokens_details -> input_tokens_details.
# OpenAI's Responses spec requires this field to always be present on
# response.completed.usage, so we emit it with defaults if upstream
# omitted prompt_tokens_details entirely.
prompt_details = getattr(usage, "prompt_tokens_details", None)
input_details_dict: Dict[str, int] = {"cached_tokens": 0}
if prompt_details is not None:
if (
hasattr(prompt_details, "cached_tokens")
and prompt_details.cached_tokens is not None
):
input_details_dict["cached_tokens"] = prompt_details.cached_tokens
else:
input_details_dict["cached_tokens"] = 0
if (
hasattr(prompt_details, "text_tokens")
@ -2119,18 +2119,14 @@ class LiteLLMCompletionResponsesConfig:
):
input_details_dict["audio_tokens"] = prompt_details.audio_tokens
if input_details_dict:
response_usage.input_tokens_details = InputTokensDetails(
**input_details_dict
)
response_usage.input_tokens_details = InputTokensDetails(**input_details_dict)
# Translate completion_tokens_details to output_tokens_details
if (
hasattr(usage, "completion_tokens_details")
and usage.completion_tokens_details is not None
):
completion_details = usage.completion_tokens_details
output_details_dict: Dict[str, int] = {}
# Translate completion_tokens_details -> output_tokens_details. Same
# always-present requirement; strict deserializers (Grok Build CLI, the
# OpenAI SDK) reject a usage object without it.
completion_details = getattr(usage, "completion_tokens_details", None)
output_details_dict: Dict[str, int] = {"reasoning_tokens": 0}
if completion_details is not None:
if (
hasattr(completion_details, "reasoning_tokens")
and completion_details.reasoning_tokens is not None
@ -2138,8 +2134,6 @@ class LiteLLMCompletionResponsesConfig:
output_details_dict["reasoning_tokens"] = (
completion_details.reasoning_tokens
)
else:
output_details_dict["reasoning_tokens"] = 0
if (
hasattr(completion_details, "text_tokens")
@ -2153,10 +2147,9 @@ class LiteLLMCompletionResponsesConfig:
):
output_details_dict["image_tokens"] = completion_details.image_tokens
if output_details_dict:
response_usage.output_tokens_details = OutputTokensDetails(
**output_details_dict
)
response_usage.output_tokens_details = OutputTokensDetails(
**output_details_dict
)
return response_usage

16
tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py
View file

@ -1601,7 +1601,13 @@ class TestUsageTransformation:
assert response_usage.input_tokens_details.text_tokens == 9
def test_transform_usage_without_details(self):
"""Test transformation when prompt_tokens_details and completion_tokens_details are None"""
"""Test transformation when prompt_tokens_details and completion_tokens_details are None.
Per the OpenAI Responses spec, `input_tokens_details` and
`output_tokens_details` must always be present on the response.completed
usage object; strict deserializers (Grok Build CLI, OpenAI SDK) reject
the event when either is missing. We emit them with zero defaults.
"""
# Setup: Usage without details (basic usage only)
usage = Usage(
prompt_tokens=9,
@ -1629,12 +1635,14 @@ class TestUsageTransformation:
chat_completion_response=chat_completion_response
)
# Assert: Basic usage should still be transformed, but details should be None
# Assert: basic counts transformed, and details always present with zero defaults
assert response_usage.input_tokens == 9
assert response_usage.output_tokens == 27
assert response_usage.total_tokens == 36
assert response_usage.input_tokens_details is None
assert response_usage.output_tokens_details is None
assert response_usage.input_tokens_details is not None
assert response_usage.input_tokens_details.cached_tokens == 0
assert response_usage.output_tokens_details is not None
assert response_usage.output_tokens_details.reasoning_tokens == 0
def test_transform_usage_with_image_tokens(self):
"""Test that image_tokens from Vertex AI/Gemini are properly transformed to output_tokens_details"""

55
tests/test_litellm/responses/litellm_completion_transformation/test_tool_call_streaming_transformation.py
View file

@ -187,11 +187,15 @@ def test_tool_call_arguments_are_chunked_to_match_openai_behavior():
# Process the chunk once - it queues all events internally
evt = iterator._transform_chat_completion_chunk_to_response_api_chunk(chunk)
# First event should be OUTPUT_ITEM_ADDED
# First event should be OUTPUT_ITEM_ADDED, carrying a serialized sequence_number.
# `sequence_number` is passed through the constructor so it lands in
# __pydantic_extra__ (BaseLiteLLMOpenAIResponseObject allows extras); a raw
# __dict__ assignment would be silently dropped by model_dump(). We assert on
# the dump because that is what hits the wire as SSE.
assert evt is not None
assert evt.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED
assert evt.output_index == 1
assert hasattr(evt, "__dict__") and "sequence_number" in evt.__dict__
assert "sequence_number" in evt.model_dump()
# Collect all remaining delta events from the pending queue by creating empty chunks
delta_events = []
@ -220,19 +224,21 @@ def test_tool_call_arguments_are_chunked_to_match_openai_behavior():
# Verify multiple delta events were created (at least 6 chunks for 67 chars)
assert len(delta_events) >= 6 # 67 chars split into chunks of max 10 chars each
# Verify each delta is at most 10 characters
# Verify each delta is at most 10 characters and carries a serializable
# sequence_number (the value lives in __pydantic_extra__, not __dict__).
for evt in delta_events:
assert len(evt.delta) <= 10
assert evt.item_id == "call_test"
assert evt.output_index == 1
assert hasattr(evt, "__dict__") and "sequence_number" in evt.__dict__
assert "sequence_number" in evt.model_dump()
# Verify all deltas concatenated equal the original arguments
concatenated = "".join(evt.delta for evt in delta_events)
assert concatenated == large_arguments
# Verify sequence numbers are increasing
sequence_numbers = [evt.__dict__["sequence_number"] for evt in delta_events]
# Verify sequence numbers are increasing and unique (read from the dumped
# payload, which is what actually hits the wire).
sequence_numbers = [evt.model_dump()["sequence_number"] for evt in delta_events]
assert sequence_numbers == sorted(sequence_numbers)
assert len(set(sequence_numbers)) == len(sequence_numbers) # All unique
@ -397,3 +403,40 @@ def test_reused_index_with_new_call_id_marks_fallback_ambiguous():
assert arguments_by_call_id["call_b"] == '{"b":'
assert arguments_by_call_id["call_a"] != '{"a":1}'
assert arguments_by_call_id["call_b"] != '{"b":1}'
def test_streaming_events_serialize_sequence_number_for_strict_clients():
"""Pin the Grok Build CLI / OpenAI SDK contract: every emitted streaming
event must carry `sequence_number` in its on-wire JSON payload.
Earlier code set `event.__dict__["sequence_number"] = N` after construction,
which silently dropped the field during Pydantic's `model_dump()` and broke
strict deserializers. The fix routes the value through the constructor so it
lands in `__pydantic_extra__` and survives serialization.
"""
iterator = LiteLLMCompletionStreamingIterator(
model="test-model",
litellm_custom_stream_wrapper=AsyncMock(),
request_input="hi",
responses_api_request={},
)
created = iterator.create_response_created_event()
in_progress = iterator.create_response_in_progress_event()
item_added = iterator.create_output_item_added_event()
part_added = iterator.create_content_part_added_event()
for evt in (created, in_progress, item_added, part_added):
dumped = evt.model_dump()
assert (
"sequence_number" in dumped
), f"{type(evt).__name__} dropped sequence_number"
assert isinstance(dumped["sequence_number"], int)
# And the values must be monotonic — strict clients enforce ordering.
seqs = [
evt.model_dump()["sequence_number"]
for evt in (created, in_progress, item_added, part_added)
]
assert seqs == sorted(seqs), seqs
assert len(set(seqs)) == len(seqs), seqs

26
tests/test_litellm/test_responses_api_bridge_non_stream.py
View file

@ -131,7 +131,11 @@ def test_transform_usage_no_token_details():
"""
Test that transformation works when completion response has NO token details.
This simulates providers that don't return detailed token breakdowns.
This simulates providers that don't return detailed token breakdowns. Per the
OpenAI Responses spec, `input_tokens_details` and `output_tokens_details` must
always be present on `response.completed.usage`; strict deserializers (Grok
Build CLI, OpenAI SDK) reject the event when either is missing. Defaults are
zeros.
"""
completion_response = create_mock_completion_response(
model="gpt-4",
@ -150,9 +154,11 @@ def test_transform_usage_no_token_details():
assert responses_usage.output_tokens == 20
assert responses_usage.total_tokens == 30
# Token details should not be present when not provided
assert responses_usage.input_tokens_details is None
assert responses_usage.output_tokens_details is None
# Token details are always emitted, defaulting to zeros when upstream omits them.
assert isinstance(responses_usage.input_tokens_details, InputTokensDetails)
assert responses_usage.input_tokens_details.cached_tokens == 0
assert isinstance(responses_usage.output_tokens_details, OutputTokensDetails)
assert responses_usage.output_tokens_details.reasoning_tokens == 0
print("✓ Transformation works with no token details")
@ -186,8 +192,10 @@ def test_transform_usage_with_cached_tokens_only():
assert isinstance(responses_usage.input_tokens_details, InputTokensDetails)
assert responses_usage.input_tokens_details.cached_tokens == 80
# Output details should not be present (no reasoning_tokens provided)
assert responses_usage.output_tokens_details is None
# Output details are always emitted (defaulting reasoning_tokens to 0) to match
# the OpenAI Responses spec — strict deserializers require the field.
assert isinstance(responses_usage.output_tokens_details, OutputTokensDetails)
assert responses_usage.output_tokens_details.reasoning_tokens == 0
print("✓ Transformation works with cached_tokens only")
@ -216,8 +224,10 @@ def test_transform_usage_with_reasoning_tokens_only():
assert responses_usage.output_tokens == 100
assert responses_usage.total_tokens == 150
# Input details should not be present (no cached_tokens provided)
assert responses_usage.input_tokens_details is None
# Input details are always emitted (defaulting cached_tokens to 0) to match
# the OpenAI Responses spec.
assert isinstance(responses_usage.input_tokens_details, InputTokensDetails)
assert responses_usage.input_tokens_details.cached_tokens == 0
# Output details should be present with reasoning_tokens
assert responses_usage.output_tokens_details is not None