fix(streaming): keep vLLM fields when a stream's only choice index is not 0

This commit is contained in:
mateo-berri 2026-11-02 12:43:12 -08:00
parent 41a51d0b9e
commit 6b37cc3261
2 changed files with 18 additions and 4 deletions

View file

@ -508,7 +508,7 @@ class ChunkProcessor:
"index": 0,
"message": {"role": role, "content": ""},
"finish_reason": finish_reason,
**_streamed_vllm_choice_fields(_first_choices_of_index_zero(chunks)),
**_streamed_vllm_choice_fields(_first_choices(chunks)),
}
]
),
@ -1510,9 +1510,8 @@ def _field_of(value: object, name: str) -> object:
return getattr(value, name, None)
def _first_choices_of_index_zero(chunks: Iterable[object]) -> tuple[object, ...]:
first_choices: Final = (next(iter(object_items(_field_of(chunk, "choices"))), None) for chunk in chunks)
return tuple(choice for choice in first_choices if _field_of(choice, "index") in (0, None))
def _first_choices(chunks: Iterable[object]) -> tuple[object, ...]:
return tuple(next(iter(object_items(_field_of(chunk, "choices"))), None) for chunk in chunks)
def _choice_extras(choice: object) -> Mapping[str, object]:

View file

@ -2789,6 +2789,21 @@ def test_stream_chunk_builder_keeps_the_vllm_stop_reason_and_every_token_id_of_e
]
def test_stream_chunk_builder_keeps_the_vllm_fields_of_a_sole_choice_streamed_on_a_nonzero_index() -> None:
built: Final = stream_chunk_builder(
chunks=[
_vllm_chat_chunk(1, "7,", stop_reason=None, token_ids=[22, 11]),
_vllm_chat_chunk(1, "8", "stop", stop_reason=",9", token_ids=[23]),
]
)
assert built is not None
assert {field: built.model_dump()["choices"][0].get(field) for field in ("stop_reason", "token_ids")} == {
"stop_reason": ",9",
"token_ids": [22, 11, 23],
}
def test_stream_chunk_builder_keeps_null_vllm_fields_and_adds_none_for_other_providers() -> None:
vllm_built: Final = stream_chunk_builder(
chunks=[