mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
fix(streaming): keep vLLM fields when a stream's only choice index is not 0
This commit is contained in:
parent
41a51d0b9e
commit
6b37cc3261
2 changed files with 18 additions and 4 deletions
|
|
@ -508,7 +508,7 @@ class ChunkProcessor:
|
|||
"index": 0,
|
||||
"message": {"role": role, "content": ""},
|
||||
"finish_reason": finish_reason,
|
||||
**_streamed_vllm_choice_fields(_first_choices_of_index_zero(chunks)),
|
||||
**_streamed_vllm_choice_fields(_first_choices(chunks)),
|
||||
}
|
||||
]
|
||||
),
|
||||
|
|
@ -1510,9 +1510,8 @@ def _field_of(value: object, name: str) -> object:
|
|||
return getattr(value, name, None)
|
||||
|
||||
|
||||
def _first_choices_of_index_zero(chunks: Iterable[object]) -> tuple[object, ...]:
|
||||
first_choices: Final = (next(iter(object_items(_field_of(chunk, "choices"))), None) for chunk in chunks)
|
||||
return tuple(choice for choice in first_choices if _field_of(choice, "index") in (0, None))
|
||||
def _first_choices(chunks: Iterable[object]) -> tuple[object, ...]:
|
||||
return tuple(next(iter(object_items(_field_of(chunk, "choices"))), None) for chunk in chunks)
|
||||
|
||||
|
||||
def _choice_extras(choice: object) -> Mapping[str, object]:
|
||||
|
|
|
|||
|
|
@ -2789,6 +2789,21 @@ def test_stream_chunk_builder_keeps_the_vllm_stop_reason_and_every_token_id_of_e
|
|||
]
|
||||
|
||||
|
||||
def test_stream_chunk_builder_keeps_the_vllm_fields_of_a_sole_choice_streamed_on_a_nonzero_index() -> None:
|
||||
built: Final = stream_chunk_builder(
|
||||
chunks=[
|
||||
_vllm_chat_chunk(1, "7,", stop_reason=None, token_ids=[22, 11]),
|
||||
_vllm_chat_chunk(1, "8", "stop", stop_reason=",9", token_ids=[23]),
|
||||
]
|
||||
)
|
||||
|
||||
assert built is not None
|
||||
assert {field: built.model_dump()["choices"][0].get(field) for field in ("stop_reason", "token_ids")} == {
|
||||
"stop_reason": ",9",
|
||||
"token_ids": [22, 11, 23],
|
||||
}
|
||||
|
||||
|
||||
def test_stream_chunk_builder_keeps_null_vllm_fields_and_adds_none_for_other_providers() -> None:
|
||||
vllm_built: Final = stream_chunk_builder(
|
||||
chunks=[
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue