fix(streaming): copy finish-chunk choice extras only from the same choice
Some checks failed
Cost map pins recheck / rerun-cost-map-guard (push) Has been cancelled
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust DB / db-queries (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled

Skip null values and vLLM token_ids, which the vLLM stream code already places
This commit is contained in:
mateo-berri 2026-11-02 20:03:07 -08:00
parent 278c3fcbe3
commit 1e077c52e8
2 changed files with 94 additions and 3 deletions

View file

@ -139,6 +139,9 @@ def _prepend_upstream_choice_extras(
choice.__pydantic_fields_set__.update(upstream_extras)
_CHOICE_EXTRAS_NOT_CARRIED_TO_FINISH: Final = frozenset({"message", *STREAM_CHUNK_CHOICE_FIELDS})
def _keep_single_upstream_choice_extras(
model_response: ModelResponseStream, response_obj: Mapping[str, object]
) -> None:
@ -147,13 +150,15 @@ def _keep_single_upstream_choice_extras(
not isinstance(original_chunk, ChatCompletionChunk)
or len(original_chunk.choices) != 1
or len(model_response.choices) != 1
or original_chunk.choices[0].index != model_response.choices[0].index
):
return
upstream_choice: Final = original_chunk.choices[0]
non_null_extras: Final = frozenset(
key for key, value in (upstream_choice.model_extra or {}).items() if value is not None
)
_prepend_upstream_choice_extras(
model_response.choices[0],
upstream_choice,
frozenset(upstream_choice.__pydantic_extra__ or {}) - {"message"},
model_response.choices[0], upstream_choice, non_null_extras - _CHOICE_EXTRAS_NOT_CARRIED_TO_FINISH
)

View file

@ -7375,6 +7375,92 @@ async def test_azure_stream_passes_on_prompt_filter_results_and_finish_content_f
assert finish_frame["choices"][0]["content_filter_results"] == _AZURE_FINISH_FILTER_RESULTS
def _openai_sdk_stream_choices(
logging_obj: Logging, upstream_choices: Sequence[Mapping[str, object]]
) -> tuple[dict[str, object], ...]:
chunks: Final = tuple(
ChatCompletionChunk.model_validate(
{"id": "c1", "created": 1, "model": "m", "object": "chat.completion.chunk", "choices": [choice]}
)
for choice in upstream_choices
)
wrapper: Final = CustomStreamWrapper(
completion_stream=iter(chunks), model="m", custom_llm_provider="openai", logging_obj=logging_obj
)
return tuple(chunk.choices[0].model_dump() for chunk in wrapper)
_TWO_CHOICE_ROLE_CHUNKS: Final = tuple(
{"index": index, "delta": {"role": "assistant", "content": ""}, "finish_reason": None} for index in (0, 1)
)
@pytest.mark.parametrize(
"upstream_choices",
[
pytest.param(
(
*_TWO_CHOICE_ROLE_CHUNKS,
{"index": 0, "delta": {"content": "1"}, "finish_reason": None},
{"index": 1, "delta": {"content": "1"}, "finish_reason": None},
{"index": 1, "delta": {}, "finish_reason": "stop", "content_filter_results": {"choice": 1}},
{"index": 0, "delta": {}, "finish_reason": "stop", "content_filter_results": {"choice": 0}},
),
id="other-choice-finishes-first",
),
pytest.param(
(
*_TWO_CHOICE_ROLE_CHUNKS,
{"index": 0, "delta": {"content": "1"}, "finish_reason": "stop", "stop_reason": ", 3", "token_ids": [16]},
{"index": 1, "delta": {"content": ""}, "finish_reason": None, "token_ids": [26]},
{"index": 1, "delta": {"content": "1"}, "finish_reason": "stop", "stop_reason": 151645, "token_ids": [27]},
),
id="other-choice-streams-after-a-finish",
),
],
)
def test_openai_sdk_stream_with_two_choices_keeps_choice_extras_on_their_own_choice(
logging_obj: Logging, upstream_choices: tuple[Mapping[str, object], ...]
) -> None:
emitted: Final = _openai_sdk_stream_choices(logging_obj, upstream_choices)
assert [
(choice["index"], choice.get("token_ids"), choice.get("stop_reason"), choice.get("content_filter_results"))
for choice in emitted
if _carries_another_choices_fields(choice)
or choice.get("content_filter_results") not in (None, {"choice": choice["index"]})
] == []
@pytest.mark.parametrize(
("dropped_extras", "post_call_rules"),
[
pytest.param({"token_ids": None}, [], id="null-token-ids"),
pytest.param({"content_filter_results": None}, [], id="null-content-filter-results"),
pytest.param({"token_ids": [20]}, [_rejects_a_two], id="token-ids-under-post-call-rules"),
],
)
def test_openai_sdk_stream_sends_the_same_finish_frame_without_choice_extras_it_drops(
logging_obj: Logging,
monkeypatch: pytest.MonkeyPatch,
dropped_extras: Mapping[str, object],
post_call_rules: list[Callable[[str], bool]],
) -> None:
monkeypatch.setattr(litellm, "post_call_rules", post_call_rules)
def finish_frame(finish_extras: Mapping[str, object]) -> dict[str, object]:
emitted: Final = _openai_sdk_stream_choices(
logging_obj,
(
{"index": 0, "delta": {"role": "assistant", "content": "1"}, "finish_reason": None},
{"index": 0, "delta": {}, "finish_reason": "stop", **finish_extras},
),
)
return emitted[-1]
assert finish_frame(dropped_extras) == finish_frame({})
def test_openai_chunk_creator_keeps_single_choice_logprobs_on_later_chunks(logging_obj: Logging) -> None:
logprobs: Final = {"content": [{"token": "hello", "logprob": -0.25, "bytes": [104], "top_logprobs": []}]}
chunk: Final = ChatCompletionChunk(