mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(streaming): copy finish-chunk choice extras only from the same choice
Some checks failed
Cost map pins recheck / rerun-cost-map-guard (push) Has been cancelled
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust DB / db-queries (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
Some checks failed
Cost map pins recheck / rerun-cost-map-guard (push) Has been cancelled
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust DB / db-queries (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
Skip null values and vLLM token_ids, which the vLLM stream code already places
This commit is contained in:
parent
278c3fcbe3
commit
1e077c52e8
2 changed files with 94 additions and 3 deletions
|
|
@ -139,6 +139,9 @@ def _prepend_upstream_choice_extras(
|
|||
choice.__pydantic_fields_set__.update(upstream_extras)
|
||||
|
||||
|
||||
_CHOICE_EXTRAS_NOT_CARRIED_TO_FINISH: Final = frozenset({"message", *STREAM_CHUNK_CHOICE_FIELDS})
|
||||
|
||||
|
||||
def _keep_single_upstream_choice_extras(
|
||||
model_response: ModelResponseStream, response_obj: Mapping[str, object]
|
||||
) -> None:
|
||||
|
|
@ -147,13 +150,15 @@ def _keep_single_upstream_choice_extras(
|
|||
not isinstance(original_chunk, ChatCompletionChunk)
|
||||
or len(original_chunk.choices) != 1
|
||||
or len(model_response.choices) != 1
|
||||
or original_chunk.choices[0].index != model_response.choices[0].index
|
||||
):
|
||||
return
|
||||
upstream_choice: Final = original_chunk.choices[0]
|
||||
non_null_extras: Final = frozenset(
|
||||
key for key, value in (upstream_choice.model_extra or {}).items() if value is not None
|
||||
)
|
||||
_prepend_upstream_choice_extras(
|
||||
model_response.choices[0],
|
||||
upstream_choice,
|
||||
frozenset(upstream_choice.__pydantic_extra__ or {}) - {"message"},
|
||||
model_response.choices[0], upstream_choice, non_null_extras - _CHOICE_EXTRAS_NOT_CARRIED_TO_FINISH
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -7375,6 +7375,92 @@ async def test_azure_stream_passes_on_prompt_filter_results_and_finish_content_f
|
|||
assert finish_frame["choices"][0]["content_filter_results"] == _AZURE_FINISH_FILTER_RESULTS
|
||||
|
||||
|
||||
def _openai_sdk_stream_choices(
|
||||
logging_obj: Logging, upstream_choices: Sequence[Mapping[str, object]]
|
||||
) -> tuple[dict[str, object], ...]:
|
||||
chunks: Final = tuple(
|
||||
ChatCompletionChunk.model_validate(
|
||||
{"id": "c1", "created": 1, "model": "m", "object": "chat.completion.chunk", "choices": [choice]}
|
||||
)
|
||||
for choice in upstream_choices
|
||||
)
|
||||
wrapper: Final = CustomStreamWrapper(
|
||||
completion_stream=iter(chunks), model="m", custom_llm_provider="openai", logging_obj=logging_obj
|
||||
)
|
||||
return tuple(chunk.choices[0].model_dump() for chunk in wrapper)
|
||||
|
||||
|
||||
_TWO_CHOICE_ROLE_CHUNKS: Final = tuple(
|
||||
{"index": index, "delta": {"role": "assistant", "content": ""}, "finish_reason": None} for index in (0, 1)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"upstream_choices",
|
||||
[
|
||||
pytest.param(
|
||||
(
|
||||
*_TWO_CHOICE_ROLE_CHUNKS,
|
||||
{"index": 0, "delta": {"content": "1"}, "finish_reason": None},
|
||||
{"index": 1, "delta": {"content": "1"}, "finish_reason": None},
|
||||
{"index": 1, "delta": {}, "finish_reason": "stop", "content_filter_results": {"choice": 1}},
|
||||
{"index": 0, "delta": {}, "finish_reason": "stop", "content_filter_results": {"choice": 0}},
|
||||
),
|
||||
id="other-choice-finishes-first",
|
||||
),
|
||||
pytest.param(
|
||||
(
|
||||
*_TWO_CHOICE_ROLE_CHUNKS,
|
||||
{"index": 0, "delta": {"content": "1"}, "finish_reason": "stop", "stop_reason": ", 3", "token_ids": [16]},
|
||||
{"index": 1, "delta": {"content": ""}, "finish_reason": None, "token_ids": [26]},
|
||||
{"index": 1, "delta": {"content": "1"}, "finish_reason": "stop", "stop_reason": 151645, "token_ids": [27]},
|
||||
),
|
||||
id="other-choice-streams-after-a-finish",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_openai_sdk_stream_with_two_choices_keeps_choice_extras_on_their_own_choice(
|
||||
logging_obj: Logging, upstream_choices: tuple[Mapping[str, object], ...]
|
||||
) -> None:
|
||||
emitted: Final = _openai_sdk_stream_choices(logging_obj, upstream_choices)
|
||||
|
||||
assert [
|
||||
(choice["index"], choice.get("token_ids"), choice.get("stop_reason"), choice.get("content_filter_results"))
|
||||
for choice in emitted
|
||||
if _carries_another_choices_fields(choice)
|
||||
or choice.get("content_filter_results") not in (None, {"choice": choice["index"]})
|
||||
] == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("dropped_extras", "post_call_rules"),
|
||||
[
|
||||
pytest.param({"token_ids": None}, [], id="null-token-ids"),
|
||||
pytest.param({"content_filter_results": None}, [], id="null-content-filter-results"),
|
||||
pytest.param({"token_ids": [20]}, [_rejects_a_two], id="token-ids-under-post-call-rules"),
|
||||
],
|
||||
)
|
||||
def test_openai_sdk_stream_sends_the_same_finish_frame_without_choice_extras_it_drops(
|
||||
logging_obj: Logging,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
dropped_extras: Mapping[str, object],
|
||||
post_call_rules: list[Callable[[str], bool]],
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "post_call_rules", post_call_rules)
|
||||
|
||||
def finish_frame(finish_extras: Mapping[str, object]) -> dict[str, object]:
|
||||
emitted: Final = _openai_sdk_stream_choices(
|
||||
logging_obj,
|
||||
(
|
||||
{"index": 0, "delta": {"role": "assistant", "content": "1"}, "finish_reason": None},
|
||||
{"index": 0, "delta": {}, "finish_reason": "stop", **finish_extras},
|
||||
),
|
||||
)
|
||||
return emitted[-1]
|
||||
|
||||
assert finish_frame(dropped_extras) == finish_frame({})
|
||||
|
||||
|
||||
def test_openai_chunk_creator_keeps_single_choice_logprobs_on_later_chunks(logging_obj: Logging) -> None:
|
||||
logprobs: Final = {"content": [{"token": "hello", "logprob": -0.25, "bytes": [104], "top_logprobs": []}]}
|
||||
chunk: Final = ChatCompletionChunk(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue