fix(streaming): keep only the logprobs keys upstream sent and type refusal logprobs
Some checks failed
Cost map pins recheck / rerun-cost-map-guard (push) Has been cancelled
LiteLLM Rust DB / db-queries (push) Has been cancelled
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled

This commit is contained in:
mateo-berri 2026-10-29 12:34:35 -07:00
parent 0bf4140737
commit f38dfce30e
4 changed files with 26 additions and 8 deletions

View file

@ -1202,7 +1202,10 @@ class CustomStreamWrapper:
choice_json.pop(
"finish_reason", None
) # for mistral etc. which return a value in their last chunk (not-openai compatible).
choices.append(StreamingChoices(**choice_json))
choice_json.pop("logprobs", None)
choices.append(
StreamingChoices(**choice_json, logprobs=getattr(choice, "logprobs", None))
)
except Exception:
choices.append(StreamingChoices())
setattr(model_response, "choices", choices)

View file

@ -1200,7 +1200,7 @@ class ChatCompletionTokenLogprob(OpenAIObject):
unlikely.
"""
top_logprobs: list[TopLogprob]
top_logprobs: list[TopLogprob] = Field(default_factory=list)
"""List of the most likely tokens and their log probability, at this token
position.
@ -1242,6 +1242,8 @@ class ChoiceLogprobs(OpenAIObject):
content: list[ChatCompletionTokenLogprob] | None = None
"""A list of message content tokens with log probability information."""
refusal: Sequence[ChatCompletionTokenLogprob] | None = None
def __contains__(self, key) -> bool:
# Define custom behavior for the 'in' operator
return hasattr(self, key)
@ -1258,7 +1260,7 @@ class ChoiceLogprobs(OpenAIObject):
def _sdk_logprobs_as_dict(logprobs: object) -> object:
if isinstance(logprobs, ChoiceLogprobs) or not isinstance(logprobs, BaseModel):
return logprobs
return logprobs.model_dump()
return logprobs.model_dump(exclude_unset=True)
CHOICE_LOGPROBS_OR_JSON: Final[TypeAdapter[ChoiceLogprobs | JsonValue]] = TypeAdapter(
@ -2149,7 +2151,7 @@ class StreamingChoices(OpenAIObject):
if enhancements is not None:
self.enhancements = enhancements
if logprobs is not None and isinstance(logprobs, dict):
if isinstance(logprobs, (dict, BaseModel)):
self.logprobs = CHOICE_LOGPROBS_OR_JSON.validate_python(logprobs)
else:
self.logprobs = logprobs

View file

@ -1815,6 +1815,13 @@ async def _sdk_stream_chunks(
[],
id="null-logprob",
),
pytest.param(
{"content": [{"token": "<|return|>", "logprob": None}]},
{},
None,
[],
id="null-logprob-keeps-only-the-keys-upstream-sent",
),
pytest.param(-0.25, {}, None, [], id="scalar-logprobs"),
pytest.param(
{"content": [{"token": "<|return|>", "bytes": None, "logprob": None, "top_logprobs": None}], "refusal": None},
@ -1868,8 +1875,9 @@ async def test_stream_with_nonstandard_logprobs_passes_them_through_and_finishes
@pytest.mark.parametrize("async_stream", [False, True])
@pytest.mark.parametrize("logprobs_field", ["content", "refusal"])
async def test_stream_logprobs_without_top_logprobs_reach_caller_as_litellm_logprobs(
async_stream: bool, logging_obj: Logging
async_stream: bool, logprobs_field: Literal["content", "refusal"], logging_obj: Logging
) -> None:
chunks: Final = await _sdk_stream_chunks(
tuple(
@ -1878,7 +1886,7 @@ async def test_stream_logprobs_without_top_logprobs_reach_caller_as_litellm_logp
{
"index": 0,
"delta": delta,
"logprobs": {"content": [{"token": token, "logprob": -0.5}]},
"logprobs": {logprobs_field: [{"token": token, "logprob": -0.5}]},
"finish_reason": finish_reason,
}
]
@ -1898,7 +1906,10 @@ async def test_stream_logprobs_without_top_logprobs_reach_caller_as_litellm_logp
if choice.logprobs is not None
)
streamed_entries: Final = tuple(
itertools.chain.from_iterable(logprobs.content or () for logprobs in streamed_logprobs)
itertools.chain.from_iterable(
(logprobs.content if logprobs_field == "content" else logprobs.refusal) or ()
for logprobs in streamed_logprobs
)
)
assert [(entry.token, entry.logprob, entry.top_logprobs) for entry in streamed_entries] == [

View file

@ -29012,7 +29012,7 @@ export interface components {
/** Token */
token: string;
/** Top Logprobs */
top_logprobs: components["schemas"]["TopLogprob"][];
top_logprobs?: components["schemas"]["TopLogprob"][];
} & {
[key: string]: unknown;
};
@ -29143,6 +29143,8 @@ export interface components {
ChoiceLogprobs: {
/** Content */
content?: components["schemas"]["ChatCompletionTokenLogprob"][] | null;
/** Refusal */
refusal?: components["schemas"]["ChatCompletionTokenLogprob"][] | null;
} & {
[key: string]: unknown;
};