fix(streaming): build ChoiceLogprobs' serializer up front and keep nonconforming logprobs as JSON on chunk rebuilds

ChoiceLogprobs inherits the openai SDK's deferred build, so a fresh worker
whose first logprobs arrive on the finish chunk still had no serializer.
Foreign logprobs models now dump in full, so a missing top_logprobs becomes
[] and SDK callers keep attribute access, and chunk rebuilds for content and
usage keep logprobs that don't fit as plain JSON instead of raising.
This commit is contained in:
mateo-berri 2026-10-29 08:10:00 -07:00
parent 9866c83ce4
commit 0bf4140737
3 changed files with 196 additions and 48 deletions

View file

@ -10,14 +10,14 @@ import traceback
from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import TYPE_CHECKING, Annotated, Any, Final, NoReturn, Protocol, TypeVar, cast
from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypeVar, cast
import anyio
import httpx
from openai.types.chat import ChatCompletionChunk
from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice
from openai.types.chat.chat_completion_chunk import ChoiceDelta
from pydantic import BaseModel, BeforeValidator, Field, JsonValue, TypeAdapter, ValidationError
from pydantic import BaseModel, ValidationError
from typing_extensions import NotRequired, TypedDict
import litellm
@ -40,8 +40,8 @@ from litellm.llms.ollama.common_utils import OllamaResponseIdNormalizer
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import (
CHOICE_LOGPROBS_OR_JSON,
CacheCreationTokenDetails,
ChoiceLogprobs,
CompletionTokensDetailsWrapper,
Delta,
LlmProviders,
@ -141,21 +141,6 @@ def _delta_from_upstream(upstream_delta: "BaseModel | _UpstreamDeltaLike | None"
return None
def _sdk_logprobs_as_dict(logprobs: object) -> object:
if isinstance(logprobs, ChoiceLogprobs) or not isinstance(logprobs, BaseModel):
return logprobs
return logprobs.model_dump(exclude_unset=True)
_UPSTREAM_LOGPROBS: Final[TypeAdapter[ChoiceLogprobs | JsonValue]] = TypeAdapter(
Annotated[
ChoiceLogprobs | JsonValue,
Field(union_mode="left_to_right"),
BeforeValidator(_sdk_logprobs_as_dict),
]
)
def _is_transport_timeout(error: BaseException) -> bool:
return isinstance(error, httpx.TimeoutException) or (
isinstance(error, Timeout) and isinstance(error.__cause__, httpx.TimeoutException)
@ -1621,7 +1606,7 @@ class CustomStreamWrapper:
model_response.system_fingerprint = response_obj["original_chunk"].system_fingerprint
self.system_fingerprint = response_obj["original_chunk"].system_fingerprint
if response_obj["logprobs"] is not None:
model_response.choices[0].logprobs = _UPSTREAM_LOGPROBS.validate_python(response_obj["logprobs"])
model_response.choices[0].logprobs = CHOICE_LOGPROBS_OR_JSON.validate_python(response_obj["logprobs"])
if response_obj["usage"] is not None:
if isinstance(response_obj["usage"], dict):

View file

@ -7,6 +7,7 @@ from enum import Enum
from types import MappingProxyType
from typing import (
TYPE_CHECKING,
Annotated,
Any,
Final,
Literal,
@ -37,6 +38,7 @@ from openai.types.moderation_create_response import (
)
from pydantic import (
BaseModel,
BeforeValidator,
ConfigDict,
Field,
FieldSerializationInfo,
@ -44,6 +46,7 @@ from pydantic import (
JsonValue,
PrivateAttr,
SkipValidation,
TypeAdapter,
field_serializer,
field_validator,
)
@ -1234,6 +1237,8 @@ class ChatCompletionTokenLogprob(OpenAIObject):
class ChoiceLogprobs(OpenAIObject):
model_config = ConfigDict(defer_build=False)
content: list[ChatCompletionTokenLogprob] | None = None
"""A list of message content tokens with log probability information."""
@ -1250,6 +1255,21 @@ class ChoiceLogprobs(OpenAIObject):
return getattr(self, key)
def _sdk_logprobs_as_dict(logprobs: object) -> object:
if isinstance(logprobs, ChoiceLogprobs) or not isinstance(logprobs, BaseModel):
return logprobs
return logprobs.model_dump()
CHOICE_LOGPROBS_OR_JSON: Final[TypeAdapter[ChoiceLogprobs | JsonValue]] = TypeAdapter(
Annotated[
ChoiceLogprobs | JsonValue,
Field(union_mode="left_to_right"),
BeforeValidator(_sdk_logprobs_as_dict),
]
)
class FunctionCall(OpenAIObject):
arguments: str
name: str | None = None
@ -2130,7 +2150,7 @@ class StreamingChoices(OpenAIObject):
self.enhancements = enhancements
if logprobs is not None and isinstance(logprobs, dict):
self.logprobs = ChoiceLogprobs(**logprobs)
self.logprobs = CHOICE_LOGPROBS_OR_JSON.validate_python(logprobs)
else:
self.logprobs = logprobs

View file

@ -2,12 +2,15 @@ import asyncio
import datetime
import itertools
import json
import os
import queue
import subprocess
import sys
import threading
import time
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from types import CodeType, FrameType, MappingProxyType
from typing import AsyncIterator, Callable, Final, Iterator, Literal, NoReturn, Optional, cast
from unittest.mock import AsyncMock, MagicMock, Mock, patch
@ -1750,42 +1753,26 @@ async def test_stream_finish_chunk_with_logprobs_reaches_caller_as_litellm_logpr
assert [chunk.usage.total_tokens for chunk in chunks if getattr(chunk, "usage", None)] == expected_usage_totals
@pytest.mark.parametrize("async_stream", [False, True])
@pytest.mark.parametrize(
("finish_logprobs", "expected_wire_logprobs"),
[
pytest.param(
{"content": [{"token": "<|return|>"}]},
{"content": [{"token": "<|return|>"}]},
id="token-without-logprob",
),
pytest.param(-0.25, -0.25, id="scalar-logprobs"),
],
)
async def test_stream_finish_chunk_with_nonstandard_logprobs_still_finishes(
async def _sdk_stream_chunks(
upstream_chunks: Sequence[Mapping[str, object]],
async_stream: bool,
finish_logprobs: object,
expected_wire_logprobs: object,
logging_obj: Logging,
) -> None:
upstream_chunks: Final = (
[{"index": 0, "delta": {"role": "assistant", "content": "hi"}, "finish_reason": None}],
[{"index": 0, "delta": {}, "logprobs": finish_logprobs, "finish_reason": "stop"}],
)
stream_options: dict[str, bool] | None = None,
) -> list[ModelResponseStream]:
body: Final = (
"".join(
"data: "
+ json.dumps(
{
"id": "chatcmpl-nonstandard-logprobs",
"id": "chatcmpl-logprobs",
"object": "chat.completion.chunk",
"created": 1,
"model": "test-model",
"choices": choices,
**chunk,
}
)
+ "\n\n"
for choices in upstream_chunks
for chunk in upstream_chunks
)
+ "data: [DONE]\n\n"
)
@ -1802,26 +1789,182 @@ async def test_stream_finish_chunk_with_nonstandard_logprobs_still_finishes(
else Stream(cast_to=ChatCompletionChunk, response=raw_response, client=client)
)
response: Final = CustomStreamWrapper(
completion_stream=stream, model="test-model", logging_obj=logging_obj, custom_llm_provider="openai"
completion_stream=stream,
model="test-model",
logging_obj=logging_obj,
custom_llm_provider="openai",
stream_options=stream_options,
)
try:
chunks: Final = [chunk async for chunk in response] if async_stream else list(response)
return [chunk async for chunk in response] if async_stream else list(response)
finally:
if isinstance(client, AsyncOpenAI):
await client.close()
else:
client.close()
@pytest.mark.parametrize("async_stream", [False, True])
@pytest.mark.parametrize(
("upstream_logprobs", "finish_chunk_extra", "stream_options", "expected_usage_totals"),
[
pytest.param(
{"content": [{"token": "<|return|>", "bytes": None, "logprob": None, "top_logprobs": None}], "refusal": None},
{},
None,
[],
id="null-logprob",
),
pytest.param(-0.25, {}, None, [], id="scalar-logprobs"),
pytest.param(
{"content": [{"token": "<|return|>", "bytes": None, "logprob": None, "top_logprobs": None}], "refusal": None},
{"usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}},
{"include_usage": True},
[5],
id="null-logprob-and-usage",
),
],
)
async def test_stream_with_nonstandard_logprobs_passes_them_through_and_finishes(
async_stream: bool,
upstream_logprobs: object,
finish_chunk_extra: dict[str, object],
stream_options: dict[str, bool] | None,
expected_usage_totals: list[int],
logging_obj: Logging,
) -> None:
chunks: Final = await _sdk_stream_chunks(
(
{
"choices": [
{
"index": 0,
"delta": {"role": "assistant", "content": "hi"},
"logprobs": upstream_logprobs,
"finish_reason": None,
}
]
},
{
"choices": [{"index": 0, "delta": {}, "logprobs": upstream_logprobs, "finish_reason": "stop"}],
**finish_chunk_extra,
},
),
async_stream,
logging_obj,
stream_options,
)
wire_choices: Final = tuple(
itertools.chain.from_iterable(
json.loads(chunk.model_dump_json(exclude_none=True, exclude_unset=True))["choices"] for chunk in chunks
)
)
finish_choices: Final = [choice for choice in wire_choices if choice.get("finish_reason")]
logprobs_choices: Final = [choice for choice in wire_choices if "logprobs" in choice]
assert [choice.get("finish_reason") for choice in logprobs_choices] == [None, "stop"]
assert [choice["logprobs"] for choice in logprobs_choices] == [upstream_logprobs, upstream_logprobs]
assert [chunk.usage.total_tokens for chunk in chunks if getattr(chunk, "usage", None)] == expected_usage_totals
@pytest.mark.parametrize("async_stream", [False, True])
async def test_stream_logprobs_without_top_logprobs_reach_caller_as_litellm_logprobs(
async_stream: bool, logging_obj: Logging
) -> None:
chunks: Final = await _sdk_stream_chunks(
tuple(
{
"choices": [
{
"index": 0,
"delta": delta,
"logprobs": {"content": [{"token": token, "logprob": -0.5}]},
"finish_reason": finish_reason,
}
]
}
for token, delta, finish_reason in (
("hi", {"role": "assistant", "content": "hi"}, None),
("<|return|>", {}, "stop"),
)
),
async_stream,
logging_obj,
)
streamed_logprobs: Final = tuple(
choice.logprobs
for choice in itertools.chain.from_iterable(chunk.choices for chunk in chunks)
if choice.logprobs is not None
)
streamed_entries: Final = tuple(
itertools.chain.from_iterable(logprobs.content or () for logprobs in streamed_logprobs)
)
assert [(entry.token, entry.logprob, entry.top_logprobs) for entry in streamed_entries] == [
("hi", -0.5, []),
("<|return|>", -0.5, []),
]
FINISH_ONLY_LOGPROBS_STREAM_SCRIPT: Final = """
import json
import sys
sys.path.insert(0, sys.argv[1])
import httpx
from openai import OpenAI
import litellm
upstream_choices = (
{"index": 0, "delta": {"role": "assistant", "content": "hi"}, "finish_reason": None},
{
"index": 0,
"delta": {},
"logprobs": {"content": [{"token": "<|return|>", "logprob": -0.5, "top_logprobs": []}]},
"finish_reason": "stop",
},
)
body = "".join(
"data: "
+ json.dumps({"id": "c", "object": "chat.completion.chunk", "created": 1, "model": "m", "choices": [choice]})
+ "\\n\\n"
for choice in upstream_choices
) + "data: [DONE]\\n\\n"
client = OpenAI(
api_key="unused",
base_url="https://provider.example/v1",
http_client=httpx.Client(
transport=httpx.MockTransport(
lambda request: httpx.Response(200, content=body, headers={"content-type": "text/event-stream"})
)
),
)
stream = litellm.completion(
model="openai/m", messages=[{"role": "user", "content": "hi"}], stream=True, logprobs=True, client=client
)
print(json.dumps([json.loads(chunk.model_dump_json(exclude_none=True)) for chunk in stream]))
"""
def test_stream_with_logprobs_only_on_finish_chunk_serializes_in_a_fresh_process() -> None:
result: Final = subprocess.run(
[sys.executable, "-I", "-c", FINISH_ONLY_LOGPROBS_STREAM_SCRIPT, str(Path(litellm.__file__).parent.parent)],
capture_output=True,
text=True,
timeout=120,
env={**os.environ, "LITELLM_LOCAL_MODEL_COST_MAP": "True"},
)
assert result.returncode == 0, result.stdout + result.stderr
wire_choices: Final = tuple(
itertools.chain.from_iterable(chunk["choices"] for chunk in json.loads(result.stdout.strip().splitlines()[-1]))
)
finish_choices: Final = [choice for choice in wire_choices if choice.get("finish_reason")]
assert [choice["finish_reason"] for choice in finish_choices] == ["stop"]
assert finish_choices[0]["logprobs"] == expected_wire_logprobs
assert [entry["token"] for entry in finish_choices[0]["logprobs"]["content"]] == ["<|return|>"]
def _hosted_vllm_stream_wrapper(logging_obj: Logging, error_payload: dict) -> CustomStreamWrapper: