mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(streaming): build ChoiceLogprobs' serializer up front and keep nonconforming logprobs as JSON on chunk rebuilds
ChoiceLogprobs inherits the openai SDK's deferred build, so a fresh worker whose first logprobs arrive on the finish chunk still had no serializer. Foreign logprobs models now dump in full, so a missing top_logprobs becomes [] and SDK callers keep attribute access, and chunk rebuilds for content and usage keep logprobs that don't fit as plain JSON instead of raising.
This commit is contained in:
parent
9866c83ce4
commit
0bf4140737
3 changed files with 196 additions and 48 deletions
|
|
@ -10,14 +10,14 @@ import traceback
|
|||
from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Annotated, Any, Final, NoReturn, Protocol, TypeVar, cast
|
||||
from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypeVar, cast
|
||||
|
||||
import anyio
|
||||
import httpx
|
||||
from openai.types.chat import ChatCompletionChunk
|
||||
from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice
|
||||
from openai.types.chat.chat_completion_chunk import ChoiceDelta
|
||||
from pydantic import BaseModel, BeforeValidator, Field, JsonValue, TypeAdapter, ValidationError
|
||||
from pydantic import BaseModel, ValidationError
|
||||
from typing_extensions import NotRequired, TypedDict
|
||||
|
||||
import litellm
|
||||
|
|
@ -40,8 +40,8 @@ from litellm.llms.ollama.common_utils import OllamaResponseIdNormalizer
|
|||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import (
|
||||
CHOICE_LOGPROBS_OR_JSON,
|
||||
CacheCreationTokenDetails,
|
||||
ChoiceLogprobs,
|
||||
CompletionTokensDetailsWrapper,
|
||||
Delta,
|
||||
LlmProviders,
|
||||
|
|
@ -141,21 +141,6 @@ def _delta_from_upstream(upstream_delta: "BaseModel | _UpstreamDeltaLike | None"
|
|||
return None
|
||||
|
||||
|
||||
def _sdk_logprobs_as_dict(logprobs: object) -> object:
|
||||
if isinstance(logprobs, ChoiceLogprobs) or not isinstance(logprobs, BaseModel):
|
||||
return logprobs
|
||||
return logprobs.model_dump(exclude_unset=True)
|
||||
|
||||
|
||||
_UPSTREAM_LOGPROBS: Final[TypeAdapter[ChoiceLogprobs | JsonValue]] = TypeAdapter(
|
||||
Annotated[
|
||||
ChoiceLogprobs | JsonValue,
|
||||
Field(union_mode="left_to_right"),
|
||||
BeforeValidator(_sdk_logprobs_as_dict),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _is_transport_timeout(error: BaseException) -> bool:
|
||||
return isinstance(error, httpx.TimeoutException) or (
|
||||
isinstance(error, Timeout) and isinstance(error.__cause__, httpx.TimeoutException)
|
||||
|
|
@ -1621,7 +1606,7 @@ class CustomStreamWrapper:
|
|||
model_response.system_fingerprint = response_obj["original_chunk"].system_fingerprint
|
||||
self.system_fingerprint = response_obj["original_chunk"].system_fingerprint
|
||||
if response_obj["logprobs"] is not None:
|
||||
model_response.choices[0].logprobs = _UPSTREAM_LOGPROBS.validate_python(response_obj["logprobs"])
|
||||
model_response.choices[0].logprobs = CHOICE_LOGPROBS_OR_JSON.validate_python(response_obj["logprobs"])
|
||||
|
||||
if response_obj["usage"] is not None:
|
||||
if isinstance(response_obj["usage"], dict):
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from enum import Enum
|
|||
from types import MappingProxyType
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Annotated,
|
||||
Any,
|
||||
Final,
|
||||
Literal,
|
||||
|
|
@ -37,6 +38,7 @@ from openai.types.moderation_create_response import (
|
|||
)
|
||||
from pydantic import (
|
||||
BaseModel,
|
||||
BeforeValidator,
|
||||
ConfigDict,
|
||||
Field,
|
||||
FieldSerializationInfo,
|
||||
|
|
@ -44,6 +46,7 @@ from pydantic import (
|
|||
JsonValue,
|
||||
PrivateAttr,
|
||||
SkipValidation,
|
||||
TypeAdapter,
|
||||
field_serializer,
|
||||
field_validator,
|
||||
)
|
||||
|
|
@ -1234,6 +1237,8 @@ class ChatCompletionTokenLogprob(OpenAIObject):
|
|||
|
||||
|
||||
class ChoiceLogprobs(OpenAIObject):
|
||||
model_config = ConfigDict(defer_build=False)
|
||||
|
||||
content: list[ChatCompletionTokenLogprob] | None = None
|
||||
"""A list of message content tokens with log probability information."""
|
||||
|
||||
|
|
@ -1250,6 +1255,21 @@ class ChoiceLogprobs(OpenAIObject):
|
|||
return getattr(self, key)
|
||||
|
||||
|
||||
def _sdk_logprobs_as_dict(logprobs: object) -> object:
|
||||
if isinstance(logprobs, ChoiceLogprobs) or not isinstance(logprobs, BaseModel):
|
||||
return logprobs
|
||||
return logprobs.model_dump()
|
||||
|
||||
|
||||
CHOICE_LOGPROBS_OR_JSON: Final[TypeAdapter[ChoiceLogprobs | JsonValue]] = TypeAdapter(
|
||||
Annotated[
|
||||
ChoiceLogprobs | JsonValue,
|
||||
Field(union_mode="left_to_right"),
|
||||
BeforeValidator(_sdk_logprobs_as_dict),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
class FunctionCall(OpenAIObject):
|
||||
arguments: str
|
||||
name: str | None = None
|
||||
|
|
@ -2130,7 +2150,7 @@ class StreamingChoices(OpenAIObject):
|
|||
self.enhancements = enhancements
|
||||
|
||||
if logprobs is not None and isinstance(logprobs, dict):
|
||||
self.logprobs = ChoiceLogprobs(**logprobs)
|
||||
self.logprobs = CHOICE_LOGPROBS_OR_JSON.validate_python(logprobs)
|
||||
else:
|
||||
self.logprobs = logprobs
|
||||
|
||||
|
|
|
|||
|
|
@ -2,12 +2,15 @@ import asyncio
|
|||
import datetime
|
||||
import itertools
|
||||
import json
|
||||
import os
|
||||
import queue
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from types import CodeType, FrameType, MappingProxyType
|
||||
from typing import AsyncIterator, Callable, Final, Iterator, Literal, NoReturn, Optional, cast
|
||||
from unittest.mock import AsyncMock, MagicMock, Mock, patch
|
||||
|
|
@ -1750,42 +1753,26 @@ async def test_stream_finish_chunk_with_logprobs_reaches_caller_as_litellm_logpr
|
|||
assert [chunk.usage.total_tokens for chunk in chunks if getattr(chunk, "usage", None)] == expected_usage_totals
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_stream", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
("finish_logprobs", "expected_wire_logprobs"),
|
||||
[
|
||||
pytest.param(
|
||||
{"content": [{"token": "<|return|>"}]},
|
||||
{"content": [{"token": "<|return|>"}]},
|
||||
id="token-without-logprob",
|
||||
),
|
||||
pytest.param(-0.25, -0.25, id="scalar-logprobs"),
|
||||
],
|
||||
)
|
||||
async def test_stream_finish_chunk_with_nonstandard_logprobs_still_finishes(
|
||||
async def _sdk_stream_chunks(
|
||||
upstream_chunks: Sequence[Mapping[str, object]],
|
||||
async_stream: bool,
|
||||
finish_logprobs: object,
|
||||
expected_wire_logprobs: object,
|
||||
logging_obj: Logging,
|
||||
) -> None:
|
||||
upstream_chunks: Final = (
|
||||
[{"index": 0, "delta": {"role": "assistant", "content": "hi"}, "finish_reason": None}],
|
||||
[{"index": 0, "delta": {}, "logprobs": finish_logprobs, "finish_reason": "stop"}],
|
||||
)
|
||||
stream_options: dict[str, bool] | None = None,
|
||||
) -> list[ModelResponseStream]:
|
||||
body: Final = (
|
||||
"".join(
|
||||
"data: "
|
||||
+ json.dumps(
|
||||
{
|
||||
"id": "chatcmpl-nonstandard-logprobs",
|
||||
"id": "chatcmpl-logprobs",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "test-model",
|
||||
"choices": choices,
|
||||
**chunk,
|
||||
}
|
||||
)
|
||||
+ "\n\n"
|
||||
for choices in upstream_chunks
|
||||
for chunk in upstream_chunks
|
||||
)
|
||||
+ "data: [DONE]\n\n"
|
||||
)
|
||||
|
|
@ -1802,26 +1789,182 @@ async def test_stream_finish_chunk_with_nonstandard_logprobs_still_finishes(
|
|||
else Stream(cast_to=ChatCompletionChunk, response=raw_response, client=client)
|
||||
)
|
||||
response: Final = CustomStreamWrapper(
|
||||
completion_stream=stream, model="test-model", logging_obj=logging_obj, custom_llm_provider="openai"
|
||||
completion_stream=stream,
|
||||
model="test-model",
|
||||
logging_obj=logging_obj,
|
||||
custom_llm_provider="openai",
|
||||
stream_options=stream_options,
|
||||
)
|
||||
|
||||
try:
|
||||
chunks: Final = [chunk async for chunk in response] if async_stream else list(response)
|
||||
return [chunk async for chunk in response] if async_stream else list(response)
|
||||
finally:
|
||||
if isinstance(client, AsyncOpenAI):
|
||||
await client.close()
|
||||
else:
|
||||
client.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_stream", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
("upstream_logprobs", "finish_chunk_extra", "stream_options", "expected_usage_totals"),
|
||||
[
|
||||
pytest.param(
|
||||
{"content": [{"token": "<|return|>", "bytes": None, "logprob": None, "top_logprobs": None}], "refusal": None},
|
||||
{},
|
||||
None,
|
||||
[],
|
||||
id="null-logprob",
|
||||
),
|
||||
pytest.param(-0.25, {}, None, [], id="scalar-logprobs"),
|
||||
pytest.param(
|
||||
{"content": [{"token": "<|return|>", "bytes": None, "logprob": None, "top_logprobs": None}], "refusal": None},
|
||||
{"usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}},
|
||||
{"include_usage": True},
|
||||
[5],
|
||||
id="null-logprob-and-usage",
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_stream_with_nonstandard_logprobs_passes_them_through_and_finishes(
|
||||
async_stream: bool,
|
||||
upstream_logprobs: object,
|
||||
finish_chunk_extra: dict[str, object],
|
||||
stream_options: dict[str, bool] | None,
|
||||
expected_usage_totals: list[int],
|
||||
logging_obj: Logging,
|
||||
) -> None:
|
||||
chunks: Final = await _sdk_stream_chunks(
|
||||
(
|
||||
{
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {"role": "assistant", "content": "hi"},
|
||||
"logprobs": upstream_logprobs,
|
||||
"finish_reason": None,
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"choices": [{"index": 0, "delta": {}, "logprobs": upstream_logprobs, "finish_reason": "stop"}],
|
||||
**finish_chunk_extra,
|
||||
},
|
||||
),
|
||||
async_stream,
|
||||
logging_obj,
|
||||
stream_options,
|
||||
)
|
||||
|
||||
wire_choices: Final = tuple(
|
||||
itertools.chain.from_iterable(
|
||||
json.loads(chunk.model_dump_json(exclude_none=True, exclude_unset=True))["choices"] for chunk in chunks
|
||||
)
|
||||
)
|
||||
finish_choices: Final = [choice for choice in wire_choices if choice.get("finish_reason")]
|
||||
logprobs_choices: Final = [choice for choice in wire_choices if "logprobs" in choice]
|
||||
|
||||
assert [choice.get("finish_reason") for choice in logprobs_choices] == [None, "stop"]
|
||||
assert [choice["logprobs"] for choice in logprobs_choices] == [upstream_logprobs, upstream_logprobs]
|
||||
assert [chunk.usage.total_tokens for chunk in chunks if getattr(chunk, "usage", None)] == expected_usage_totals
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_stream", [False, True])
|
||||
async def test_stream_logprobs_without_top_logprobs_reach_caller_as_litellm_logprobs(
|
||||
async_stream: bool, logging_obj: Logging
|
||||
) -> None:
|
||||
chunks: Final = await _sdk_stream_chunks(
|
||||
tuple(
|
||||
{
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": delta,
|
||||
"logprobs": {"content": [{"token": token, "logprob": -0.5}]},
|
||||
"finish_reason": finish_reason,
|
||||
}
|
||||
]
|
||||
}
|
||||
for token, delta, finish_reason in (
|
||||
("hi", {"role": "assistant", "content": "hi"}, None),
|
||||
("<|return|>", {}, "stop"),
|
||||
)
|
||||
),
|
||||
async_stream,
|
||||
logging_obj,
|
||||
)
|
||||
|
||||
streamed_logprobs: Final = tuple(
|
||||
choice.logprobs
|
||||
for choice in itertools.chain.from_iterable(chunk.choices for chunk in chunks)
|
||||
if choice.logprobs is not None
|
||||
)
|
||||
streamed_entries: Final = tuple(
|
||||
itertools.chain.from_iterable(logprobs.content or () for logprobs in streamed_logprobs)
|
||||
)
|
||||
|
||||
assert [(entry.token, entry.logprob, entry.top_logprobs) for entry in streamed_entries] == [
|
||||
("hi", -0.5, []),
|
||||
("<|return|>", -0.5, []),
|
||||
]
|
||||
|
||||
|
||||
FINISH_ONLY_LOGPROBS_STREAM_SCRIPT: Final = """
|
||||
import json
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, sys.argv[1])
|
||||
|
||||
import httpx
|
||||
from openai import OpenAI
|
||||
|
||||
import litellm
|
||||
|
||||
upstream_choices = (
|
||||
{"index": 0, "delta": {"role": "assistant", "content": "hi"}, "finish_reason": None},
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {},
|
||||
"logprobs": {"content": [{"token": "<|return|>", "logprob": -0.5, "top_logprobs": []}]},
|
||||
"finish_reason": "stop",
|
||||
},
|
||||
)
|
||||
body = "".join(
|
||||
"data: "
|
||||
+ json.dumps({"id": "c", "object": "chat.completion.chunk", "created": 1, "model": "m", "choices": [choice]})
|
||||
+ "\\n\\n"
|
||||
for choice in upstream_choices
|
||||
) + "data: [DONE]\\n\\n"
|
||||
client = OpenAI(
|
||||
api_key="unused",
|
||||
base_url="https://provider.example/v1",
|
||||
http_client=httpx.Client(
|
||||
transport=httpx.MockTransport(
|
||||
lambda request: httpx.Response(200, content=body, headers={"content-type": "text/event-stream"})
|
||||
)
|
||||
),
|
||||
)
|
||||
stream = litellm.completion(
|
||||
model="openai/m", messages=[{"role": "user", "content": "hi"}], stream=True, logprobs=True, client=client
|
||||
)
|
||||
print(json.dumps([json.loads(chunk.model_dump_json(exclude_none=True)) for chunk in stream]))
|
||||
"""
|
||||
|
||||
|
||||
def test_stream_with_logprobs_only_on_finish_chunk_serializes_in_a_fresh_process() -> None:
|
||||
result: Final = subprocess.run(
|
||||
[sys.executable, "-I", "-c", FINISH_ONLY_LOGPROBS_STREAM_SCRIPT, str(Path(litellm.__file__).parent.parent)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120,
|
||||
env={**os.environ, "LITELLM_LOCAL_MODEL_COST_MAP": "True"},
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
wire_choices: Final = tuple(
|
||||
itertools.chain.from_iterable(chunk["choices"] for chunk in json.loads(result.stdout.strip().splitlines()[-1]))
|
||||
)
|
||||
finish_choices: Final = [choice for choice in wire_choices if choice.get("finish_reason")]
|
||||
assert [choice["finish_reason"] for choice in finish_choices] == ["stop"]
|
||||
assert finish_choices[0]["logprobs"] == expected_wire_logprobs
|
||||
assert [entry["token"] for entry in finish_choices[0]["logprobs"]["content"]] == ["<|return|>"]
|
||||
|
||||
|
||||
def _hosted_vllm_stream_wrapper(logging_obj: Logging, error_payload: dict) -> CustomStreamWrapper:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue