mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
perf(openai): decode streamed chat chunks with pydantic's compiled validator (#51905)
Strictly typed chunks skip the SDK's per-chunk construct_type pass, loose ones still fall back to it, so the decoded chunks are unchanged.
This commit is contained in:
parent
57b9afa506
commit
2ab55976f5
2 changed files with 156 additions and 4 deletions
|
|
@ -16,13 +16,14 @@ from openai import AsyncOpenAI, AsyncStream, OpenAI, Stream
|
|||
from openai._base_client import make_request_options
|
||||
from openai._constants import RAW_RESPONSE_HEADER
|
||||
from openai._legacy_response import LegacyAPIResponse
|
||||
from openai._models import construct_type
|
||||
from openai._types import RequestOptions
|
||||
from openai.pagination import SyncCursorPage
|
||||
from openai.types import CreateEmbeddingResponse
|
||||
from openai.types.beta.assistant_deleted import AssistantDeleted
|
||||
from openai.types.chat import ChatCompletion, ChatCompletionChunk
|
||||
from openai.types.file_deleted import FileDeleted
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
from pydantic import BaseModel, TypeAdapter, ValidationError
|
||||
from typing_extensions import overload
|
||||
|
||||
import litellm
|
||||
|
|
@ -390,13 +391,48 @@ _CLIENT_API_KEY_ONLY_AUTH: Final[Mapping[str, Mapping[str, bool]]] = (
|
|||
)
|
||||
|
||||
|
||||
class _ChatCompletionChunkDecoder:
|
||||
"""
|
||||
Decodes stream chunks for the SDK's `ModelBuilderProtocol` hook in one compiled pydantic pass. Strict
|
||||
validation only accepts payloads whose values already have the declared types, where it builds the same
|
||||
chunk as the SDK's `construct_type`, and everything else keeps the SDK's lenient `construct_type` result.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def build(cls, *, response: httpx.Response, data: object) -> object:
|
||||
try:
|
||||
return ChatCompletionChunk.model_validate(data, strict=True)
|
||||
except ValidationError:
|
||||
return construct_type(type_=ChatCompletionChunk, value=data)
|
||||
|
||||
|
||||
def _raw_chat_response(
|
||||
result: ChatCompletion | Stream[ChatCompletionChunk] | AsyncStream[ChatCompletionChunk],
|
||||
result: ChatCompletion
|
||||
| Stream[ChatCompletionChunk]
|
||||
| AsyncStream[ChatCompletionChunk]
|
||||
| Stream[_ChatCompletionChunkDecoder]
|
||||
| AsyncStream[_ChatCompletionChunkDecoder],
|
||||
) -> LegacyAPIResponse[ChatCompletion]:
|
||||
assert isinstance(result, LegacyAPIResponse)
|
||||
return result
|
||||
|
||||
|
||||
def _sync_chat_stream_cls(
|
||||
client: OpenAI,
|
||||
) -> type[Stream[ChatCompletionChunk]] | type[Stream[_ChatCompletionChunkDecoder]]:
|
||||
if client._strict_response_validation: # pyright: ignore[reportPrivateUsage] # SDK has no public getter
|
||||
return Stream[ChatCompletionChunk]
|
||||
return Stream[_ChatCompletionChunkDecoder]
|
||||
|
||||
|
||||
def _async_chat_stream_cls(
|
||||
client: AsyncOpenAI,
|
||||
) -> type[AsyncStream[ChatCompletionChunk]] | type[AsyncStream[_ChatCompletionChunkDecoder]]:
|
||||
if client._strict_response_validation: # pyright: ignore[reportPrivateUsage] # SDK has no public getter
|
||||
return AsyncStream[ChatCompletionChunk]
|
||||
return AsyncStream[_ChatCompletionChunkDecoder]
|
||||
|
||||
|
||||
def _request_without_sdk_transform(
|
||||
data: Mapping[str, object], timeout: float | httpx.Timeout
|
||||
) -> tuple[Mapping[str, object], RequestOptions]:
|
||||
|
|
@ -569,7 +605,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
|
|||
cast_to=ChatCompletion,
|
||||
options=options,
|
||||
stream=bool(data.get("stream")),
|
||||
stream_cls=AsyncStream[ChatCompletionChunk],
|
||||
stream_cls=_async_chat_stream_cls(openai_aclient),
|
||||
)
|
||||
)
|
||||
end_time = time.time()
|
||||
|
|
@ -625,7 +661,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
|
|||
cast_to=ChatCompletion,
|
||||
options=options,
|
||||
stream=bool(data.get("stream")),
|
||||
stream_cls=Stream[ChatCompletionChunk],
|
||||
stream_cls=_sync_chat_stream_cls(openai_client),
|
||||
)
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -2815,3 +2815,119 @@ async def test_sync_batch_list_given_an_async_client_returns_a_page_the_caller_c
|
|||
)
|
||||
|
||||
assert [(batch.id, batch.status) for batch in (await page).data] == [("batch_5f8dac", "pending")]
|
||||
|
||||
|
||||
_STREAM_CHUNK_BASE: Final = {
|
||||
"id": "chatcmpl-decode",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1700000000,
|
||||
"model": "gpt-5.5",
|
||||
}
|
||||
_STREAM_CHUNK_PAYLOADS: Final = (
|
||||
{
|
||||
**_STREAM_CHUNK_BASE,
|
||||
"system_fingerprint": "fp_1",
|
||||
"choices": [
|
||||
{"index": 0, "delta": {"role": "assistant", "content": "hi"}, "logprobs": None, "finish_reason": None}
|
||||
],
|
||||
},
|
||||
{
|
||||
**_STREAM_CHUNK_BASE,
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {
|
||||
"tool_calls": [
|
||||
{"index": 0, "id": "call_1", "type": "function", "function": {"name": "f", "arguments": ""}},
|
||||
{"index": 1, "type": None, "function": {"arguments": None}},
|
||||
]
|
||||
},
|
||||
"finish_reason": None,
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
**_STREAM_CHUNK_BASE,
|
||||
"choices": [],
|
||||
"usage": {
|
||||
"prompt_tokens": 9,
|
||||
"completion_tokens": 2,
|
||||
"total_tokens": 11,
|
||||
"prompt_tokens_details": {"cached_tokens": 3},
|
||||
"cost": 0.5,
|
||||
},
|
||||
},
|
||||
{
|
||||
**_STREAM_CHUNK_BASE,
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {"content": "a"},
|
||||
"logprobs": {"content": [{"token": "a", "logprob": 0, "bytes": [97], "top_logprobs": []}]},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
**_STREAM_CHUNK_BASE,
|
||||
"provider": "upstream",
|
||||
"citations": ["https://e.example"],
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {"content": "", "reasoning_content": "think", "provider_specific_fields": {"k": 1}},
|
||||
"finish_reason": None,
|
||||
"stop_reason": 128009,
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
**_STREAM_CHUNK_BASE,
|
||||
"created": "1700000000",
|
||||
"choices": [{"index": 0, "delta": {"content": "numeric string"}, "finish_reason": None}],
|
||||
},
|
||||
{
|
||||
**_STREAM_CHUNK_BASE,
|
||||
"object": "chat.completion",
|
||||
"choices": [{"index": 0, "delta": {"content": "loose"}, "finish_reason": "eos"}],
|
||||
},
|
||||
{**_STREAM_CHUNK_BASE, "created": 1700000000.5, "choices": [{"index": "0", "delta": None}]},
|
||||
)
|
||||
|
||||
|
||||
def _model_state(value: object) -> object:
|
||||
if isinstance(value, BaseModel):
|
||||
return (
|
||||
type(value),
|
||||
{key: _model_state(item) for key, item in value.__dict__.items()},
|
||||
{key: _model_state(item) for key, item in (value.__pydantic_extra__ or {}).items()},
|
||||
frozenset(value.model_fields_set & type(value).model_fields.keys()),
|
||||
)
|
||||
if isinstance(value, list):
|
||||
return [_model_state(item) for item in value]
|
||||
if isinstance(value, dict):
|
||||
return {key: _model_state(item) for key, item in value.items()}
|
||||
return (type(value), value)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamed_chat_chunks_decode_exactly_like_the_sdk_for_strict_and_loose_payloads():
|
||||
from openai._models import construct_type
|
||||
from openai.types.chat import ChatCompletionChunk
|
||||
|
||||
body: Final = b"".join(f"data: {json.dumps(p)}\n\n".encode() for p in _STREAM_CHUNK_PAYLOADS) + b"data: [DONE]\n\n"
|
||||
transport: Final = httpx.MockTransport(
|
||||
lambda _request: httpx.Response(200, content=body, headers={"content-type": "text/event-stream"})
|
||||
)
|
||||
async with httpx.AsyncClient(transport=transport) as http_client:
|
||||
_, stream = await OpenAIChatCompletion().make_openai_chat_completion_request(
|
||||
openai_aclient=AsyncOpenAI(api_key="transport-only", http_client=http_client),
|
||||
data={"model": "gpt-5.5", "messages": [{"role": "user", "content": "q"}], "stream": True},
|
||||
timeout=10,
|
||||
logging_obj=MagicMock(),
|
||||
)
|
||||
decoded: Final = [chunk async for chunk in stream]
|
||||
|
||||
assert [_model_state(chunk) for chunk in decoded] == [
|
||||
_model_state(construct_type(type_=ChatCompletionChunk, value=payload)) for payload in _STREAM_CHUNK_PAYLOADS
|
||||
]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue