perf(openai): decode streamed chat chunks with pydantic's compiled validator (#51905)

Strictly typed chunks skip the SDK's per-chunk construct_type pass, loose ones still fall back to it, so the decoded chunks are unchanged.
This commit is contained in:
Nate Armstrong 2026-11-16 12:29:08 -08:00 • committed by GitHub
parent 57b9afa506
commit 2ab55976f5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 156 additions and 4 deletions

View file

@ -16,13 +16,14 @@ from openai import AsyncOpenAI, AsyncStream, OpenAI, Stream
from openai._base_client import make_request_options
from openai._constants import RAW_RESPONSE_HEADER
from openai._legacy_response import LegacyAPIResponse
from openai._models import construct_type
from openai._types import RequestOptions
from openai.pagination import SyncCursorPage
from openai.types import CreateEmbeddingResponse
from openai.types.beta.assistant_deleted import AssistantDeleted
from openai.types.chat import ChatCompletion, ChatCompletionChunk
from openai.types.file_deleted import FileDeleted
from pydantic import BaseModel, TypeAdapter
from pydantic import BaseModel, TypeAdapter, ValidationError
from typing_extensions import overload
import litellm
@ -390,13 +391,48 @@ _CLIENT_API_KEY_ONLY_AUTH: Final[Mapping[str, Mapping[str, bool]]] = (
)
class _ChatCompletionChunkDecoder:
"""
Decodes stream chunks for the SDK's `ModelBuilderProtocol` hook in one compiled pydantic pass. Strict
validation only accepts payloads whose values already have the declared types, where it builds the same
chunk as the SDK's `construct_type`, and everything else keeps the SDK's lenient `construct_type` result.
"""
@classmethod
def build(cls, *, response: httpx.Response, data: object) -> object:
try:
return ChatCompletionChunk.model_validate(data, strict=True)
except ValidationError:
return construct_type(type_=ChatCompletionChunk, value=data)
def _raw_chat_response(
result: ChatCompletion | Stream[ChatCompletionChunk] | AsyncStream[ChatCompletionChunk],
result: ChatCompletion
| Stream[ChatCompletionChunk]
| AsyncStream[ChatCompletionChunk]
| Stream[_ChatCompletionChunkDecoder]
| AsyncStream[_ChatCompletionChunkDecoder],
) -> LegacyAPIResponse[ChatCompletion]:
assert isinstance(result, LegacyAPIResponse)
return result
def _sync_chat_stream_cls(
client: OpenAI,
) -> type[Stream[ChatCompletionChunk]] | type[Stream[_ChatCompletionChunkDecoder]]:
if client._strict_response_validation: # pyright: ignore[reportPrivateUsage] # SDK has no public getter
return Stream[ChatCompletionChunk]
return Stream[_ChatCompletionChunkDecoder]
def _async_chat_stream_cls(
client: AsyncOpenAI,
) -> type[AsyncStream[ChatCompletionChunk]] | type[AsyncStream[_ChatCompletionChunkDecoder]]:
if client._strict_response_validation: # pyright: ignore[reportPrivateUsage] # SDK has no public getter
return AsyncStream[ChatCompletionChunk]
return AsyncStream[_ChatCompletionChunkDecoder]
def _request_without_sdk_transform(
data: Mapping[str, object], timeout: float | httpx.Timeout
) -> tuple[Mapping[str, object], RequestOptions]:
@ -569,7 +605,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
cast_to=ChatCompletion,
options=options,
stream=bool(data.get("stream")),
stream_cls=AsyncStream[ChatCompletionChunk],
stream_cls=_async_chat_stream_cls(openai_aclient),
)
)
end_time = time.time()
@ -625,7 +661,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
cast_to=ChatCompletion,
options=options,
stream=bool(data.get("stream")),
stream_cls=Stream[ChatCompletionChunk],
stream_cls=_sync_chat_stream_cls(openai_client),
)
)

View file

@ -2815,3 +2815,119 @@ async def test_sync_batch_list_given_an_async_client_returns_a_page_the_caller_c
)
assert [(batch.id, batch.status) for batch in (await page).data] == [("batch_5f8dac", "pending")]
_STREAM_CHUNK_BASE: Final = {
"id": "chatcmpl-decode",
"object": "chat.completion.chunk",
"created": 1700000000,
"model": "gpt-5.5",
}
_STREAM_CHUNK_PAYLOADS: Final = (
{
**_STREAM_CHUNK_BASE,
"system_fingerprint": "fp_1",
"choices": [
{"index": 0, "delta": {"role": "assistant", "content": "hi"}, "logprobs": None, "finish_reason": None}
],
},
{
**_STREAM_CHUNK_BASE,
"choices": [
{
"index": 0,
"delta": {
"tool_calls": [
{"index": 0, "id": "call_1", "type": "function", "function": {"name": "f", "arguments": ""}},
{"index": 1, "type": None, "function": {"arguments": None}},
]
},
"finish_reason": None,
}
],
},
{
**_STREAM_CHUNK_BASE,
"choices": [],
"usage": {
"prompt_tokens": 9,
"completion_tokens": 2,
"total_tokens": 11,
"prompt_tokens_details": {"cached_tokens": 3},
"cost": 0.5,
},
},
{
**_STREAM_CHUNK_BASE,
"choices": [
{
"index": 0,
"delta": {"content": "a"},
"logprobs": {"content": [{"token": "a", "logprob": 0, "bytes": [97], "top_logprobs": []}]},
"finish_reason": "stop",
}
],
},
{
**_STREAM_CHUNK_BASE,
"provider": "upstream",
"citations": ["https://e.example"],
"choices": [
{
"index": 0,
"delta": {"content": "", "reasoning_content": "think", "provider_specific_fields": {"k": 1}},
"finish_reason": None,
"stop_reason": 128009,
}
],
},
{
**_STREAM_CHUNK_BASE,
"created": "1700000000",
"choices": [{"index": 0, "delta": {"content": "numeric string"}, "finish_reason": None}],
},
{
**_STREAM_CHUNK_BASE,
"object": "chat.completion",
"choices": [{"index": 0, "delta": {"content": "loose"}, "finish_reason": "eos"}],
},
{**_STREAM_CHUNK_BASE, "created": 1700000000.5, "choices": [{"index": "0", "delta": None}]},
)
def _model_state(value: object) -> object:
if isinstance(value, BaseModel):
return (
type(value),
{key: _model_state(item) for key, item in value.__dict__.items()},
{key: _model_state(item) for key, item in (value.__pydantic_extra__ or {}).items()},
frozenset(value.model_fields_set & type(value).model_fields.keys()),
)
if isinstance(value, list):
return [_model_state(item) for item in value]
if isinstance(value, dict):
return {key: _model_state(item) for key, item in value.items()}
return (type(value), value)
@pytest.mark.asyncio
async def test_streamed_chat_chunks_decode_exactly_like_the_sdk_for_strict_and_loose_payloads():
from openai._models import construct_type
from openai.types.chat import ChatCompletionChunk
body: Final = b"".join(f"data: {json.dumps(p)}\n\n".encode() for p in _STREAM_CHUNK_PAYLOADS) + b"data: [DONE]\n\n"
transport: Final = httpx.MockTransport(
lambda _request: httpx.Response(200, content=body, headers={"content-type": "text/event-stream"})
)
async with httpx.AsyncClient(transport=transport) as http_client:
_, stream = await OpenAIChatCompletion().make_openai_chat_completion_request(
openai_aclient=AsyncOpenAI(api_key="transport-only", http_client=http_client),
data={"model": "gpt-5.5", "messages": [{"role": "user", "content": "q"}], "stream": True},
timeout=10,
logging_obj=MagicMock(),
)
decoded: Final = [chunk async for chunk in stream]
assert [_model_state(chunk) for chunk in decoded] == [
_model_state(construct_type(type_=ChatCompletionChunk, value=payload)) for payload in _STREAM_CHUNK_PAYLOADS
]