diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 55442dfb1cf..595525df757 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -16,13 +16,14 @@ from openai import AsyncOpenAI, AsyncStream, OpenAI, Stream from openai._base_client import make_request_options from openai._constants import RAW_RESPONSE_HEADER from openai._legacy_response import LegacyAPIResponse +from openai._models import construct_type from openai._types import RequestOptions from openai.pagination import SyncCursorPage from openai.types import CreateEmbeddingResponse from openai.types.beta.assistant_deleted import AssistantDeleted from openai.types.chat import ChatCompletion, ChatCompletionChunk from openai.types.file_deleted import FileDeleted -from pydantic import BaseModel, TypeAdapter +from pydantic import BaseModel, TypeAdapter, ValidationError from typing_extensions import overload import litellm @@ -390,13 +391,48 @@ _CLIENT_API_KEY_ONLY_AUTH: Final[Mapping[str, Mapping[str, bool]]] = ( ) +class _ChatCompletionChunkDecoder: + """ + Decodes stream chunks for the SDK's `ModelBuilderProtocol` hook in one compiled pydantic pass. Strict + validation only accepts payloads whose values already have the declared types, where it builds the same + chunk as the SDK's `construct_type`, and everything else keeps the SDK's lenient `construct_type` result. + """ + + @classmethod + def build(cls, *, response: httpx.Response, data: object) -> object: + try: + return ChatCompletionChunk.model_validate(data, strict=True) + except ValidationError: + return construct_type(type_=ChatCompletionChunk, value=data) + + def _raw_chat_response( - result: ChatCompletion | Stream[ChatCompletionChunk] | AsyncStream[ChatCompletionChunk], + result: ChatCompletion + | Stream[ChatCompletionChunk] + | AsyncStream[ChatCompletionChunk] + | Stream[_ChatCompletionChunkDecoder] + | AsyncStream[_ChatCompletionChunkDecoder], ) -> LegacyAPIResponse[ChatCompletion]: assert isinstance(result, LegacyAPIResponse) return result +def _sync_chat_stream_cls( + client: OpenAI, +) -> type[Stream[ChatCompletionChunk]] | type[Stream[_ChatCompletionChunkDecoder]]: + if client._strict_response_validation: # pyright: ignore[reportPrivateUsage] # SDK has no public getter + return Stream[ChatCompletionChunk] + return Stream[_ChatCompletionChunkDecoder] + + +def _async_chat_stream_cls( + client: AsyncOpenAI, +) -> type[AsyncStream[ChatCompletionChunk]] | type[AsyncStream[_ChatCompletionChunkDecoder]]: + if client._strict_response_validation: # pyright: ignore[reportPrivateUsage] # SDK has no public getter + return AsyncStream[ChatCompletionChunk] + return AsyncStream[_ChatCompletionChunkDecoder] + + def _request_without_sdk_transform( data: Mapping[str, object], timeout: float | httpx.Timeout ) -> tuple[Mapping[str, object], RequestOptions]: @@ -569,7 +605,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): cast_to=ChatCompletion, options=options, stream=bool(data.get("stream")), - stream_cls=AsyncStream[ChatCompletionChunk], + stream_cls=_async_chat_stream_cls(openai_aclient), ) ) end_time = time.time() @@ -625,7 +661,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): cast_to=ChatCompletion, options=options, stream=bool(data.get("stream")), - stream_cls=Stream[ChatCompletionChunk], + stream_cls=_sync_chat_stream_cls(openai_client), ) ) diff --git a/tests/unit/llms/openai/test_openai.py b/tests/unit/llms/openai/test_openai.py index 72afb840766..18444796906 100644 --- a/tests/unit/llms/openai/test_openai.py +++ b/tests/unit/llms/openai/test_openai.py @@ -2815,3 +2815,119 @@ async def test_sync_batch_list_given_an_async_client_returns_a_page_the_caller_c ) assert [(batch.id, batch.status) for batch in (await page).data] == [("batch_5f8dac", "pending")] + + +_STREAM_CHUNK_BASE: Final = { + "id": "chatcmpl-decode", + "object": "chat.completion.chunk", + "created": 1700000000, + "model": "gpt-5.5", +} +_STREAM_CHUNK_PAYLOADS: Final = ( + { + **_STREAM_CHUNK_BASE, + "system_fingerprint": "fp_1", + "choices": [ + {"index": 0, "delta": {"role": "assistant", "content": "hi"}, "logprobs": None, "finish_reason": None} + ], + }, + { + **_STREAM_CHUNK_BASE, + "choices": [ + { + "index": 0, + "delta": { + "tool_calls": [ + {"index": 0, "id": "call_1", "type": "function", "function": {"name": "f", "arguments": ""}}, + {"index": 1, "type": None, "function": {"arguments": None}}, + ] + }, + "finish_reason": None, + } + ], + }, + { + **_STREAM_CHUNK_BASE, + "choices": [], + "usage": { + "prompt_tokens": 9, + "completion_tokens": 2, + "total_tokens": 11, + "prompt_tokens_details": {"cached_tokens": 3}, + "cost": 0.5, + }, + }, + { + **_STREAM_CHUNK_BASE, + "choices": [ + { + "index": 0, + "delta": {"content": "a"}, + "logprobs": {"content": [{"token": "a", "logprob": 0, "bytes": [97], "top_logprobs": []}]}, + "finish_reason": "stop", + } + ], + }, + { + **_STREAM_CHUNK_BASE, + "provider": "upstream", + "citations": ["https://e.example"], + "choices": [ + { + "index": 0, + "delta": {"content": "", "reasoning_content": "think", "provider_specific_fields": {"k": 1}}, + "finish_reason": None, + "stop_reason": 128009, + } + ], + }, + { + **_STREAM_CHUNK_BASE, + "created": "1700000000", + "choices": [{"index": 0, "delta": {"content": "numeric string"}, "finish_reason": None}], + }, + { + **_STREAM_CHUNK_BASE, + "object": "chat.completion", + "choices": [{"index": 0, "delta": {"content": "loose"}, "finish_reason": "eos"}], + }, + {**_STREAM_CHUNK_BASE, "created": 1700000000.5, "choices": [{"index": "0", "delta": None}]}, +) + + +def _model_state(value: object) -> object: + if isinstance(value, BaseModel): + return ( + type(value), + {key: _model_state(item) for key, item in value.__dict__.items()}, + {key: _model_state(item) for key, item in (value.__pydantic_extra__ or {}).items()}, + frozenset(value.model_fields_set & type(value).model_fields.keys()), + ) + if isinstance(value, list): + return [_model_state(item) for item in value] + if isinstance(value, dict): + return {key: _model_state(item) for key, item in value.items()} + return (type(value), value) + + +@pytest.mark.asyncio +async def test_streamed_chat_chunks_decode_exactly_like_the_sdk_for_strict_and_loose_payloads(): + from openai._models import construct_type + from openai.types.chat import ChatCompletionChunk + + body: Final = b"".join(f"data: {json.dumps(p)}\n\n".encode() for p in _STREAM_CHUNK_PAYLOADS) + b"data: [DONE]\n\n" + transport: Final = httpx.MockTransport( + lambda _request: httpx.Response(200, content=body, headers={"content-type": "text/event-stream"}) + ) + async with httpx.AsyncClient(transport=transport) as http_client: + _, stream = await OpenAIChatCompletion().make_openai_chat_completion_request( + openai_aclient=AsyncOpenAI(api_key="transport-only", http_client=http_client), + data={"model": "gpt-5.5", "messages": [{"role": "user", "content": "q"}], "stream": True}, + timeout=10, + logging_obj=MagicMock(), + ) + decoded: Final = [chunk async for chunk in stream] + + assert [_model_state(chunk) for chunk in decoded] == [ + _model_state(construct_type(type_=ChatCompletionChunk, value=payload)) for payload in _STREAM_CHUNK_PAYLOADS + ]