diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index c76975a5006..409f2cda287 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -10,14 +10,14 @@ import traceback from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType -from typing import TYPE_CHECKING, Annotated, Any, Final, NoReturn, Protocol, TypeVar, cast +from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypeVar, cast import anyio import httpx from openai.types.chat import ChatCompletionChunk from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice from openai.types.chat.chat_completion_chunk import ChoiceDelta -from pydantic import BaseModel, BeforeValidator, Field, JsonValue, TypeAdapter, ValidationError +from pydantic import BaseModel, ValidationError from typing_extensions import NotRequired, TypedDict import litellm @@ -40,8 +40,8 @@ from litellm.llms.ollama.common_utils import OllamaResponseIdNormalizer from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( + CHOICE_LOGPROBS_OR_JSON, CacheCreationTokenDetails, - ChoiceLogprobs, CompletionTokensDetailsWrapper, Delta, LlmProviders, @@ -141,21 +141,6 @@ def _delta_from_upstream(upstream_delta: "BaseModel | _UpstreamDeltaLike | None" return None -def _sdk_logprobs_as_dict(logprobs: object) -> object: - if isinstance(logprobs, ChoiceLogprobs) or not isinstance(logprobs, BaseModel): - return logprobs - return logprobs.model_dump(exclude_unset=True) - - -_UPSTREAM_LOGPROBS: Final[TypeAdapter[ChoiceLogprobs | JsonValue]] = TypeAdapter( - Annotated[ - ChoiceLogprobs | JsonValue, - Field(union_mode="left_to_right"), - BeforeValidator(_sdk_logprobs_as_dict), - ] -) - - def _is_transport_timeout(error: BaseException) -> bool: return isinstance(error, httpx.TimeoutException) or ( isinstance(error, Timeout) and isinstance(error.__cause__, httpx.TimeoutException) @@ -1621,7 +1606,7 @@ class CustomStreamWrapper: model_response.system_fingerprint = response_obj["original_chunk"].system_fingerprint self.system_fingerprint = response_obj["original_chunk"].system_fingerprint if response_obj["logprobs"] is not None: - model_response.choices[0].logprobs = _UPSTREAM_LOGPROBS.validate_python(response_obj["logprobs"]) + model_response.choices[0].logprobs = CHOICE_LOGPROBS_OR_JSON.validate_python(response_obj["logprobs"]) if response_obj["usage"] is not None: if isinstance(response_obj["usage"], dict): diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 266a6a20131..30e43e46233 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -7,6 +7,7 @@ from enum import Enum from types import MappingProxyType from typing import ( TYPE_CHECKING, + Annotated, Any, Final, Literal, @@ -37,6 +38,7 @@ from openai.types.moderation_create_response import ( ) from pydantic import ( BaseModel, + BeforeValidator, ConfigDict, Field, FieldSerializationInfo, @@ -44,6 +46,7 @@ from pydantic import ( JsonValue, PrivateAttr, SkipValidation, + TypeAdapter, field_serializer, field_validator, ) @@ -1234,6 +1237,8 @@ class ChatCompletionTokenLogprob(OpenAIObject): class ChoiceLogprobs(OpenAIObject): + model_config = ConfigDict(defer_build=False) + content: list[ChatCompletionTokenLogprob] | None = None """A list of message content tokens with log probability information.""" @@ -1250,6 +1255,21 @@ class ChoiceLogprobs(OpenAIObject): return getattr(self, key) +def _sdk_logprobs_as_dict(logprobs: object) -> object: + if isinstance(logprobs, ChoiceLogprobs) or not isinstance(logprobs, BaseModel): + return logprobs + return logprobs.model_dump() + + +CHOICE_LOGPROBS_OR_JSON: Final[TypeAdapter[ChoiceLogprobs | JsonValue]] = TypeAdapter( + Annotated[ + ChoiceLogprobs | JsonValue, + Field(union_mode="left_to_right"), + BeforeValidator(_sdk_logprobs_as_dict), + ] +) + + class FunctionCall(OpenAIObject): arguments: str name: str | None = None @@ -2130,7 +2150,7 @@ class StreamingChoices(OpenAIObject): self.enhancements = enhancements if logprobs is not None and isinstance(logprobs, dict): - self.logprobs = ChoiceLogprobs(**logprobs) + self.logprobs = CHOICE_LOGPROBS_OR_JSON.validate_python(logprobs) else: self.logprobs = logprobs diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index 7796241011c..7e2e263b7d7 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -2,12 +2,15 @@ import asyncio import datetime import itertools import json +import os import queue +import subprocess import sys import threading import time from collections.abc import Mapping, Sequence from dataclasses import dataclass +from pathlib import Path from types import CodeType, FrameType, MappingProxyType from typing import AsyncIterator, Callable, Final, Iterator, Literal, NoReturn, Optional, cast from unittest.mock import AsyncMock, MagicMock, Mock, patch @@ -1750,42 +1753,26 @@ async def test_stream_finish_chunk_with_logprobs_reaches_caller_as_litellm_logpr assert [chunk.usage.total_tokens for chunk in chunks if getattr(chunk, "usage", None)] == expected_usage_totals -@pytest.mark.parametrize("async_stream", [False, True]) -@pytest.mark.parametrize( - ("finish_logprobs", "expected_wire_logprobs"), - [ - pytest.param( - {"content": [{"token": "<|return|>"}]}, - {"content": [{"token": "<|return|>"}]}, - id="token-without-logprob", - ), - pytest.param(-0.25, -0.25, id="scalar-logprobs"), - ], -) -async def test_stream_finish_chunk_with_nonstandard_logprobs_still_finishes( +async def _sdk_stream_chunks( + upstream_chunks: Sequence[Mapping[str, object]], async_stream: bool, - finish_logprobs: object, - expected_wire_logprobs: object, logging_obj: Logging, -) -> None: - upstream_chunks: Final = ( - [{"index": 0, "delta": {"role": "assistant", "content": "hi"}, "finish_reason": None}], - [{"index": 0, "delta": {}, "logprobs": finish_logprobs, "finish_reason": "stop"}], - ) + stream_options: dict[str, bool] | None = None, +) -> list[ModelResponseStream]: body: Final = ( "".join( "data: " + json.dumps( { - "id": "chatcmpl-nonstandard-logprobs", + "id": "chatcmpl-logprobs", "object": "chat.completion.chunk", "created": 1, "model": "test-model", - "choices": choices, + **chunk, } ) + "\n\n" - for choices in upstream_chunks + for chunk in upstream_chunks ) + "data: [DONE]\n\n" ) @@ -1802,26 +1789,182 @@ async def test_stream_finish_chunk_with_nonstandard_logprobs_still_finishes( else Stream(cast_to=ChatCompletionChunk, response=raw_response, client=client) ) response: Final = CustomStreamWrapper( - completion_stream=stream, model="test-model", logging_obj=logging_obj, custom_llm_provider="openai" + completion_stream=stream, + model="test-model", + logging_obj=logging_obj, + custom_llm_provider="openai", + stream_options=stream_options, ) - try: - chunks: Final = [chunk async for chunk in response] if async_stream else list(response) + return [chunk async for chunk in response] if async_stream else list(response) finally: if isinstance(client, AsyncOpenAI): await client.close() else: client.close() + +@pytest.mark.parametrize("async_stream", [False, True]) +@pytest.mark.parametrize( + ("upstream_logprobs", "finish_chunk_extra", "stream_options", "expected_usage_totals"), + [ + pytest.param( + {"content": [{"token": "<|return|>", "bytes": None, "logprob": None, "top_logprobs": None}], "refusal": None}, + {}, + None, + [], + id="null-logprob", + ), + pytest.param(-0.25, {}, None, [], id="scalar-logprobs"), + pytest.param( + {"content": [{"token": "<|return|>", "bytes": None, "logprob": None, "top_logprobs": None}], "refusal": None}, + {"usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}}, + {"include_usage": True}, + [5], + id="null-logprob-and-usage", + ), + ], +) +async def test_stream_with_nonstandard_logprobs_passes_them_through_and_finishes( + async_stream: bool, + upstream_logprobs: object, + finish_chunk_extra: dict[str, object], + stream_options: dict[str, bool] | None, + expected_usage_totals: list[int], + logging_obj: Logging, +) -> None: + chunks: Final = await _sdk_stream_chunks( + ( + { + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": "hi"}, + "logprobs": upstream_logprobs, + "finish_reason": None, + } + ] + }, + { + "choices": [{"index": 0, "delta": {}, "logprobs": upstream_logprobs, "finish_reason": "stop"}], + **finish_chunk_extra, + }, + ), + async_stream, + logging_obj, + stream_options, + ) + wire_choices: Final = tuple( itertools.chain.from_iterable( json.loads(chunk.model_dump_json(exclude_none=True, exclude_unset=True))["choices"] for chunk in chunks ) ) - finish_choices: Final = [choice for choice in wire_choices if choice.get("finish_reason")] + logprobs_choices: Final = [choice for choice in wire_choices if "logprobs" in choice] + assert [choice.get("finish_reason") for choice in logprobs_choices] == [None, "stop"] + assert [choice["logprobs"] for choice in logprobs_choices] == [upstream_logprobs, upstream_logprobs] + assert [chunk.usage.total_tokens for chunk in chunks if getattr(chunk, "usage", None)] == expected_usage_totals + + +@pytest.mark.parametrize("async_stream", [False, True]) +async def test_stream_logprobs_without_top_logprobs_reach_caller_as_litellm_logprobs( + async_stream: bool, logging_obj: Logging +) -> None: + chunks: Final = await _sdk_stream_chunks( + tuple( + { + "choices": [ + { + "index": 0, + "delta": delta, + "logprobs": {"content": [{"token": token, "logprob": -0.5}]}, + "finish_reason": finish_reason, + } + ] + } + for token, delta, finish_reason in ( + ("hi", {"role": "assistant", "content": "hi"}, None), + ("<|return|>", {}, "stop"), + ) + ), + async_stream, + logging_obj, + ) + + streamed_logprobs: Final = tuple( + choice.logprobs + for choice in itertools.chain.from_iterable(chunk.choices for chunk in chunks) + if choice.logprobs is not None + ) + streamed_entries: Final = tuple( + itertools.chain.from_iterable(logprobs.content or () for logprobs in streamed_logprobs) + ) + + assert [(entry.token, entry.logprob, entry.top_logprobs) for entry in streamed_entries] == [ + ("hi", -0.5, []), + ("<|return|>", -0.5, []), + ] + + +FINISH_ONLY_LOGPROBS_STREAM_SCRIPT: Final = """ +import json +import sys + +sys.path.insert(0, sys.argv[1]) + +import httpx +from openai import OpenAI + +import litellm + +upstream_choices = ( + {"index": 0, "delta": {"role": "assistant", "content": "hi"}, "finish_reason": None}, + { + "index": 0, + "delta": {}, + "logprobs": {"content": [{"token": "<|return|>", "logprob": -0.5, "top_logprobs": []}]}, + "finish_reason": "stop", + }, +) +body = "".join( + "data: " + + json.dumps({"id": "c", "object": "chat.completion.chunk", "created": 1, "model": "m", "choices": [choice]}) + + "\\n\\n" + for choice in upstream_choices +) + "data: [DONE]\\n\\n" +client = OpenAI( + api_key="unused", + base_url="https://provider.example/v1", + http_client=httpx.Client( + transport=httpx.MockTransport( + lambda request: httpx.Response(200, content=body, headers={"content-type": "text/event-stream"}) + ) + ), +) +stream = litellm.completion( + model="openai/m", messages=[{"role": "user", "content": "hi"}], stream=True, logprobs=True, client=client +) +print(json.dumps([json.loads(chunk.model_dump_json(exclude_none=True)) for chunk in stream])) +""" + + +def test_stream_with_logprobs_only_on_finish_chunk_serializes_in_a_fresh_process() -> None: + result: Final = subprocess.run( + [sys.executable, "-I", "-c", FINISH_ONLY_LOGPROBS_STREAM_SCRIPT, str(Path(litellm.__file__).parent.parent)], + capture_output=True, + text=True, + timeout=120, + env={**os.environ, "LITELLM_LOCAL_MODEL_COST_MAP": "True"}, + ) + + assert result.returncode == 0, result.stdout + result.stderr + wire_choices: Final = tuple( + itertools.chain.from_iterable(chunk["choices"] for chunk in json.loads(result.stdout.strip().splitlines()[-1])) + ) + finish_choices: Final = [choice for choice in wire_choices if choice.get("finish_reason")] assert [choice["finish_reason"] for choice in finish_choices] == ["stop"] - assert finish_choices[0]["logprobs"] == expected_wire_logprobs + assert [entry["token"] for entry in finish_choices[0]["logprobs"]["content"]] == ["<|return|>"] def _hosted_vllm_stream_wrapper(logging_obj: Logging, error_payload: dict) -> CustomStreamWrapper: