diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 409f2cda287..682b2a2f538 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1202,7 +1202,10 @@ class CustomStreamWrapper: choice_json.pop( "finish_reason", None ) # for mistral etc. which return a value in their last chunk (not-openai compatible). - choices.append(StreamingChoices(**choice_json)) + choice_json.pop("logprobs", None) + choices.append( + StreamingChoices(**choice_json, logprobs=getattr(choice, "logprobs", None)) + ) except Exception: choices.append(StreamingChoices()) setattr(model_response, "choices", choices) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 30e43e46233..6ed21ad06e1 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1200,7 +1200,7 @@ class ChatCompletionTokenLogprob(OpenAIObject): unlikely. """ - top_logprobs: list[TopLogprob] + top_logprobs: list[TopLogprob] = Field(default_factory=list) """List of the most likely tokens and their log probability, at this token position. @@ -1242,6 +1242,8 @@ class ChoiceLogprobs(OpenAIObject): content: list[ChatCompletionTokenLogprob] | None = None """A list of message content tokens with log probability information.""" + refusal: Sequence[ChatCompletionTokenLogprob] | None = None + def __contains__(self, key) -> bool: # Define custom behavior for the 'in' operator return hasattr(self, key) @@ -1258,7 +1260,7 @@ class ChoiceLogprobs(OpenAIObject): def _sdk_logprobs_as_dict(logprobs: object) -> object: if isinstance(logprobs, ChoiceLogprobs) or not isinstance(logprobs, BaseModel): return logprobs - return logprobs.model_dump() + return logprobs.model_dump(exclude_unset=True) CHOICE_LOGPROBS_OR_JSON: Final[TypeAdapter[ChoiceLogprobs | JsonValue]] = TypeAdapter( @@ -2149,7 +2151,7 @@ class StreamingChoices(OpenAIObject): if enhancements is not None: self.enhancements = enhancements - if logprobs is not None and isinstance(logprobs, dict): + if isinstance(logprobs, (dict, BaseModel)): self.logprobs = CHOICE_LOGPROBS_OR_JSON.validate_python(logprobs) else: self.logprobs = logprobs diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index 7e2e263b7d7..8f78b50dba5 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -1815,6 +1815,13 @@ async def _sdk_stream_chunks( [], id="null-logprob", ), + pytest.param( + {"content": [{"token": "<|return|>", "logprob": None}]}, + {}, + None, + [], + id="null-logprob-keeps-only-the-keys-upstream-sent", + ), pytest.param(-0.25, {}, None, [], id="scalar-logprobs"), pytest.param( {"content": [{"token": "<|return|>", "bytes": None, "logprob": None, "top_logprobs": None}], "refusal": None}, @@ -1868,8 +1875,9 @@ async def test_stream_with_nonstandard_logprobs_passes_them_through_and_finishes @pytest.mark.parametrize("async_stream", [False, True]) +@pytest.mark.parametrize("logprobs_field", ["content", "refusal"]) async def test_stream_logprobs_without_top_logprobs_reach_caller_as_litellm_logprobs( - async_stream: bool, logging_obj: Logging + async_stream: bool, logprobs_field: Literal["content", "refusal"], logging_obj: Logging ) -> None: chunks: Final = await _sdk_stream_chunks( tuple( @@ -1878,7 +1886,7 @@ async def test_stream_logprobs_without_top_logprobs_reach_caller_as_litellm_logp { "index": 0, "delta": delta, - "logprobs": {"content": [{"token": token, "logprob": -0.5}]}, + "logprobs": {logprobs_field: [{"token": token, "logprob": -0.5}]}, "finish_reason": finish_reason, } ] @@ -1898,7 +1906,10 @@ async def test_stream_logprobs_without_top_logprobs_reach_caller_as_litellm_logp if choice.logprobs is not None ) streamed_entries: Final = tuple( - itertools.chain.from_iterable(logprobs.content or () for logprobs in streamed_logprobs) + itertools.chain.from_iterable( + (logprobs.content if logprobs_field == "content" else logprobs.refusal) or () + for logprobs in streamed_logprobs + ) ) assert [(entry.token, entry.logprob, entry.top_logprobs) for entry in streamed_entries] == [ diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 51348fa9c75..244b5d15417 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -29012,7 +29012,7 @@ export interface components { /** Token */ token: string; /** Top Logprobs */ - top_logprobs: components["schemas"]["TopLogprob"][]; + top_logprobs?: components["schemas"]["TopLogprob"][]; } & { [key: string]: unknown; }; @@ -29143,6 +29143,8 @@ export interface components { ChoiceLogprobs: { /** Content */ content?: components["schemas"]["ChatCompletionTokenLogprob"][] | null; + /** Refusal */ + refusal?: components["schemas"]["ChatCompletionTokenLogprob"][] | null; } & { [key: string]: unknown; };