From e2a826c47eae8a4240a25b521dd6207b9031dd79 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 18:33:13 -0700 Subject: [PATCH] fix(nvidia_nim): forward logprobs and return them on chat and /v1/responses NVIDIA NIM serves logprobs and top_logprobs, but nvidia_nim refused both with UnsupportedParamsError. The Responses bridge also never asked chat for logprobs when include requested message.output_text.logprobs, leaked top_logprobs on its own, and dropped the logprobs the model returned. It now maps include to chat logprobs and puts the tokens on output_text and the streaming events --- .../llms/nvidia_nim/chat/transformation.py | 2 + litellm/proxy/_lazy_openapi_snapshot.json | 69 +++++++++++ .../handler.py | 17 ++- .../streaming_iterator.py | 36 +++++- .../transformation.py | 28 ++++- litellm/types/llms/openai.py | 16 +-- litellm/types/responses/main.py | 16 ++- tests/unit/llms/nvidia_nim/test_nvidia_nim.py | 47 ++++++++ .../test_litellm_completion_responses.py | 112 +++++++++++++++++- .../test_streaming_iterator_transformation.py | 102 +++++++++++++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 26 ++++ 11 files changed, 447 insertions(+), 24 deletions(-) diff --git a/litellm/llms/nvidia_nim/chat/transformation.py b/litellm/llms/nvidia_nim/chat/transformation.py index 4bb005ed08c..8993c0e6b15 100644 --- a/litellm/llms/nvidia_nim/chat/transformation.py +++ b/litellm/llms/nvidia_nim/chat/transformation.py @@ -95,6 +95,8 @@ class NvidiaNimConfig(OpenAIGPTConfig): "tool_choice", "parallel_tool_calls", "response_format", + "logprobs", + "top_logprobs", ] def map_openai_params( diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index dc8b9415020..f320e790544 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -25762,6 +25762,61 @@ "title": "NoulAnswer", "type": "object" }, + "OpenAIChatCompletionLogprobsContent": { + "additionalProperties": true, + "properties": { + "bytes": { + "items": {}, + "title": "Bytes", + "type": "array" + }, + "logprob": { + "title": "Logprob", + "type": "number" + }, + "token": { + "title": "Token", + "type": "string" + }, + "top_logprobs": { + "items": { + "$ref": "#/components/schemas/OpenAIChatCompletionLogprobsContentTopLogprobs" + }, + "title": "Top Logprobs", + "type": "array" + } + }, + "required": [ + "logprob", + "token" + ], + "title": "OpenAIChatCompletionLogprobsContent", + "type": "object" + }, + "OpenAIChatCompletionLogprobsContentTopLogprobs": { + "additionalProperties": true, + "properties": { + "bytes": { + "items": {}, + "title": "Bytes", + "type": "array" + }, + "logprob": { + "title": "Logprob", + "type": "number" + }, + "token": { + "title": "Token", + "type": "string" + } + }, + "required": [ + "logprob", + "token" + ], + "title": "OpenAIChatCompletionLogprobsContentTopLogprobs", + "type": "object" + }, "OperationCreateFile": { "additionalProperties": true, "description": "Instruction describing how to create a file via the apply_patch tool.", @@ -26206,6 +26261,20 @@ ], "title": "Annotations" }, + "logprobs": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/OpenAIChatCompletionLogprobsContent" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Logprobs" + }, "text": { "anyOf": [ { diff --git a/litellm/responses/litellm_completion_transformation/handler.py b/litellm/responses/litellm_completion_transformation/handler.py index 1f56ddf863e..fc987cc51a4 100644 --- a/litellm/responses/litellm_completion_transformation/handler.py +++ b/litellm/responses/litellm_completion_transformation/handler.py @@ -31,6 +31,11 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ModelResponse _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(dict[str, object]) +_RESPONSES_FIELDS_THE_BRIDGE_TRANSLATES: Final = frozenset({"top_logprobs"}) + + +def _passthrough_kwargs(kwargs: Mapping[str, object]) -> Mapping[str, object]: + return {key: value for key, value in kwargs.items() if key not in _RESPONSES_FIELDS_THE_BRIDGE_TRANSLATES} @dataclass(frozen=True, slots=True) @@ -117,7 +122,11 @@ class LiteLLMCompletionTransformationHandler: ) litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = litellm.completion( - **{**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True}, + **{ + **_passthrough_kwargs(_REQUEST_KWARGS_ADAPTER.validate_python(kwargs)), + **litellm_completion_request, + "_skip_responses_api_bridge": True, + }, ) if isinstance(litellm_completion_response, ModelResponse): @@ -209,7 +218,11 @@ class LiteLLMCompletionTransformationHandler: ) litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = await litellm.acompletion( - **{**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True}, + **{ + **_passthrough_kwargs(_REQUEST_KWARGS_ADAPTER.validate_python(kwargs)), + **litellm_completion_request, + "_skip_responses_api_bridge": True, + }, ) if isinstance(litellm_completion_response, ModelResponse): responses_api_response: Final[ResponsesAPIResponse] = ( diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 892da541935..f455ec71c89 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -25,6 +25,7 @@ from litellm.responses.litellm_completion_transformation.tool_search import ( ) from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, + output_text_logprobs, ) from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator, stamp_responses_usage_cost from litellm.responses.utils import ResponsesAPIRequestUtils @@ -58,7 +59,11 @@ from litellm.types.llms.openai import ( WebSearchCallInProgressEvent, WebSearchCallSearchingEvent, ) -from litellm.types.responses.main import GenericResponseOutputItemContentAnnotation +from litellm.types.responses.main import ( + GenericResponseOutputItem, + GenericResponseOutputItemContentAnnotation, + OpenAIChatCompletionLogprobsContent, +) from litellm.types.utils import Delta as ChatCompletionDelta from litellm.types.utils import ( ModelResponse, @@ -88,6 +93,14 @@ def _output_items_with_id(items: tuple[Any, ...], item_type: str, item_id: str | ) +def _message_with_output_text_logprobs( + message: GenericResponseOutputItem, logprobs: Sequence[OpenAIChatCompletionLogprobsContent] +) -> GenericResponseOutputItem: + return message.model_copy( + update={"content": [part.model_copy(update={"logprobs": list(logprobs)}) for part in message.content]} + ) + + def _is_signed_thinking_block(block: object) -> bool: fields: Final = as_str_mapping(block) return fields is not None and isinstance(block, dict) and bool(fields.get("signature") or fields.get("data")) @@ -157,6 +170,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self.litellm_model_response: ModelResponse | TextCompletionResponse | None = None self.completed_response = None self.final_text: str = "" + self._output_text_logprobs: list[OpenAIChatCompletionLogprobsContent] = [] # mutable-ok: per-delta append self._cached_item_id: str | None = None self._message_output_index: int | None = None self._reasoning_output_index: int | None = None @@ -792,7 +806,9 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): item_id=self._cached_item_id, output_index=self._message_index(), content_index=0, - part=BaseLiteLLMOpenAIResponseObject(**{"type": "output_text", "text": "", "annotations": []}), + part=BaseLiteLLMOpenAIResponseObject( + **{"type": "output_text", "text": "", "annotations": [], "logprobs": []} + ), ) event.__dict__["sequence_number"] = self._sequence_number return event @@ -941,6 +957,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): output_index=self._message_index(), content_index=0, text=getattr(litellm_complete_object.choices[0].message, "content", "") or "", + logprobs=list(self._output_text_logprobs), ) def create_output_content_part_done_event(self, litellm_complete_object: ModelResponse) -> ContentPartDoneEvent: @@ -959,7 +976,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): type="output_text", text=text, annotations=response_annotations, - logprobs=None, + logprobs=[], ) return ContentPartDoneEvent( @@ -997,6 +1014,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): "type": "output_text", "text": text, "annotations": response_annotations, + "logprobs": list(self._output_text_logprobs), } ], } @@ -1416,6 +1434,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if not self.sent_message_item_added_event: self._queue_message_item_added_events() self._hold_tool_calls_behind_text(getattr(chunk.choices[0].delta, "tool_calls", None)) + delta_logprobs: Final = output_text_logprobs(chunk.choices[0].logprobs) + self._output_text_logprobs.extend(delta_logprobs) self._sequence_number += 1 text_delta_event: Final = OutputTextDeltaEvent( type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, @@ -1423,6 +1443,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): output_index=self._message_index(), content_index=0, delta=delta_content, + logprobs=delta_logprobs, ) text_delta_event.__dict__["sequence_number"] = self._sequence_number return text_delta_event @@ -1465,8 +1486,15 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): "message", self._cached_item_id, ) + message_index: Final = _index_of_output_item_type(message_aligned, "message") + logprobs_aligned: Final = tuple( + _message_with_output_text_logprobs(item, self._output_text_logprobs) + if index == message_index and isinstance(item, GenericResponseOutputItem) + else item + for index, item in enumerate(message_aligned) + ) reasoning_aligned: Final = _output_items_with_id( - message_aligned, + logprobs_aligned, "reasoning", self._cached_reasoning_item_id, ) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 8f0a85d67de..834e1f29303 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -109,6 +109,7 @@ from litellm.types.responses.main import ( CustomToolCallOutputItem, GenericResponseOutputItem, GenericResponseOutputItemContentAnnotation, + OpenAIChatCompletionLogprobsContent, OutputCodeInterpreterCall, OutputFunctionToolCall, OutputImageGenerationCall, @@ -118,6 +119,7 @@ from litellm.types.responses.main import ( from litellm.types.utils import ( ChatCompletionAnnotation, ChatCompletionMessageToolCall, + ChoiceLogprobs, Choices, Function, Message, @@ -217,6 +219,27 @@ def _echoable_request_params(request: Mapping[str, object]) -> _EchoedResponsesR return _ECHOED_PARAMS_ADAPTER.validate_python(echoable) +_OUTPUT_TEXT_LOGPROBS_INCLUDE: Final = "message.output_text.logprobs" +_OUTPUT_TEXT_LOGPROB_ADAPTER: Final = TypeAdapter(OpenAIChatCompletionLogprobsContent) + + +def _chat_logprob_params(request: ResponsesAPIOptionalRequestParams) -> Mapping[str, object]: + if "include" not in request or _OUTPUT_TEXT_LOGPROBS_INCLUDE not in (request["include"] or ()): + return {} + if "top_logprobs" not in request: + return {"logprobs": True} + return {"logprobs": True, "top_logprobs": request["top_logprobs"]} + + +def output_text_logprobs(chat_logprobs: object) -> Sequence[OpenAIChatCompletionLogprobsContent]: + if not isinstance(chat_logprobs, ChoiceLogprobs): + return [] + return [ + _OUTPUT_TEXT_LOGPROB_ADAPTER.validate_python(token_logprob.model_dump(exclude_none=True)) + for token_logprob in chat_logprobs.content or () + ] + + @dataclass(frozen=True, slots=True) class ResponsesToolChatForm: chat_tools: tuple[ChatToolParam, ...] @@ -1216,6 +1239,7 @@ class LiteLLMCompletionResponsesConfig: "metadata": kwargs.get("metadata"), "service_tier": responses_api_request.get("service_tier"), "safety_identifier": responses_api_request.get("safety_identifier"), + **_chat_logprob_params(responses_api_request), "web_search_options": web_search_options, "response_format": response_format, "reasoning_effort": reasoning.effort, @@ -3576,7 +3600,7 @@ class LiteLLMCompletionResponsesConfig: role=choice.message.role, content=[ LiteLLMCompletionResponsesConfig._transform_chat_message_to_response_output_text( - choice.message + choice.message, getattr(choice, "logprobs", None) ) ], ) @@ -3629,6 +3653,7 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def _transform_chat_message_to_response_output_text( message: Message, + logprobs: object, ) -> OutputText: annotations: Final = getattr(message, "annotations", None) transformed_annotations: Final = ( @@ -3641,6 +3666,7 @@ class LiteLLMCompletionResponsesConfig: type="output_text", text=message.content or "", annotations=transformed_annotations, + logprobs=output_text_logprobs(logprobs), ) @staticmethod diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index a54d66c4beb..9820d773c78 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -89,6 +89,7 @@ from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject, LiteLLMBase from litellm.types.responses.main import ( CustomToolCallOutputItem, GenericResponseOutputItem, + OpenAIChatCompletionLogprobsContent, OutputCodeInterpreterCall, OutputFunctionToolCall, OutputImageGenerationCall, @@ -1832,19 +1833,6 @@ class OutputItemDoneEvent(BaseLiteLLMOpenAIResponseObject): item: BaseLiteLLMOpenAIResponseObject -class OpenAIChatCompletionLogprobsContentTopLogprobs(TypedDict, total=False): - bytes: list - logprob: Required[float] - token: Required[str] - - -class OpenAIChatCompletionLogprobsContent(TypedDict, total=False): - bytes: list - logprob: Required[float] - token: Required[str] - top_logprobs: list[OpenAIChatCompletionLogprobsContentTopLogprobs] - - class ContentPartAddedEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.CONTENT_PART_ADDED] item_id: str @@ -1887,6 +1875,7 @@ class OutputTextDeltaEvent(BaseLiteLLMOpenAIResponseObject): output_index: int content_index: int delta: str + logprobs: Sequence[OpenAIChatCompletionLogprobsContent] | None = None class OutputTextAnnotationAddedEvent(BaseLiteLLMOpenAIResponseObject): @@ -1904,6 +1893,7 @@ class OutputTextDoneEvent(BaseLiteLLMOpenAIResponseObject): output_index: int content_index: int text: str + logprobs: Sequence[OpenAIChatCompletionLogprobsContent] | None = None class RefusalDeltaEvent(BaseLiteLLMOpenAIResponseObject): diff --git a/litellm/types/responses/main.py b/litellm/types/responses/main.py index 72dc63a581f..69689c935a2 100644 --- a/litellm/types/responses/main.py +++ b/litellm/types/responses/main.py @@ -5,7 +5,7 @@ from typing import Final, Literal, Optional, Union from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall from openai.types.responses.response_function_web_search import ActionSearchSource, ResponseFunctionWebSearch from pydantic import ConfigDict, PrivateAttr -from typing_extensions import TypedDict +from typing_extensions import Required, TypedDict from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject @@ -22,12 +22,26 @@ class GenericResponseOutputItemContentAnnotation(BaseLiteLLMOpenAIResponseObject title: str | None +class OpenAIChatCompletionLogprobsContentTopLogprobs(TypedDict, total=False): + bytes: list + logprob: Required[float] + token: Required[str] + + +class OpenAIChatCompletionLogprobsContent(TypedDict, total=False): + bytes: list + logprob: Required[float] + token: Required[str] + top_logprobs: list[OpenAIChatCompletionLogprobsContentTopLogprobs] + + class OutputText(BaseLiteLLMOpenAIResponseObject): """Text output content from an assistant message""" type: str | None # "output_text" text: str | None annotations: list[GenericResponseOutputItemContentAnnotation] | None + logprobs: Sequence[OpenAIChatCompletionLogprobsContent] | None = None class OutputFunctionToolCall(BaseLiteLLMOpenAIResponseObject): diff --git a/tests/unit/llms/nvidia_nim/test_nvidia_nim.py b/tests/unit/llms/nvidia_nim/test_nvidia_nim.py index 3633ac97265..c1bbbda8794 100644 --- a/tests/unit/llms/nvidia_nim/test_nvidia_nim.py +++ b/tests/unit/llms/nvidia_nim/test_nvidia_nim.py @@ -237,3 +237,50 @@ def test_completion_nvidia_nim(): assert request_body["model"] == "databricks/dbrx-instruct" assert request_body["frequency_penalty"] == 0.1 assert request_body["presence_penalty"] == 0.5 + + +def test_completion_nvidia_nim_forwards_logprobs_and_returns_them(): + from openai import OpenAI + from openai.types.chat import ChatCompletion + from openai.types.chat.chat_completion import Choice, ChoiceLogprobs + from openai.types.chat.chat_completion_message import ChatCompletionMessage + from openai.types.chat.chat_completion_token_logprob import ChatCompletionTokenLogprob, TopLogprob + + token_logprob: Final = ChatCompletionTokenLogprob( + token="hi", + logprob=-0.25, + bytes=[104, 105], + top_logprobs=[TopLogprob(token="hi", logprob=-0.25, bytes=[104, 105])], + ) + transport: Final = CapturingTransport( + ChatCompletion( + id="chatcmpl-nim", + object="chat.completion", + created=1, + model="openai/gpt-oss-20b", + choices=[ + Choice( + index=0, + finish_reason="stop", + message=ChatCompletionMessage(role="assistant", content="hi"), + logprobs=ChoiceLogprobs(content=[token_logprob]), + ) + ], + ) + ) + client: Final = OpenAI(api_key="fake-api-key", http_client=httpx.Client(transport=transport)) + + response: Final = completion( + model="nvidia_nim/openai/gpt-oss-20b", + messages=[{"role": "user", "content": "Say hi"}], + logprobs=True, + top_logprobs=1, + client=client, + ) + + request_body: Final = transport.request_bodies[0] + assert request_body["logprobs"] is True + assert request_body["top_logprobs"] == 1 + assert isinstance(response, ModelResponse) + assert response.choices[0].logprobs.content[0].token == token_logprob.token + assert response.choices[0].logprobs.content[0].logprob == token_logprob.logprob diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 4250c78c669..65d891acb06 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -5,8 +5,10 @@ from copy import deepcopy from types import MappingProxyType from typing import Final, Literal, cast +import httpx import pytest -from openai.types.responses import ResponseFunctionToolCall +from openai import OpenAI +from openai.types.responses import ResponseFunctionToolCall, ResponseIncludable from openai.types.responses.response_create_params import ResponseInputParam from openai.types.responses.response_function_web_search import ( ActionFind, @@ -33,7 +35,7 @@ from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) from litellm.types.llms.base import HiddenParams -from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams +from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams, ResponsesAPIResponse from litellm.types.responses.main import OutputToolSearchCall, build_web_search_call from litellm.types.utils import ( ChatCompletionMessageToolCall, @@ -45,6 +47,7 @@ from litellm.types.utils import ( PromptTokensDetailsWrapper, Usage, ) +from tests.capturing_transport import CapturingTransport class TestLiteLLMCompletionResponsesConfig: @@ -7101,3 +7104,108 @@ async def test_bridge_partial_response_on_disconnect_bills_only_the_input_since_ assert await _input_tokens_billed_on_disconnect( cast(ResponseInputParam, [*history_it_replaces, *since_compaction]) ) == await _input_tokens_billed_on_disconnect(since_compaction) + + +_NIM_CHAT_MODEL: Final = "nvidia_nim/meta/llama-3.3-70b-instruct" +_CHAT_TOKEN_LOGPROBS: Final = ( + { + "token": "Hi", + "logprob": -0.01, + "bytes": [72, 105], + "top_logprobs": [ + {"token": "Hi", "logprob": -0.01, "bytes": [72, 105]}, + {"token": "Hey", "logprob": -4.6, "bytes": [72, 101, 121]}, + ], + }, + { + "token": "!", + "logprob": -0.2, + "bytes": [33], + "top_logprobs": [ + {"token": "!", "logprob": -0.2, "bytes": [33]}, + {"token": ".", "logprob": -1.7, "bytes": [46]}, + ], + }, +) + + +def _bridged_nim_call( + include: Sequence[ResponseIncludable], + top_logprobs: int | None, + chat_logprobs: Mapping[str, object] | None = None, +) -> tuple[Mapping[str, object], ResponsesAPIResponse]: + transport: Final = CapturingTransport( + ModelResponse( + id="chatcmpl-logprobs", + created=1717000000, + model="meta/llama-3.3-70b-instruct", + object="chat.completion", + choices=[ + Choices( + index=0, + finish_reason="stop", + message=Message(role="assistant", content="Hi!"), + logprobs=chat_logprobs, + ) + ], + usage=Usage(prompt_tokens=8, completion_tokens=2, total_tokens=10), + ) + ) + response: Final = litellm.responses( + model=_NIM_CHAT_MODEL, + input="Say hi", + include=list(include) or None, + top_logprobs=top_logprobs, + api_key="nvapi-unused", + client=OpenAI(api_key="nvapi-unused", http_client=httpx.Client(transport=transport)), + ) + assert isinstance(response, ResponsesAPIResponse) + return transport.request_bodies[-1], response + + +@pytest.mark.parametrize( + ("include", "top_logprobs", "chat_logprob_params"), + [ + pytest.param(("message.output_text.logprobs",), 2, {"logprobs": True, "top_logprobs": 2}, id="include-and-top"), + pytest.param(("message.output_text.logprobs",), None, {"logprobs": True}, id="include-only"), + pytest.param((), 2, {}, id="top-logprobs-only"), + ], +) +def test_bridge_asks_chat_for_logprobs_only_when_include_requests_output_text_logprobs( + include: Sequence[ResponseIncludable], top_logprobs: int | None, chat_logprob_params: Mapping[str, object] +): + chat_request, _ = _bridged_nim_call(include, top_logprobs) + + assert { + key: chat_request[key] for key in ("logprobs", "top_logprobs") if key in chat_request + } == chat_logprob_params + + +def test_bridge_rejects_output_text_logprobs_for_a_provider_without_logprobs_with_a_400(): + with pytest.raises(litellm.UnsupportedParamsError) as exc_info: + litellm.responses( + model="anthropic/claude-haiku-4-5", + input="Say hi", + include=["message.output_text.logprobs"], + api_key="sk-unused", + ) + + assert exc_info.value.status_code == 400 + assert "logprobs" in str(exc_info.value) + + +@pytest.mark.parametrize( + ("chat_logprobs", "output_text_logprobs"), + [ + pytest.param({"content": list(_CHAT_TOKEN_LOGPROBS)}, list(_CHAT_TOKEN_LOGPROBS), id="chat-logprobs"), + pytest.param(None, [], id="no-chat-logprobs"), + ], +) +def test_bridged_output_text_carries_the_chat_token_logprobs( + chat_logprobs: Mapping[str, object] | None, output_text_logprobs: Sequence[Mapping[str, object]] +): + _, response = _bridged_nim_call(("message.output_text.logprobs",), 2, chat_logprobs) + + message: Final = response.model_dump(mode="json")["output"][0] + assert message["type"] == "message" + assert message["content"][0]["logprobs"] == output_text_logprobs diff --git a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index 86773dc7ddf..2ac52ef5eb9 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -12,7 +12,7 @@ spend tracking stores, so a follow-up previous_response_id still finds the conve import itertools import json -from collections.abc import Callable, Iterable +from collections.abc import Callable, Iterable, Mapping, Sequence from itertools import accumulate, chain, islice, product from typing import Final from unittest.mock import AsyncMock, MagicMock @@ -1988,3 +1988,103 @@ def test_the_partial_response_billed_on_disconnect_carries_the_logging_objects_b partial.response.usage.input_tokens, partial.response.usage.output_tokens, ), "the bill is priced on the partial's own usage" + + +_HI_LOGPROB: Final = { + "token": "Hi", + "logprob": -0.01, + "bytes": [72, 105], + "top_logprobs": [{"token": "Hi", "logprob": -0.01, "bytes": [72, 105]}], +} +_BANG_LOGPROB: Final = { + "token": "!", + "logprob": -0.2, + "bytes": [33], + "top_logprobs": [{"token": "!", "logprob": -0.2, "bytes": [33]}], +} +_THINK_LOGPROB: Final = {"token": "hmm", "logprob": -1.3, "bytes": [104, 109, 109], "top_logprobs": []} + + +def _logprobs_chunk( + delta: Delta, token_logprobs: Sequence[Mapping[str, object]], finish_reason: str | None = None +) -> ModelResponseStream: + return ModelResponseStream( + id=CHAT_COMPLETION_ID, + created=1748575031, + model="claude-haiku-4-5", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + index=0, + delta=delta, + finish_reason=finish_reason, + logprobs={"content": list(token_logprobs)}, + ) + ], + ) + + +async def _wire_events(chunks: Sequence[ModelResponseStream], sync_mode: bool) -> tuple[Mapping[str, object], ...]: + events: Final = await _collect_events(_build_iterator(chunks), sync_mode) + return tuple(event.model_dump(mode="json", exclude_none=True) for event in events) + + +def _events_of_type(events: Sequence[Mapping[str, object]], event_type: str) -> tuple[Mapping[str, object], ...]: + return tuple(event for event in events if event["type"] == event_type) + + +def _streamed_full_logprobs(events: Sequence[Mapping[str, object]]) -> Mapping[str, object]: + (text_done,) = _events_of_type(events, "response.output_text.done") + (message_done,) = [ + event for event in _events_of_type(events, "response.output_item.done") if event["item"]["type"] == "message" + ] + (completed,) = _events_of_type(events, "response.completed") + (completed_message,) = [item for item in completed["response"]["output"] if item["type"] == "message"] + return { + "response.output_text.done": text_done["logprobs"], + "response.output_item.done": message_done["item"]["content"][0]["logprobs"], + "response.completed": completed_message["content"][0]["logprobs"], + } + + +def _content_part_logprobs(events: Sequence[Mapping[str, object]]) -> tuple[object, ...]: + return tuple( + event["part"]["logprobs"] + for event in events + if event["type"] in ("response.content_part.added", "response.content_part.done") + ) + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_streamed_output_text_carries_each_deltas_logprobs_and_the_full_list_when_done(sync_mode: bool): + events: Final = await _wire_events( + ( + _logprobs_chunk(Delta(role="assistant", reasoning_content="hmm"), [_THINK_LOGPROB]), + _logprobs_chunk(Delta(role="assistant", content="Hi"), [_HI_LOGPROB]), + _logprobs_chunk(Delta(role="assistant", content="!"), [_BANG_LOGPROB], finish_reason="stop"), + ), + sync_mode, + ) + + assert [event["logprobs"] for event in _events_of_type(events, "response.output_text.delta")] == [ + [_HI_LOGPROB], + [_BANG_LOGPROB], + ] + assert _streamed_full_logprobs(events) == dict.fromkeys( + ("response.output_text.done", "response.output_item.done", "response.completed"), + [_HI_LOGPROB, _BANG_LOGPROB], + ) + assert _content_part_logprobs(events) == ([], []) + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_streamed_output_text_without_chat_logprobs_carries_empty_logprobs(sync_mode: bool): + events: Final = await _wire_events((_chunk("Hel"), _chunk("lo", finish_reason="stop")), sync_mode) + + assert [event["logprobs"] for event in _events_of_type(events, "response.output_text.delta")] == [[], []] + assert _streamed_full_logprobs(events) == dict.fromkeys( + ("response.output_text.done", "response.output_item.done", "response.completed"), [] + ) + assert _content_part_logprobs(events) == ([], []) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 30d61f181cb..cb84b933087 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -40049,6 +40049,30 @@ export interface components { */ type: "box"; }; + /** OpenAIChatCompletionLogprobsContent */ + OpenAIChatCompletionLogprobsContent: { + /** Bytes */ + bytes?: unknown[]; + /** Logprob */ + logprob: number; + /** Token */ + token: string; + /** Top Logprobs */ + top_logprobs?: components["schemas"]["OpenAIChatCompletionLogprobsContentTopLogprobs"][]; + } & { + [key: string]: unknown; + }; + /** OpenAIChatCompletionLogprobsContentTopLogprobs */ + OpenAIChatCompletionLogprobsContentTopLogprobs: { + /** Bytes */ + bytes?: unknown[]; + /** Logprob */ + logprob: number; + /** Token */ + token: string; + } & { + [key: string]: unknown; + }; /** OpenAIChoiceAnswer */ OpenAIChoiceAnswer: { /** Choice */ @@ -40582,6 +40606,8 @@ export interface components { OutputText: { /** Annotations */ annotations: components["schemas"]["GenericResponseOutputItemContentAnnotation"][] | null; + /** Logprobs */ + logprobs?: components["schemas"]["OpenAIChatCompletionLogprobsContent"][] | null; /** Text */ text: string | null; /** Type */