scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-02 07:13:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 148

fix(nvidia_nim): forward logprobs and return them on chat and /v1/responses

Browse source 
NVIDIA NIM serves logprobs and top_logprobs, but nvidia_nim refused both with
UnsupportedParamsError. The Responses bridge also never asked chat for logprobs
when include requested message.output_text.logprobs, leaked top_logprobs on its
own, and dropped the logprobs the model returned. It now maps include to chat
logprobs and puts the tokens on output_text and the streaming events
This commit is contained in:
mateo-berri 2026-10-30 18:33:13 -07:00
parent ef9385c712
commit e2a826c47e
11 changed files with 447 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
litellm/llms/nvidia_nim/chat/transformation.py
View file

@ -95,6 +95,8 @@ class NvidiaNimConfig(OpenAIGPTConfig):
"tool_choice",
"parallel_tool_calls",
"response_format",
"logprobs",
"top_logprobs",
]
def map_openai_params(

69
litellm/proxy/_lazy_openapi_snapshot.json
View file

@ -25762,6 +25762,61 @@
"title": "NoulAnswer",
"type": "object"
},
"OpenAIChatCompletionLogprobsContent": {
"additionalProperties": true,
"properties": {
"bytes": {
"items": {},
"title": "Bytes",
"type": "array"
},
"logprob": {
"title": "Logprob",
"type": "number"
},
"token": {
"title": "Token",
"type": "string"
},
"top_logprobs": {
"items": {
"$ref": "#/components/schemas/OpenAIChatCompletionLogprobsContentTopLogprobs"
},
"title": "Top Logprobs",
"type": "array"
}
},
"required": [
"logprob",
"token"
],
"title": "OpenAIChatCompletionLogprobsContent",
"type": "object"
},
"OpenAIChatCompletionLogprobsContentTopLogprobs": {
"additionalProperties": true,
"properties": {
"bytes": {
"items": {},
"title": "Bytes",
"type": "array"
},
"logprob": {
"title": "Logprob",
"type": "number"
},
"token": {
"title": "Token",
"type": "string"
}
},
"required": [
"logprob",
"token"
],
"title": "OpenAIChatCompletionLogprobsContentTopLogprobs",
"type": "object"
},
"OperationCreateFile": {
"additionalProperties": true,
"description": "Instruction describing how to create a file via the apply_patch tool.",
@ -26206,6 +26261,20 @@
],
"title": "Annotations"
},
"logprobs": {
"anyOf": [
{
"items": {
"$ref": "#/components/schemas/OpenAIChatCompletionLogprobsContent"
},
"type": "array"
},
{
"type": "null"
}
],
"title": "Logprobs"
},
"text": {
"anyOf": [
{

17
litellm/responses/litellm_completion_transformation/handler.py
View file

@ -31,6 +31,11 @@ from litellm.types.llms.openai import (
from litellm.types.utils import ModelResponse
_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(dict[str, object])
_RESPONSES_FIELDS_THE_BRIDGE_TRANSLATES: Final = frozenset({"top_logprobs"})
def _passthrough_kwargs(kwargs: Mapping[str, object]) -> Mapping[str, object]:
return {key: value for key, value in kwargs.items() if key not in _RESPONSES_FIELDS_THE_BRIDGE_TRANSLATES}
@dataclass(frozen=True, slots=True)
@ -117,7 +122,11 @@ class LiteLLMCompletionTransformationHandler:
)
litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = litellm.completion(
**{**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True},
**{
**_passthrough_kwargs(_REQUEST_KWARGS_ADAPTER.validate_python(kwargs)),
**litellm_completion_request,
"_skip_responses_api_bridge": True,
},
)
if isinstance(litellm_completion_response, ModelResponse):
@ -209,7 +218,11 @@ class LiteLLMCompletionTransformationHandler:
)
litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = await litellm.acompletion(
**{**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True},
**{
**_passthrough_kwargs(_REQUEST_KWARGS_ADAPTER.validate_python(kwargs)),
**litellm_completion_request,
"_skip_responses_api_bridge": True,
},
)
if isinstance(litellm_completion_response, ModelResponse):
responses_api_response: Final[ResponsesAPIResponse] = (

36
litellm/responses/litellm_completion_transformation/streaming_iterator.py
View file

@ -25,6 +25,7 @@ from litellm.responses.litellm_completion_transformation.tool_search import (
)
from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
output_text_logprobs,
)
from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator, stamp_responses_usage_cost
from litellm.responses.utils import ResponsesAPIRequestUtils
@ -58,7 +59,11 @@ from litellm.types.llms.openai import (
WebSearchCallInProgressEvent,
WebSearchCallSearchingEvent,
)
from litellm.types.responses.main import GenericResponseOutputItemContentAnnotation
from litellm.types.responses.main import (
GenericResponseOutputItem,
GenericResponseOutputItemContentAnnotation,
OpenAIChatCompletionLogprobsContent,
)
from litellm.types.utils import Delta as ChatCompletionDelta
from litellm.types.utils import (
ModelResponse,
@ -88,6 +93,14 @@ def _output_items_with_id(items: tuple[Any, ...], item_type: str, item_id: str |
)
def _message_with_output_text_logprobs(
message: GenericResponseOutputItem, logprobs: Sequence[OpenAIChatCompletionLogprobsContent]
) -> GenericResponseOutputItem:
return message.model_copy(
update={"content": [part.model_copy(update={"logprobs": list(logprobs)}) for part in message.content]}
)
def _is_signed_thinking_block(block: object) -> bool:
fields: Final = as_str_mapping(block)
return fields is not None and isinstance(block, dict) and bool(fields.get("signature") or fields.get("data"))
@ -157,6 +170,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
self.litellm_model_response: ModelResponse | TextCompletionResponse | None = None
self.completed_response = None
self.final_text: str = ""
self._output_text_logprobs: list[OpenAIChatCompletionLogprobsContent] = [] # mutable-ok: per-delta append
self._cached_item_id: str | None = None
self._message_output_index: int | None = None
self._reasoning_output_index: int | None = None
@ -792,7 +806,9 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
item_id=self._cached_item_id,
output_index=self._message_index(),
content_index=0,
part=BaseLiteLLMOpenAIResponseObject(**{"type": "output_text", "text": "", "annotations": []}),
part=BaseLiteLLMOpenAIResponseObject(
**{"type": "output_text", "text": "", "annotations": [], "logprobs": []}
),
)
event.__dict__["sequence_number"] = self._sequence_number
return event
@ -941,6 +957,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
output_index=self._message_index(),
content_index=0,
text=getattr(litellm_complete_object.choices[0].message, "content", "") or "",
logprobs=list(self._output_text_logprobs),
)
def create_output_content_part_done_event(self, litellm_complete_object: ModelResponse) -> ContentPartDoneEvent:
@ -959,7 +976,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
type="output_text",
text=text,
annotations=response_annotations,
logprobs=None,
logprobs=[],
)
return ContentPartDoneEvent(
@ -997,6 +1014,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
"type": "output_text",
"text": text,
"annotations": response_annotations,
"logprobs": list(self._output_text_logprobs),
}
],
}
@ -1416,6 +1434,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
if not self.sent_message_item_added_event:
self._queue_message_item_added_events()
self._hold_tool_calls_behind_text(getattr(chunk.choices[0].delta, "tool_calls", None))
delta_logprobs: Final = output_text_logprobs(chunk.choices[0].logprobs)
self._output_text_logprobs.extend(delta_logprobs)
self._sequence_number += 1
text_delta_event: Final = OutputTextDeltaEvent(
type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA,
@ -1423,6 +1443,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
output_index=self._message_index(),
content_index=0,
delta=delta_content,
logprobs=delta_logprobs,
)
text_delta_event.__dict__["sequence_number"] = self._sequence_number
return text_delta_event
@ -1465,8 +1486,15 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
"message",
self._cached_item_id,
)
message_index: Final = _index_of_output_item_type(message_aligned, "message")
logprobs_aligned: Final = tuple(
_message_with_output_text_logprobs(item, self._output_text_logprobs)
if index == message_index and isinstance(item, GenericResponseOutputItem)
else item
for index, item in enumerate(message_aligned)
)
reasoning_aligned: Final = _output_items_with_id(
message_aligned,
logprobs_aligned,
"reasoning",
self._cached_reasoning_item_id,
)

28
litellm/responses/litellm_completion_transformation/transformation.py
View file

@ -109,6 +109,7 @@ from litellm.types.responses.main import (
CustomToolCallOutputItem,
GenericResponseOutputItem,
GenericResponseOutputItemContentAnnotation,
OpenAIChatCompletionLogprobsContent,
OutputCodeInterpreterCall,
OutputFunctionToolCall,
OutputImageGenerationCall,
@ -118,6 +119,7 @@ from litellm.types.responses.main import (
from litellm.types.utils import (
ChatCompletionAnnotation,
ChatCompletionMessageToolCall,
ChoiceLogprobs,
Choices,
Function,
Message,
@ -217,6 +219,27 @@ def _echoable_request_params(request: Mapping[str, object]) -> _EchoedResponsesR
return _ECHOED_PARAMS_ADAPTER.validate_python(echoable)
_OUTPUT_TEXT_LOGPROBS_INCLUDE: Final = "message.output_text.logprobs"
_OUTPUT_TEXT_LOGPROB_ADAPTER: Final = TypeAdapter(OpenAIChatCompletionLogprobsContent)
def _chat_logprob_params(request: ResponsesAPIOptionalRequestParams) -> Mapping[str, object]:
if "include" not in request or _OUTPUT_TEXT_LOGPROBS_INCLUDE not in (request["include"] or ()):
return {}
if "top_logprobs" not in request:
return {"logprobs": True}
return {"logprobs": True, "top_logprobs": request["top_logprobs"]}
def output_text_logprobs(chat_logprobs: object) -> Sequence[OpenAIChatCompletionLogprobsContent]:
if not isinstance(chat_logprobs, ChoiceLogprobs):
return []
return [
_OUTPUT_TEXT_LOGPROB_ADAPTER.validate_python(token_logprob.model_dump(exclude_none=True))
for token_logprob in chat_logprobs.content or ()
]
@dataclass(frozen=True, slots=True)
class ResponsesToolChatForm:
chat_tools: tuple[ChatToolParam, ...]
@ -1216,6 +1239,7 @@ class LiteLLMCompletionResponsesConfig:
"metadata": kwargs.get("metadata"),
"service_tier": responses_api_request.get("service_tier"),
"safety_identifier": responses_api_request.get("safety_identifier"),
**_chat_logprob_params(responses_api_request),
"web_search_options": web_search_options,
"response_format": response_format,
"reasoning_effort": reasoning.effort,
@ -3576,7 +3600,7 @@ class LiteLLMCompletionResponsesConfig:
role=choice.message.role,
content=[
LiteLLMCompletionResponsesConfig._transform_chat_message_to_response_output_text(
choice.message
choice.message, getattr(choice, "logprobs", None)
)
],
)
@ -3629,6 +3653,7 @@ class LiteLLMCompletionResponsesConfig:
@staticmethod
def _transform_chat_message_to_response_output_text(
message: Message,
logprobs: object,
) -> OutputText:
annotations: Final = getattr(message, "annotations", None)
transformed_annotations: Final = (
@ -3641,6 +3666,7 @@ class LiteLLMCompletionResponsesConfig:
type="output_text",
text=message.content or "",
annotations=transformed_annotations,
logprobs=output_text_logprobs(logprobs),
)
@staticmethod

16
litellm/types/llms/openai.py
View file

@ -89,6 +89,7 @@ from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject, LiteLLMBase
from litellm.types.responses.main import (
CustomToolCallOutputItem,
GenericResponseOutputItem,
OpenAIChatCompletionLogprobsContent,
OutputCodeInterpreterCall,
OutputFunctionToolCall,
OutputImageGenerationCall,
@ -1832,19 +1833,6 @@ class OutputItemDoneEvent(BaseLiteLLMOpenAIResponseObject):
item: BaseLiteLLMOpenAIResponseObject
class OpenAIChatCompletionLogprobsContentTopLogprobs(TypedDict, total=False):
bytes: list
logprob: Required[float]
token: Required[str]
class OpenAIChatCompletionLogprobsContent(TypedDict, total=False):
bytes: list
logprob: Required[float]
token: Required[str]
top_logprobs: list[OpenAIChatCompletionLogprobsContentTopLogprobs]
class ContentPartAddedEvent(BaseLiteLLMOpenAIResponseObject):
type: Literal[ResponsesAPIStreamEvents.CONTENT_PART_ADDED]
item_id: str
@ -1887,6 +1875,7 @@ class OutputTextDeltaEvent(BaseLiteLLMOpenAIResponseObject):
output_index: int
content_index: int
delta: str
logprobs: Sequence[OpenAIChatCompletionLogprobsContent] | None = None
class OutputTextAnnotationAddedEvent(BaseLiteLLMOpenAIResponseObject):
@ -1904,6 +1893,7 @@ class OutputTextDoneEvent(BaseLiteLLMOpenAIResponseObject):
output_index: int
content_index: int
text: str
logprobs: Sequence[OpenAIChatCompletionLogprobsContent] | None = None
class RefusalDeltaEvent(BaseLiteLLMOpenAIResponseObject):

16
litellm/types/responses/main.py
View file

@ -5,7 +5,7 @@ from typing import Final, Literal, Optional, Union
from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall
from openai.types.responses.response_function_web_search import ActionSearchSource, ResponseFunctionWebSearch
from pydantic import ConfigDict, PrivateAttr
from typing_extensions import TypedDict
from typing_extensions import Required, TypedDict
from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject
@ -22,12 +22,26 @@ class GenericResponseOutputItemContentAnnotation(BaseLiteLLMOpenAIResponseObject
title: str | None
class OpenAIChatCompletionLogprobsContentTopLogprobs(TypedDict, total=False):
bytes: list
logprob: Required[float]
token: Required[str]
class OpenAIChatCompletionLogprobsContent(TypedDict, total=False):
bytes: list
logprob: Required[float]
token: Required[str]
top_logprobs: list[OpenAIChatCompletionLogprobsContentTopLogprobs]
class OutputText(BaseLiteLLMOpenAIResponseObject):
"""Text output content from an assistant message"""
type: str | None # "output_text"
text: str | None
annotations: list[GenericResponseOutputItemContentAnnotation] | None
logprobs: Sequence[OpenAIChatCompletionLogprobsContent] | None = None
class OutputFunctionToolCall(BaseLiteLLMOpenAIResponseObject):

47
tests/unit/llms/nvidia_nim/test_nvidia_nim.py
View file

@ -237,3 +237,50 @@ def test_completion_nvidia_nim():
assert request_body["model"] == "databricks/dbrx-instruct"
assert request_body["frequency_penalty"] == 0.1
assert request_body["presence_penalty"] == 0.5
def test_completion_nvidia_nim_forwards_logprobs_and_returns_them():
from openai import OpenAI
from openai.types.chat import ChatCompletion
from openai.types.chat.chat_completion import Choice, ChoiceLogprobs
from openai.types.chat.chat_completion_message import ChatCompletionMessage
from openai.types.chat.chat_completion_token_logprob import ChatCompletionTokenLogprob, TopLogprob
token_logprob: Final = ChatCompletionTokenLogprob(
token="hi",
logprob=-0.25,
bytes=[104, 105],
top_logprobs=[TopLogprob(token="hi", logprob=-0.25, bytes=[104, 105])],
)
transport: Final = CapturingTransport(
ChatCompletion(
id="chatcmpl-nim",
object="chat.completion",
created=1,
model="openai/gpt-oss-20b",
choices=[
Choice(
index=0,
finish_reason="stop",
message=ChatCompletionMessage(role="assistant", content="hi"),
logprobs=ChoiceLogprobs(content=[token_logprob]),
)
],
)
)
client: Final = OpenAI(api_key="fake-api-key", http_client=httpx.Client(transport=transport))
response: Final = completion(
model="nvidia_nim/openai/gpt-oss-20b",
messages=[{"role": "user", "content": "Say hi"}],
logprobs=True,
top_logprobs=1,
client=client,
)
request_body: Final = transport.request_bodies[0]
assert request_body["logprobs"] is True
assert request_body["top_logprobs"] == 1
assert isinstance(response, ModelResponse)
assert response.choices[0].logprobs.content[0].token == token_logprob.token
assert response.choices[0].logprobs.content[0].logprob == token_logprob.logprob

112
tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py
View file

@ -5,8 +5,10 @@ from copy import deepcopy
from types import MappingProxyType
from typing import Final, Literal, cast
import httpx
import pytest
from openai.types.responses import ResponseFunctionToolCall
from openai import OpenAI
from openai.types.responses import ResponseFunctionToolCall, ResponseIncludable
from openai.types.responses.response_create_params import ResponseInputParam
from openai.types.responses.response_function_web_search import (
ActionFind,
@ -33,7 +35,7 @@ from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
)
from litellm.types.llms.base import HiddenParams
from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams
from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams, ResponsesAPIResponse
from litellm.types.responses.main import OutputToolSearchCall, build_web_search_call
from litellm.types.utils import (
ChatCompletionMessageToolCall,
@ -45,6 +47,7 @@ from litellm.types.utils import (
PromptTokensDetailsWrapper,
Usage,
)
from tests.capturing_transport import CapturingTransport
class TestLiteLLMCompletionResponsesConfig:
@ -7101,3 +7104,108 @@ async def test_bridge_partial_response_on_disconnect_bills_only_the_input_since_
assert await _input_tokens_billed_on_disconnect(
cast(ResponseInputParam, [*history_it_replaces, *since_compaction])
) == await _input_tokens_billed_on_disconnect(since_compaction)
_NIM_CHAT_MODEL: Final = "nvidia_nim/meta/llama-3.3-70b-instruct"
_CHAT_TOKEN_LOGPROBS: Final = (
{
"token": "Hi",
"logprob": -0.01,
"bytes": [72, 105],
"top_logprobs": [
{"token": "Hi", "logprob": -0.01, "bytes": [72, 105]},
{"token": "Hey", "logprob": -4.6, "bytes": [72, 101, 121]},
],
},
{
"token": "!",
"logprob": -0.2,
"bytes": [33],
"top_logprobs": [
{"token": "!", "logprob": -0.2, "bytes": [33]},
{"token": ".", "logprob": -1.7, "bytes": [46]},
],
},
)
def _bridged_nim_call(
include: Sequence[ResponseIncludable],
top_logprobs: int | None,
chat_logprobs: Mapping[str, object] | None = None,
) -> tuple[Mapping[str, object], ResponsesAPIResponse]:
transport: Final = CapturingTransport(
ModelResponse(
id="chatcmpl-logprobs",
created=1717000000,
model="meta/llama-3.3-70b-instruct",
object="chat.completion",
choices=[
Choices(
index=0,
finish_reason="stop",
message=Message(role="assistant", content="Hi!"),
logprobs=chat_logprobs,
)
],
usage=Usage(prompt_tokens=8, completion_tokens=2, total_tokens=10),
)
)
response: Final = litellm.responses(
model=_NIM_CHAT_MODEL,
input="Say hi",
include=list(include) or None,
top_logprobs=top_logprobs,
api_key="nvapi-unused",
client=OpenAI(api_key="nvapi-unused", http_client=httpx.Client(transport=transport)),
)
assert isinstance(response, ResponsesAPIResponse)
return transport.request_bodies[-1], response
@pytest.mark.parametrize(
("include", "top_logprobs", "chat_logprob_params"),
[
pytest.param(("message.output_text.logprobs",), 2, {"logprobs": True, "top_logprobs": 2}, id="include-and-top"),
pytest.param(("message.output_text.logprobs",), None, {"logprobs": True}, id="include-only"),
pytest.param((), 2, {}, id="top-logprobs-only"),
],
)
def test_bridge_asks_chat_for_logprobs_only_when_include_requests_output_text_logprobs(
include: Sequence[ResponseIncludable], top_logprobs: int | None, chat_logprob_params: Mapping[str, object]
):
chat_request, _ = _bridged_nim_call(include, top_logprobs)
assert {
key: chat_request[key] for key in ("logprobs", "top_logprobs") if key in chat_request
} == chat_logprob_params
def test_bridge_rejects_output_text_logprobs_for_a_provider_without_logprobs_with_a_400():
with pytest.raises(litellm.UnsupportedParamsError) as exc_info:
litellm.responses(
model="anthropic/claude-haiku-4-5",
input="Say hi",
include=["message.output_text.logprobs"],
api_key="sk-unused",
)
assert exc_info.value.status_code == 400
assert "logprobs" in str(exc_info.value)
@pytest.mark.parametrize(
("chat_logprobs", "output_text_logprobs"),
[
pytest.param({"content": list(_CHAT_TOKEN_LOGPROBS)}, list(_CHAT_TOKEN_LOGPROBS), id="chat-logprobs"),
pytest.param(None, [], id="no-chat-logprobs"),
],
)
def test_bridged_output_text_carries_the_chat_token_logprobs(
chat_logprobs: Mapping[str, object] | None, output_text_logprobs: Sequence[Mapping[str, object]]
):
_, response = _bridged_nim_call(("message.output_text.logprobs",), 2, chat_logprobs)
message: Final = response.model_dump(mode="json")["output"][0]
assert message["type"] == "message"
assert message["content"][0]["logprobs"] == output_text_logprobs

102
tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py
View file

@ -12,7 +12,7 @@ spend tracking stores, so a follow-up previous_response_id still finds the conve
import itertools
import json
from collections.abc import Callable, Iterable
from collections.abc import Callable, Iterable, Mapping, Sequence
from itertools import accumulate, chain, islice, product
from typing import Final
from unittest.mock import AsyncMock, MagicMock
@ -1988,3 +1988,103 @@ def test_the_partial_response_billed_on_disconnect_carries_the_logging_objects_b
partial.response.usage.input_tokens,
partial.response.usage.output_tokens,
), "the bill is priced on the partial's own usage"
_HI_LOGPROB: Final = {
"token": "Hi",
"logprob": -0.01,
"bytes": [72, 105],
"top_logprobs": [{"token": "Hi", "logprob": -0.01, "bytes": [72, 105]}],
}
_BANG_LOGPROB: Final = {
"token": "!",
"logprob": -0.2,
"bytes": [33],
"top_logprobs": [{"token": "!", "logprob": -0.2, "bytes": [33]}],
}
_THINK_LOGPROB: Final = {"token": "hmm", "logprob": -1.3, "bytes": [104, 109, 109], "top_logprobs": []}
def _logprobs_chunk(
delta: Delta, token_logprobs: Sequence[Mapping[str, object]], finish_reason: str | None = None
) -> ModelResponseStream:
return ModelResponseStream(
id=CHAT_COMPLETION_ID,
created=1748575031,
model="claude-haiku-4-5",
object="chat.completion.chunk",
choices=[
StreamingChoices(
index=0,
delta=delta,
finish_reason=finish_reason,
logprobs={"content": list(token_logprobs)},
)
],
)
async def _wire_events(chunks: Sequence[ModelResponseStream], sync_mode: bool) -> tuple[Mapping[str, object], ...]:
events: Final = await _collect_events(_build_iterator(chunks), sync_mode)
return tuple(event.model_dump(mode="json", exclude_none=True) for event in events)
def _events_of_type(events: Sequence[Mapping[str, object]], event_type: str) -> tuple[Mapping[str, object], ...]:
return tuple(event for event in events if event["type"] == event_type)
def _streamed_full_logprobs(events: Sequence[Mapping[str, object]]) -> Mapping[str, object]:
(text_done,) = _events_of_type(events, "response.output_text.done")
(message_done,) = [
event for event in _events_of_type(events, "response.output_item.done") if event["item"]["type"] == "message"
]
(completed,) = _events_of_type(events, "response.completed")
(completed_message,) = [item for item in completed["response"]["output"] if item["type"] == "message"]
return {
"response.output_text.done": text_done["logprobs"],
"response.output_item.done": message_done["item"]["content"][0]["logprobs"],
"response.completed": completed_message["content"][0]["logprobs"],
}
def _content_part_logprobs(events: Sequence[Mapping[str, object]]) -> tuple[object, ...]:
return tuple(
event["part"]["logprobs"]
for event in events
if event["type"] in ("response.content_part.added", "response.content_part.done")
)
@pytest.mark.parametrize("sync_mode", [True, False])
@pytest.mark.asyncio
async def test_streamed_output_text_carries_each_deltas_logprobs_and_the_full_list_when_done(sync_mode: bool):
events: Final = await _wire_events(
(
_logprobs_chunk(Delta(role="assistant", reasoning_content="hmm"), [_THINK_LOGPROB]),
_logprobs_chunk(Delta(role="assistant", content="Hi"), [_HI_LOGPROB]),
_logprobs_chunk(Delta(role="assistant", content="!"), [_BANG_LOGPROB], finish_reason="stop"),
),
sync_mode,
)
assert [event["logprobs"] for event in _events_of_type(events, "response.output_text.delta")] == [
[_HI_LOGPROB],
[_BANG_LOGPROB],
]
assert _streamed_full_logprobs(events) == dict.fromkeys(
("response.output_text.done", "response.output_item.done", "response.completed"),
[_HI_LOGPROB, _BANG_LOGPROB],
)
assert _content_part_logprobs(events) == ([], [])
@pytest.mark.parametrize("sync_mode", [True, False])
@pytest.mark.asyncio
async def test_streamed_output_text_without_chat_logprobs_carries_empty_logprobs(sync_mode: bool):
events: Final = await _wire_events((_chunk("Hel"), _chunk("lo", finish_reason="stop")), sync_mode)
assert [event["logprobs"] for event in _events_of_type(events, "response.output_text.delta")] == [[], []]
assert _streamed_full_logprobs(events) == dict.fromkeys(
("response.output_text.done", "response.output_item.done", "response.completed"), []
)
assert _content_part_logprobs(events) == ([], [])

26
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -40049,6 +40049,30 @@ export interface components {
*/
type: "box";
};
/** OpenAIChatCompletionLogprobsContent */
OpenAIChatCompletionLogprobsContent: {
/** Bytes */
bytes?: unknown[];
/** Logprob */
logprob: number;
/** Token */
token: string;
/** Top Logprobs */
top_logprobs?: components["schemas"]["OpenAIChatCompletionLogprobsContentTopLogprobs"][];
} & {
[key: string]: unknown;
};
/** OpenAIChatCompletionLogprobsContentTopLogprobs */
OpenAIChatCompletionLogprobsContentTopLogprobs: {
/** Bytes */
bytes?: unknown[];
/** Logprob */
logprob: number;
/** Token */
token: string;
} & {
[key: string]: unknown;
};
/** OpenAIChoiceAnswer */
OpenAIChoiceAnswer: {
/** Choice */
@ -40582,6 +40606,8 @@ export interface components {
OutputText: {
/** Annotations */
annotations: components["schemas"]["GenericResponseOutputItemContentAnnotation"][] | null;
/** Logprobs */
logprobs?: components["schemas"]["OpenAIChatCompletionLogprobsContent"][] | null;
/** Text */
text: string | null;
/** Type */