scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-11 08:38:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 280

refactor(responses): leave the bridge logprobs to the open bridge PR and keep this change to nvidia_nim params

Browse source
This commit is contained in:
mateo-berri 2026-11-02 11:18:54 -08:00
parent 2ea71ad748
commit 695e0e8090
9 changed files with 27 additions and 419 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

69
litellm/proxy/_lazy_openapi_snapshot.json
View file

@ -26124,61 +26124,6 @@
"title": "NoulAnswer",
"type": "object"
},
"OpenAIChatCompletionLogprobsContent": {
"additionalProperties": true,
"properties": {
"bytes": {
"items": {},
"title": "Bytes",
"type": "array"
},
"logprob": {
"title": "Logprob",
"type": "number"
},
"token": {
"title": "Token",
"type": "string"
},
"top_logprobs": {
"items": {
"$ref": "#/components/schemas/OpenAIChatCompletionLogprobsContentTopLogprobs"
},
"title": "Top Logprobs",
"type": "array"
}
},
"required": [
"logprob",
"token"
],
"title": "OpenAIChatCompletionLogprobsContent",
"type": "object"
},
"OpenAIChatCompletionLogprobsContentTopLogprobs": {
"additionalProperties": true,
"properties": {
"bytes": {
"items": {},
"title": "Bytes",
"type": "array"
},
"logprob": {
"title": "Logprob",
"type": "number"
},
"token": {
"title": "Token",
"type": "string"
}
},
"required": [
"logprob",
"token"
],
"title": "OpenAIChatCompletionLogprobsContentTopLogprobs",
"type": "object"
},
"OperationCreateFile": {
"additionalProperties": true,
"description": "Instruction describing how to create a file via the apply_patch tool.",
@ -26623,20 +26568,6 @@
],
"title": "Annotations"
},
"logprobs": {
"anyOf": [
{
"items": {
"$ref": "#/components/schemas/OpenAIChatCompletionLogprobsContent"
},
"type": "array"
},
{
"type": "null"
}
],
"title": "Logprobs"
},
"text": {
"anyOf": [
{

21
litellm/responses/litellm_completion_transformation/handler.py
View file

@ -33,15 +33,6 @@ from litellm.types.llms.openai import (
from litellm.types.utils import ModelResponse
_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(dict[str, object])
_RESPONSES_FIELDS_THE_BRIDGE_TRANSLATES: Final = frozenset({"top_logprobs"})
def _passthrough_kwargs(kwargs: Mapping[str, object]) -> Mapping[str, object]:
return MappingProxyType(
{key: value for key, value in kwargs.items() if key not in _RESPONSES_FIELDS_THE_BRIDGE_TRANSLATES}
)
_BRIDGE_TRANSFORM_KEYWORDS: Final = frozenset({"caller_input", "names_non_string_fields"})
@ -147,11 +138,7 @@ class LiteLLMCompletionTransformationHandler:
)
litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = litellm.completion(
**{
**_passthrough_kwargs(_REQUEST_KWARGS_ADAPTER.validate_python(kwargs)),
**litellm_completion_request,
"_skip_responses_api_bridge": True,
},
**{**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True},
)
if isinstance(litellm_completion_response, ModelResponse):
@ -250,11 +237,7 @@ class LiteLLMCompletionTransformationHandler:
)
litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = await litellm.acompletion(
**{
**_passthrough_kwargs(_REQUEST_KWARGS_ADAPTER.validate_python(kwargs)),
**litellm_completion_request,
"_skip_responses_api_bridge": True,
},
**{**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True},
)
if isinstance(litellm_completion_response, ModelResponse):
responses_api_response: Final[ResponsesAPIResponse] = (

50
litellm/responses/litellm_completion_transformation/streaming_iterator.py
View file

@ -26,7 +26,6 @@ from litellm.responses.litellm_completion_transformation.tool_search import (
)
from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
output_text_logprobs,
)
from litellm.responses.streaming_iterator import (
SEQUENCE_NUMBER_STAMPED_ON_EMISSION,
@ -67,12 +66,7 @@ from litellm.types.llms.openai import (
WebSearchCallInProgressEvent,
WebSearchCallSearchingEvent,
)
from litellm.types.responses.main import (
GenericResponseOutputItem,
GenericResponseOutputItemContentAnnotation,
OpenAIChatCompletionLogprobsContent,
OutputText,
)
from litellm.types.responses.main import GenericResponseOutputItemContentAnnotation
from litellm.types.utils import Delta as ChatCompletionDelta
from litellm.types.utils import (
ModelResponse,
@ -102,19 +96,6 @@ def _output_items_with_id(items: tuple[Any, ...], item_type: str, item_id: str |
)
def _message_with_output_text_logprobs(
message: GenericResponseOutputItem, logprobs: Sequence[OpenAIChatCompletionLogprobsContent]
) -> GenericResponseOutputItem:
return message.model_copy(
update={
"content": [
part.model_copy(update={"logprobs": list(logprobs)}) if isinstance(part, OutputText) else part
for part in message.content
]
}
)
def _tool_call_objects(tool_calls: object) -> tuple[object, ...]:
return tuple(tool_calls) if isinstance(tool_calls, list) else ()
@ -209,7 +190,6 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
self.litellm_model_response: ModelResponse | TextCompletionResponse | None = None
self.completed_response = None
self.final_text: str = ""
self._output_text_logprobs: list[OpenAIChatCompletionLogprobsContent] = [] # mutable-ok: per-delta append
self._cached_item_id: str | None = None
self._message_output_index: int | None = None
self._reasoning_output_index: int | None = None
@ -813,9 +793,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
item_id=self._cached_item_id,
output_index=self._message_index(),
content_index=0,
part=BaseLiteLLMOpenAIResponseObject(
**{"type": "output_text", "text": "", "annotations": [], "logprobs": []}
),
part=BaseLiteLLMOpenAIResponseObject(**{"type": "output_text", "text": "", "annotations": []}),
)
def _queue_message_item_added_events(self) -> None:
@ -996,7 +974,6 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
output_index=self._message_index(),
content_index=0,
text=getattr(litellm_complete_object.choices[0].message, "content", "") or "",
logprobs=list(self._output_text_logprobs),
)
def create_output_content_part_done_event(self, litellm_complete_object: ModelResponse) -> ContentPartDoneEvent:
@ -1015,7 +992,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
type="output_text",
text=text,
annotations=response_annotations,
logprobs=[],
logprobs=None,
)
return ContentPartDoneEvent(
@ -1040,14 +1017,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
)
)
text_parts: Final = (
(
{
"type": "output_text",
"text": text,
"annotations": response_annotations,
"logprobs": list(self._output_text_logprobs),
},
)
({"type": "output_text", "text": text, "annotations": response_annotations},)
if self.sent_content_part_added_event
else ()
)
@ -1498,15 +1468,12 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
if not self.sent_content_part_added_event:
self._queue_message_item_added_events()
self._hold_tool_calls_behind_text(getattr(chunk.choices[0].delta, "tool_calls", None))
delta_logprobs: Final = output_text_logprobs(chunk.choices[0].logprobs)
self._output_text_logprobs.extend(delta_logprobs)
return OutputTextDeltaEvent(
type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA,
item_id=item_id,
output_index=self._message_index(),
content_index=0,
delta=delta_content,
logprobs=delta_logprobs,
)
# Priority 3: Handle tool call deltas (if any) -> queue events and emit them
@ -1547,15 +1514,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
"message",
self._cached_item_id,
)
message_index: Final = _index_of_output_item_type(message_aligned, "message")
logprobs_aligned: Final = tuple(
_message_with_output_text_logprobs(item, self._output_text_logprobs)
if index == message_index and isinstance(item, GenericResponseOutputItem)
else item
for index, item in enumerate(message_aligned)
)
reasoning_aligned: Final = _output_items_with_id(
logprobs_aligned,
message_aligned,
"reasoning",
self._cached_reasoning_item_id,
)

34
litellm/responses/litellm_completion_transformation/transformation.py
View file

@ -126,7 +126,6 @@ from litellm.types.responses.main import (
CustomToolCallOutputItem,
GenericResponseOutputItem,
GenericResponseOutputItemContentAnnotation,
OpenAIChatCompletionLogprobsContent,
OutputCodeInterpreterCall,
OutputFunctionToolCall,
OutputImageGenerationCall,
@ -136,7 +135,6 @@ from litellm.types.responses.main import (
from litellm.types.utils import (
ChatCompletionAnnotation,
ChatCompletionMessageToolCall,
ChoiceLogprobs,
Choices,
Function,
Message,
@ -236,27 +234,6 @@ def _echoable_request_params(request: Mapping[str, object]) -> _EchoedResponsesR
return _ECHOED_PARAMS_ADAPTER.validate_python(echoable)
_OUTPUT_TEXT_LOGPROBS_INCLUDE: Final = "message.output_text.logprobs"
_OUTPUT_TEXT_LOGPROB_ADAPTER: Final = TypeAdapter(OpenAIChatCompletionLogprobsContent)
def _chat_logprob_params(request: ResponsesAPIOptionalRequestParams) -> Mapping[str, object]:
if "include" not in request or _OUTPUT_TEXT_LOGPROBS_INCLUDE not in (request["include"] or ()):
return {}
if "top_logprobs" not in request:
return {"logprobs": True}
return {"logprobs": True, "top_logprobs": request["top_logprobs"]}
def output_text_logprobs(chat_logprobs: object) -> Sequence[OpenAIChatCompletionLogprobsContent]:
if not isinstance(chat_logprobs, ChoiceLogprobs):
return []
return [
_OUTPUT_TEXT_LOGPROB_ADAPTER.validate_python(token_logprob.model_dump(exclude_none=True))
for token_logprob in chat_logprobs.content or ()
]
@dataclass(frozen=True, slots=True)
class ResponsesToolChatForm:
chat_tools: tuple[ChatToolParam, ...]
@ -1320,7 +1297,6 @@ class LiteLLMCompletionResponsesConfig:
"metadata": kwargs.get("metadata"),
"service_tier": responses_api_request.get("service_tier"),
"safety_identifier": responses_api_request.get("safety_identifier"),
**_chat_logprob_params(responses_api_request),
"web_search_options": web_search_options,
"response_format": response_format,
"reasoning_effort": reasoning.effort,
@ -3719,9 +3695,7 @@ class LiteLLMCompletionResponsesConfig:
choice=choice,
)
message_output_items.extend(image_generation_items)
elif content := LiteLLMCompletionResponsesConfig._message_output_content(
choice.message, getattr(choice, "logprobs", None)
):
elif content := LiteLLMCompletionResponsesConfig._message_output_content(choice.message):
message_output_items.append(
GenericResponseOutputItem(
type="message",
@ -3734,7 +3708,7 @@ class LiteLLMCompletionResponsesConfig:
return message_output_items
@staticmethod
def _message_output_content(message: Message, logprobs: object) -> tuple[OutputText | ResponseOutputRefusal, ...]:
def _message_output_content(message: Message) -> tuple[OutputText | ResponseOutputRefusal, ...]:
refusal: Final = openai_chat_refusal_text(message)
refusal_parts: Final = (ResponseOutputRefusal(type="refusal", refusal=refusal),) if refusal is not None else ()
has_text_part: Final = (
@ -3743,7 +3717,7 @@ class LiteLLMCompletionResponsesConfig:
or (message.content is not None and not refusal_parts)
)
text_parts: Final = (
(LiteLLMCompletionResponsesConfig._transform_chat_message_to_response_output_text(message, logprobs),)
(LiteLLMCompletionResponsesConfig._transform_chat_message_to_response_output_text(message),)
if has_text_part
else ()
)
@ -3795,7 +3769,6 @@ class LiteLLMCompletionResponsesConfig:
@staticmethod
def _transform_chat_message_to_response_output_text(
message: Message,
logprobs: object,
) -> OutputText:
annotations: Final = getattr(message, "annotations", None)
transformed_annotations: Final = (
@ -3808,7 +3781,6 @@ class LiteLLMCompletionResponsesConfig:
type="output_text",
text=message.content or "",
annotations=transformed_annotations,
logprobs=output_text_logprobs(logprobs),
)
@staticmethod

16
litellm/types/llms/openai.py
View file

@ -91,7 +91,6 @@ from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject, LiteLLMBase
from litellm.types.responses.main import (
CustomToolCallOutputItem,
GenericResponseOutputItem,
OpenAIChatCompletionLogprobsContent,
OutputCodeInterpreterCall,
OutputFunctionToolCall,
OutputImageGenerationCall,
@ -1863,6 +1862,19 @@ class OutputItemDoneEvent(BaseResponsesStreamEvent):
item: BaseLiteLLMOpenAIResponseObject
class OpenAIChatCompletionLogprobsContentTopLogprobs(TypedDict, total=False):
bytes: list
logprob: Required[float]
token: Required[str]
class OpenAIChatCompletionLogprobsContent(TypedDict, total=False):
bytes: list
logprob: Required[float]
token: Required[str]
top_logprobs: list[OpenAIChatCompletionLogprobsContentTopLogprobs]
class ContentPartAddedEvent(BaseResponsesStreamEvent):
type: Literal[ResponsesAPIStreamEvents.CONTENT_PART_ADDED]
item_id: str
@ -1905,7 +1917,6 @@ class OutputTextDeltaEvent(BaseResponsesStreamEvent):
output_index: int
content_index: int
delta: str
logprobs: Sequence[OpenAIChatCompletionLogprobsContent] | None = None
class OutputTextAnnotationAddedEvent(BaseResponsesStreamEvent):
@ -1923,7 +1934,6 @@ class OutputTextDoneEvent(BaseResponsesStreamEvent):
output_index: int
content_index: int
text: str
logprobs: Sequence[OpenAIChatCompletionLogprobsContent] | None = None
class RefusalDeltaEvent(BaseResponsesStreamEvent):

16
litellm/types/responses/main.py
View file

@ -6,7 +6,7 @@ from openai.types.responses.response_function_tool_call import ResponseFunctionT
from openai.types.responses.response_function_web_search import ActionSearchSource, ResponseFunctionWebSearch
from openai.types.responses.response_output_refusal import ResponseOutputRefusal
from pydantic import ConfigDict, PrivateAttr
from typing_extensions import Required, TypedDict
from typing_extensions import TypedDict
from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject
@ -23,26 +23,12 @@ class GenericResponseOutputItemContentAnnotation(BaseLiteLLMOpenAIResponseObject
title: str | None
class OpenAIChatCompletionLogprobsContentTopLogprobs(TypedDict, total=False):
bytes: list
logprob: Required[float]
token: Required[str]
class OpenAIChatCompletionLogprobsContent(TypedDict, total=False):
bytes: list
logprob: Required[float]
token: Required[str]
top_logprobs: list[OpenAIChatCompletionLogprobsContentTopLogprobs]
class OutputText(BaseLiteLLMOpenAIResponseObject):
"""Text output content from an assistant message"""
type: str | None # "output_text"
text: str | None
annotations: list[GenericResponseOutputItemContentAnnotation] | None
logprobs: Sequence[OpenAIChatCompletionLogprobsContent] | None = None
class OutputFunctionToolCall(BaseLiteLLMOpenAIResponseObject):

112
tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py
View file

@ -5,10 +5,8 @@ from copy import deepcopy
from types import MappingProxyType
from typing import Final, Literal, cast
import httpx
import pytest
from openai import OpenAI
from openai.types.responses import ResponseFunctionToolCall, ResponseIncludable
from openai.types.responses import ResponseFunctionToolCall
from openai.types.responses.response_create_params import ResponseInputParam
from openai.types.responses.response_function_web_search import (
ActionFind,
@ -36,7 +34,7 @@ from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
)
from litellm.types.llms.base import HiddenParams
from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams, ResponsesAPIResponse
from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams
from litellm.types.responses.main import OutputToolSearchCall, build_web_search_call
from litellm.types.utils import (
ChatCompletionMessageToolCall,
@ -48,7 +46,6 @@ from litellm.types.utils import (
PromptTokensDetailsWrapper,
Usage,
)
from tests.capturing_transport import CapturingTransport
class TestLiteLLMCompletionResponsesConfig:
@ -7397,108 +7394,3 @@ def test_a_malformed_item_after_a_codex_additional_tools_item_is_named_by_its_in
)
assert f"'{field_path}'" in raised.value.message
_NIM_CHAT_MODEL: Final = "nvidia_nim/meta/llama-3.3-70b-instruct"
_CHAT_TOKEN_LOGPROBS: Final = (
{
"token": "Hi",
"logprob": -0.01,
"bytes": [72, 105],
"top_logprobs": [
{"token": "Hi", "logprob": -0.01, "bytes": [72, 105]},
{"token": "Hey", "logprob": -4.6, "bytes": [72, 101, 121]},
],
},
{
"token": "!",
"logprob": -0.2,
"bytes": [33],
"top_logprobs": [
{"token": "!", "logprob": -0.2, "bytes": [33]},
{"token": ".", "logprob": -1.7, "bytes": [46]},
],
},
)
def _bridged_nim_call(
include: Sequence[ResponseIncludable],
top_logprobs: int | None,
chat_logprobs: Mapping[str, object] | None = None,
) -> tuple[Mapping[str, object], ResponsesAPIResponse]:
transport: Final = CapturingTransport(
ModelResponse(
id="chatcmpl-logprobs",
created=1717000000,
model="meta/llama-3.3-70b-instruct",
object="chat.completion",
choices=[
Choices(
index=0,
finish_reason="stop",
message=Message(role="assistant", content="Hi!"),
logprobs=chat_logprobs,
)
],
usage=Usage(prompt_tokens=8, completion_tokens=2, total_tokens=10),
)
)
response: Final = litellm.responses(
model=_NIM_CHAT_MODEL,
input="Say hi",
include=list(include) or None,
top_logprobs=top_logprobs,
api_key="nvapi-unused",
client=OpenAI(api_key="nvapi-unused", http_client=httpx.Client(transport=transport)),
)
assert isinstance(response, ResponsesAPIResponse)
return transport.request_bodies[-1], response
@pytest.mark.parametrize(
("include", "top_logprobs", "chat_logprob_params"),
[
pytest.param(("message.output_text.logprobs",), 2, {"logprobs": True, "top_logprobs": 2}, id="include-and-top"),
pytest.param(("message.output_text.logprobs",), None, {"logprobs": True}, id="include-only"),
pytest.param((), 2, {}, id="top-logprobs-only"),
],
)
def test_bridge_asks_chat_for_logprobs_only_when_include_requests_output_text_logprobs(
include: Sequence[ResponseIncludable], top_logprobs: int | None, chat_logprob_params: Mapping[str, object]
):
chat_request, _ = _bridged_nim_call(include, top_logprobs)
assert {
key: chat_request[key] for key in ("logprobs", "top_logprobs") if key in chat_request
} == chat_logprob_params
def test_bridge_rejects_output_text_logprobs_for_a_provider_without_logprobs_with_a_400():
with pytest.raises(litellm.UnsupportedParamsError) as exc_info:
litellm.responses(
model="anthropic/claude-haiku-4-5",
input="Say hi",
include=["message.output_text.logprobs"],
api_key="sk-unused",
)
assert exc_info.value.status_code == 400
assert "logprobs" in str(exc_info.value)
@pytest.mark.parametrize(
("chat_logprobs", "output_text_logprobs"),
[
pytest.param({"content": list(_CHAT_TOKEN_LOGPROBS)}, list(_CHAT_TOKEN_LOGPROBS), id="chat-logprobs"),
pytest.param(None, [], id="no-chat-logprobs"),
],
)
def test_bridged_output_text_carries_the_chat_token_logprobs(
chat_logprobs: Mapping[str, object] | None, output_text_logprobs: Sequence[Mapping[str, object]]
):
_, response = _bridged_nim_call(("message.output_text.logprobs",), 2, chat_logprobs)
message: Final = response.model_dump(mode="json")["output"][0]
assert message["type"] == "message"
assert message["content"][0]["logprobs"] == output_text_logprobs

102
tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py
View file

@ -12,7 +12,7 @@ spend tracking stores, so a follow-up previous_response_id still finds the conve
import itertools
import json
from collections.abc import Callable, Iterable, Mapping, Sequence
from collections.abc import Callable, Iterable
from itertools import accumulate, chain, islice, product
from typing import Final
from unittest.mock import AsyncMock, MagicMock
@ -2078,103 +2078,3 @@ async def test_streamed_refusal_deltas_become_one_refusal_part_of_the_message(sy
assert [event.refusal for event in refusal_done] == [expected_content[0]["refusal"]]
assert [item.model_dump()["content"] for item in message_done] == [expected_content]
assert [item.model_dump()["content"] for item in finished.response.output] == [expected_content]
_HI_LOGPROB: Final = {
"token": "Hi",
"logprob": -0.01,
"bytes": [72, 105],
"top_logprobs": [{"token": "Hi", "logprob": -0.01, "bytes": [72, 105]}],
}
_BANG_LOGPROB: Final = {
"token": "!",
"logprob": -0.2,
"bytes": [33],
"top_logprobs": [{"token": "!", "logprob": -0.2, "bytes": [33]}],
}
_THINK_LOGPROB: Final = {"token": "hmm", "logprob": -1.3, "bytes": [104, 109, 109], "top_logprobs": []}
def _logprobs_chunk(
delta: Delta, token_logprobs: Sequence[Mapping[str, object]], finish_reason: str | None = None
) -> ModelResponseStream:
return ModelResponseStream(
id=CHAT_COMPLETION_ID,
created=1748575031,
model="claude-haiku-4-5",
object="chat.completion.chunk",
choices=[
StreamingChoices(
index=0,
delta=delta,
finish_reason=finish_reason,
logprobs={"content": list(token_logprobs)},
)
],
)
async def _wire_events(chunks: Sequence[ModelResponseStream], sync_mode: bool) -> tuple[Mapping[str, object], ...]:
events: Final = await _collect_events(_build_iterator(chunks), sync_mode)
return tuple(event.model_dump(mode="json", exclude_none=True) for event in events)
def _events_of_type(events: Sequence[Mapping[str, object]], event_type: str) -> tuple[Mapping[str, object], ...]:
return tuple(event for event in events if event["type"] == event_type)
def _streamed_full_logprobs(events: Sequence[Mapping[str, object]]) -> Mapping[str, object]:
(text_done,) = _events_of_type(events, "response.output_text.done")
(message_done,) = [
event for event in _events_of_type(events, "response.output_item.done") if event["item"]["type"] == "message"
]
(completed,) = _events_of_type(events, "response.completed")
(completed_message,) = [item for item in completed["response"]["output"] if item["type"] == "message"]
return {
"response.output_text.done": text_done["logprobs"],
"response.output_item.done": message_done["item"]["content"][0]["logprobs"],
"response.completed": completed_message["content"][0]["logprobs"],
}
def _content_part_logprobs(events: Sequence[Mapping[str, object]]) -> tuple[object, ...]:
return tuple(
event["part"]["logprobs"]
for event in events
if event["type"] in ("response.content_part.added", "response.content_part.done")
)
@pytest.mark.parametrize("sync_mode", [True, False])
@pytest.mark.asyncio
async def test_streamed_output_text_carries_each_deltas_logprobs_and_the_full_list_when_done(sync_mode: bool):
events: Final = await _wire_events(
(
_logprobs_chunk(Delta(role="assistant", reasoning_content="hmm"), [_THINK_LOGPROB]),
_logprobs_chunk(Delta(role="assistant", content="Hi"), [_HI_LOGPROB]),
_logprobs_chunk(Delta(role="assistant", content="!"), [_BANG_LOGPROB], finish_reason="stop"),
),
sync_mode,
)
assert [event["logprobs"] for event in _events_of_type(events, "response.output_text.delta")] == [
[_HI_LOGPROB],
[_BANG_LOGPROB],
]
assert _streamed_full_logprobs(events) == dict.fromkeys(
("response.output_text.done", "response.output_item.done", "response.completed"),
[_HI_LOGPROB, _BANG_LOGPROB],
)
assert _content_part_logprobs(events) == ([], [])
@pytest.mark.parametrize("sync_mode", [True, False])
@pytest.mark.asyncio
async def test_streamed_output_text_without_chat_logprobs_carries_empty_logprobs(sync_mode: bool):
events: Final = await _wire_events((_chunk("Hel"), _chunk("lo", finish_reason="stop")), sync_mode)
assert [event["logprobs"] for event in _events_of_type(events, "response.output_text.delta")] == [[], []]
assert _streamed_full_logprobs(events) == dict.fromkeys(
("response.output_text.done", "response.output_item.done", "response.completed"), []
)
assert _content_part_logprobs(events) == ([], [])

26
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -40216,30 +40216,6 @@ export interface components {
*/
type: "box";
};
/** OpenAIChatCompletionLogprobsContent */
OpenAIChatCompletionLogprobsContent: {
/** Bytes */
bytes?: unknown[];
/** Logprob */
logprob: number;
/** Token */
token: string;
/** Top Logprobs */
top_logprobs?: components["schemas"]["OpenAIChatCompletionLogprobsContentTopLogprobs"][];
} & {
[key: string]: unknown;
};
/** OpenAIChatCompletionLogprobsContentTopLogprobs */
OpenAIChatCompletionLogprobsContentTopLogprobs: {
/** Bytes */
bytes?: unknown[];
/** Logprob */
logprob: number;
/** Token */
token: string;
} & {
[key: string]: unknown;
};
/** OpenAIChoiceAnswer */
OpenAIChoiceAnswer: {
/** Choice */
@ -40773,8 +40749,6 @@ export interface components {
OutputText: {
/** Annotations */
annotations: components["schemas"]["GenericResponseOutputItemContentAnnotation"][] | null;
/** Logprobs */
logprobs?: components["schemas"]["OpenAIChatCompletionLogprobsContent"][] | null;
/** Text */
text: string | null;
/** Type */