diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 5329eb508c4..5a80583b207 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -26124,61 +26124,6 @@ "title": "NoulAnswer", "type": "object" }, - "OpenAIChatCompletionLogprobsContent": { - "additionalProperties": true, - "properties": { - "bytes": { - "items": {}, - "title": "Bytes", - "type": "array" - }, - "logprob": { - "title": "Logprob", - "type": "number" - }, - "token": { - "title": "Token", - "type": "string" - }, - "top_logprobs": { - "items": { - "$ref": "#/components/schemas/OpenAIChatCompletionLogprobsContentTopLogprobs" - }, - "title": "Top Logprobs", - "type": "array" - } - }, - "required": [ - "logprob", - "token" - ], - "title": "OpenAIChatCompletionLogprobsContent", - "type": "object" - }, - "OpenAIChatCompletionLogprobsContentTopLogprobs": { - "additionalProperties": true, - "properties": { - "bytes": { - "items": {}, - "title": "Bytes", - "type": "array" - }, - "logprob": { - "title": "Logprob", - "type": "number" - }, - "token": { - "title": "Token", - "type": "string" - } - }, - "required": [ - "logprob", - "token" - ], - "title": "OpenAIChatCompletionLogprobsContentTopLogprobs", - "type": "object" - }, "OperationCreateFile": { "additionalProperties": true, "description": "Instruction describing how to create a file via the apply_patch tool.", @@ -26623,20 +26568,6 @@ ], "title": "Annotations" }, - "logprobs": { - "anyOf": [ - { - "items": { - "$ref": "#/components/schemas/OpenAIChatCompletionLogprobsContent" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Logprobs" - }, "text": { "anyOf": [ { diff --git a/litellm/responses/litellm_completion_transformation/handler.py b/litellm/responses/litellm_completion_transformation/handler.py index 2bcef9d1df3..802ed6bf9c5 100644 --- a/litellm/responses/litellm_completion_transformation/handler.py +++ b/litellm/responses/litellm_completion_transformation/handler.py @@ -33,15 +33,6 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ModelResponse _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(dict[str, object]) -_RESPONSES_FIELDS_THE_BRIDGE_TRANSLATES: Final = frozenset({"top_logprobs"}) - - -def _passthrough_kwargs(kwargs: Mapping[str, object]) -> Mapping[str, object]: - return MappingProxyType( - {key: value for key, value in kwargs.items() if key not in _RESPONSES_FIELDS_THE_BRIDGE_TRANSLATES} - ) - - _BRIDGE_TRANSFORM_KEYWORDS: Final = frozenset({"caller_input", "names_non_string_fields"}) @@ -147,11 +138,7 @@ class LiteLLMCompletionTransformationHandler: ) litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = litellm.completion( - **{ - **_passthrough_kwargs(_REQUEST_KWARGS_ADAPTER.validate_python(kwargs)), - **litellm_completion_request, - "_skip_responses_api_bridge": True, - }, + **{**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True}, ) if isinstance(litellm_completion_response, ModelResponse): @@ -250,11 +237,7 @@ class LiteLLMCompletionTransformationHandler: ) litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = await litellm.acompletion( - **{ - **_passthrough_kwargs(_REQUEST_KWARGS_ADAPTER.validate_python(kwargs)), - **litellm_completion_request, - "_skip_responses_api_bridge": True, - }, + **{**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True}, ) if isinstance(litellm_completion_response, ModelResponse): responses_api_response: Final[ResponsesAPIResponse] = ( diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index f6f05ac8f21..521a698b276 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -26,7 +26,6 @@ from litellm.responses.litellm_completion_transformation.tool_search import ( ) from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, - output_text_logprobs, ) from litellm.responses.streaming_iterator import ( SEQUENCE_NUMBER_STAMPED_ON_EMISSION, @@ -67,12 +66,7 @@ from litellm.types.llms.openai import ( WebSearchCallInProgressEvent, WebSearchCallSearchingEvent, ) -from litellm.types.responses.main import ( - GenericResponseOutputItem, - GenericResponseOutputItemContentAnnotation, - OpenAIChatCompletionLogprobsContent, - OutputText, -) +from litellm.types.responses.main import GenericResponseOutputItemContentAnnotation from litellm.types.utils import Delta as ChatCompletionDelta from litellm.types.utils import ( ModelResponse, @@ -102,19 +96,6 @@ def _output_items_with_id(items: tuple[Any, ...], item_type: str, item_id: str | ) -def _message_with_output_text_logprobs( - message: GenericResponseOutputItem, logprobs: Sequence[OpenAIChatCompletionLogprobsContent] -) -> GenericResponseOutputItem: - return message.model_copy( - update={ - "content": [ - part.model_copy(update={"logprobs": list(logprobs)}) if isinstance(part, OutputText) else part - for part in message.content - ] - } - ) - - def _tool_call_objects(tool_calls: object) -> tuple[object, ...]: return tuple(tool_calls) if isinstance(tool_calls, list) else () @@ -209,7 +190,6 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self.litellm_model_response: ModelResponse | TextCompletionResponse | None = None self.completed_response = None self.final_text: str = "" - self._output_text_logprobs: list[OpenAIChatCompletionLogprobsContent] = [] # mutable-ok: per-delta append self._cached_item_id: str | None = None self._message_output_index: int | None = None self._reasoning_output_index: int | None = None @@ -813,9 +793,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): item_id=self._cached_item_id, output_index=self._message_index(), content_index=0, - part=BaseLiteLLMOpenAIResponseObject( - **{"type": "output_text", "text": "", "annotations": [], "logprobs": []} - ), + part=BaseLiteLLMOpenAIResponseObject(**{"type": "output_text", "text": "", "annotations": []}), ) def _queue_message_item_added_events(self) -> None: @@ -996,7 +974,6 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): output_index=self._message_index(), content_index=0, text=getattr(litellm_complete_object.choices[0].message, "content", "") or "", - logprobs=list(self._output_text_logprobs), ) def create_output_content_part_done_event(self, litellm_complete_object: ModelResponse) -> ContentPartDoneEvent: @@ -1015,7 +992,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): type="output_text", text=text, annotations=response_annotations, - logprobs=[], + logprobs=None, ) return ContentPartDoneEvent( @@ -1040,14 +1017,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): ) ) text_parts: Final = ( - ( - { - "type": "output_text", - "text": text, - "annotations": response_annotations, - "logprobs": list(self._output_text_logprobs), - }, - ) + ({"type": "output_text", "text": text, "annotations": response_annotations},) if self.sent_content_part_added_event else () ) @@ -1498,15 +1468,12 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if not self.sent_content_part_added_event: self._queue_message_item_added_events() self._hold_tool_calls_behind_text(getattr(chunk.choices[0].delta, "tool_calls", None)) - delta_logprobs: Final = output_text_logprobs(chunk.choices[0].logprobs) - self._output_text_logprobs.extend(delta_logprobs) return OutputTextDeltaEvent( type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, item_id=item_id, output_index=self._message_index(), content_index=0, delta=delta_content, - logprobs=delta_logprobs, ) # Priority 3: Handle tool call deltas (if any) -> queue events and emit them @@ -1547,15 +1514,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): "message", self._cached_item_id, ) - message_index: Final = _index_of_output_item_type(message_aligned, "message") - logprobs_aligned: Final = tuple( - _message_with_output_text_logprobs(item, self._output_text_logprobs) - if index == message_index and isinstance(item, GenericResponseOutputItem) - else item - for index, item in enumerate(message_aligned) - ) reasoning_aligned: Final = _output_items_with_id( - logprobs_aligned, + message_aligned, "reasoning", self._cached_reasoning_item_id, ) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index d1d7a523a35..0f4ed0611be 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -126,7 +126,6 @@ from litellm.types.responses.main import ( CustomToolCallOutputItem, GenericResponseOutputItem, GenericResponseOutputItemContentAnnotation, - OpenAIChatCompletionLogprobsContent, OutputCodeInterpreterCall, OutputFunctionToolCall, OutputImageGenerationCall, @@ -136,7 +135,6 @@ from litellm.types.responses.main import ( from litellm.types.utils import ( ChatCompletionAnnotation, ChatCompletionMessageToolCall, - ChoiceLogprobs, Choices, Function, Message, @@ -236,27 +234,6 @@ def _echoable_request_params(request: Mapping[str, object]) -> _EchoedResponsesR return _ECHOED_PARAMS_ADAPTER.validate_python(echoable) -_OUTPUT_TEXT_LOGPROBS_INCLUDE: Final = "message.output_text.logprobs" -_OUTPUT_TEXT_LOGPROB_ADAPTER: Final = TypeAdapter(OpenAIChatCompletionLogprobsContent) - - -def _chat_logprob_params(request: ResponsesAPIOptionalRequestParams) -> Mapping[str, object]: - if "include" not in request or _OUTPUT_TEXT_LOGPROBS_INCLUDE not in (request["include"] or ()): - return {} - if "top_logprobs" not in request: - return {"logprobs": True} - return {"logprobs": True, "top_logprobs": request["top_logprobs"]} - - -def output_text_logprobs(chat_logprobs: object) -> Sequence[OpenAIChatCompletionLogprobsContent]: - if not isinstance(chat_logprobs, ChoiceLogprobs): - return [] - return [ - _OUTPUT_TEXT_LOGPROB_ADAPTER.validate_python(token_logprob.model_dump(exclude_none=True)) - for token_logprob in chat_logprobs.content or () - ] - - @dataclass(frozen=True, slots=True) class ResponsesToolChatForm: chat_tools: tuple[ChatToolParam, ...] @@ -1320,7 +1297,6 @@ class LiteLLMCompletionResponsesConfig: "metadata": kwargs.get("metadata"), "service_tier": responses_api_request.get("service_tier"), "safety_identifier": responses_api_request.get("safety_identifier"), - **_chat_logprob_params(responses_api_request), "web_search_options": web_search_options, "response_format": response_format, "reasoning_effort": reasoning.effort, @@ -3719,9 +3695,7 @@ class LiteLLMCompletionResponsesConfig: choice=choice, ) message_output_items.extend(image_generation_items) - elif content := LiteLLMCompletionResponsesConfig._message_output_content( - choice.message, getattr(choice, "logprobs", None) - ): + elif content := LiteLLMCompletionResponsesConfig._message_output_content(choice.message): message_output_items.append( GenericResponseOutputItem( type="message", @@ -3734,7 +3708,7 @@ class LiteLLMCompletionResponsesConfig: return message_output_items @staticmethod - def _message_output_content(message: Message, logprobs: object) -> tuple[OutputText | ResponseOutputRefusal, ...]: + def _message_output_content(message: Message) -> tuple[OutputText | ResponseOutputRefusal, ...]: refusal: Final = openai_chat_refusal_text(message) refusal_parts: Final = (ResponseOutputRefusal(type="refusal", refusal=refusal),) if refusal is not None else () has_text_part: Final = ( @@ -3743,7 +3717,7 @@ class LiteLLMCompletionResponsesConfig: or (message.content is not None and not refusal_parts) ) text_parts: Final = ( - (LiteLLMCompletionResponsesConfig._transform_chat_message_to_response_output_text(message, logprobs),) + (LiteLLMCompletionResponsesConfig._transform_chat_message_to_response_output_text(message),) if has_text_part else () ) @@ -3795,7 +3769,6 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def _transform_chat_message_to_response_output_text( message: Message, - logprobs: object, ) -> OutputText: annotations: Final = getattr(message, "annotations", None) transformed_annotations: Final = ( @@ -3808,7 +3781,6 @@ class LiteLLMCompletionResponsesConfig: type="output_text", text=message.content or "", annotations=transformed_annotations, - logprobs=output_text_logprobs(logprobs), ) @staticmethod diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index dedeac61e18..43d71a6c0b4 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -91,7 +91,6 @@ from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject, LiteLLMBase from litellm.types.responses.main import ( CustomToolCallOutputItem, GenericResponseOutputItem, - OpenAIChatCompletionLogprobsContent, OutputCodeInterpreterCall, OutputFunctionToolCall, OutputImageGenerationCall, @@ -1863,6 +1862,19 @@ class OutputItemDoneEvent(BaseResponsesStreamEvent): item: BaseLiteLLMOpenAIResponseObject +class OpenAIChatCompletionLogprobsContentTopLogprobs(TypedDict, total=False): + bytes: list + logprob: Required[float] + token: Required[str] + + +class OpenAIChatCompletionLogprobsContent(TypedDict, total=False): + bytes: list + logprob: Required[float] + token: Required[str] + top_logprobs: list[OpenAIChatCompletionLogprobsContentTopLogprobs] + + class ContentPartAddedEvent(BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.CONTENT_PART_ADDED] item_id: str @@ -1905,7 +1917,6 @@ class OutputTextDeltaEvent(BaseResponsesStreamEvent): output_index: int content_index: int delta: str - logprobs: Sequence[OpenAIChatCompletionLogprobsContent] | None = None class OutputTextAnnotationAddedEvent(BaseResponsesStreamEvent): @@ -1923,7 +1934,6 @@ class OutputTextDoneEvent(BaseResponsesStreamEvent): output_index: int content_index: int text: str - logprobs: Sequence[OpenAIChatCompletionLogprobsContent] | None = None class RefusalDeltaEvent(BaseResponsesStreamEvent): diff --git a/litellm/types/responses/main.py b/litellm/types/responses/main.py index bea00931d45..86dc857cfd0 100644 --- a/litellm/types/responses/main.py +++ b/litellm/types/responses/main.py @@ -6,7 +6,7 @@ from openai.types.responses.response_function_tool_call import ResponseFunctionT from openai.types.responses.response_function_web_search import ActionSearchSource, ResponseFunctionWebSearch from openai.types.responses.response_output_refusal import ResponseOutputRefusal from pydantic import ConfigDict, PrivateAttr -from typing_extensions import Required, TypedDict +from typing_extensions import TypedDict from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject @@ -23,26 +23,12 @@ class GenericResponseOutputItemContentAnnotation(BaseLiteLLMOpenAIResponseObject title: str | None -class OpenAIChatCompletionLogprobsContentTopLogprobs(TypedDict, total=False): - bytes: list - logprob: Required[float] - token: Required[str] - - -class OpenAIChatCompletionLogprobsContent(TypedDict, total=False): - bytes: list - logprob: Required[float] - token: Required[str] - top_logprobs: list[OpenAIChatCompletionLogprobsContentTopLogprobs] - - class OutputText(BaseLiteLLMOpenAIResponseObject): """Text output content from an assistant message""" type: str | None # "output_text" text: str | None annotations: list[GenericResponseOutputItemContentAnnotation] | None - logprobs: Sequence[OpenAIChatCompletionLogprobsContent] | None = None class OutputFunctionToolCall(BaseLiteLLMOpenAIResponseObject): diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 633b7d07932..b4cd606374a 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -5,10 +5,8 @@ from copy import deepcopy from types import MappingProxyType from typing import Final, Literal, cast -import httpx import pytest -from openai import OpenAI -from openai.types.responses import ResponseFunctionToolCall, ResponseIncludable +from openai.types.responses import ResponseFunctionToolCall from openai.types.responses.response_create_params import ResponseInputParam from openai.types.responses.response_function_web_search import ( ActionFind, @@ -36,7 +34,7 @@ from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) from litellm.types.llms.base import HiddenParams -from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams, ResponsesAPIResponse +from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams from litellm.types.responses.main import OutputToolSearchCall, build_web_search_call from litellm.types.utils import ( ChatCompletionMessageToolCall, @@ -48,7 +46,6 @@ from litellm.types.utils import ( PromptTokensDetailsWrapper, Usage, ) -from tests.capturing_transport import CapturingTransport class TestLiteLLMCompletionResponsesConfig: @@ -7397,108 +7394,3 @@ def test_a_malformed_item_after_a_codex_additional_tools_item_is_named_by_its_in ) assert f"'{field_path}'" in raised.value.message - - -_NIM_CHAT_MODEL: Final = "nvidia_nim/meta/llama-3.3-70b-instruct" -_CHAT_TOKEN_LOGPROBS: Final = ( - { - "token": "Hi", - "logprob": -0.01, - "bytes": [72, 105], - "top_logprobs": [ - {"token": "Hi", "logprob": -0.01, "bytes": [72, 105]}, - {"token": "Hey", "logprob": -4.6, "bytes": [72, 101, 121]}, - ], - }, - { - "token": "!", - "logprob": -0.2, - "bytes": [33], - "top_logprobs": [ - {"token": "!", "logprob": -0.2, "bytes": [33]}, - {"token": ".", "logprob": -1.7, "bytes": [46]}, - ], - }, -) - - -def _bridged_nim_call( - include: Sequence[ResponseIncludable], - top_logprobs: int | None, - chat_logprobs: Mapping[str, object] | None = None, -) -> tuple[Mapping[str, object], ResponsesAPIResponse]: - transport: Final = CapturingTransport( - ModelResponse( - id="chatcmpl-logprobs", - created=1717000000, - model="meta/llama-3.3-70b-instruct", - object="chat.completion", - choices=[ - Choices( - index=0, - finish_reason="stop", - message=Message(role="assistant", content="Hi!"), - logprobs=chat_logprobs, - ) - ], - usage=Usage(prompt_tokens=8, completion_tokens=2, total_tokens=10), - ) - ) - response: Final = litellm.responses( - model=_NIM_CHAT_MODEL, - input="Say hi", - include=list(include) or None, - top_logprobs=top_logprobs, - api_key="nvapi-unused", - client=OpenAI(api_key="nvapi-unused", http_client=httpx.Client(transport=transport)), - ) - assert isinstance(response, ResponsesAPIResponse) - return transport.request_bodies[-1], response - - -@pytest.mark.parametrize( - ("include", "top_logprobs", "chat_logprob_params"), - [ - pytest.param(("message.output_text.logprobs",), 2, {"logprobs": True, "top_logprobs": 2}, id="include-and-top"), - pytest.param(("message.output_text.logprobs",), None, {"logprobs": True}, id="include-only"), - pytest.param((), 2, {}, id="top-logprobs-only"), - ], -) -def test_bridge_asks_chat_for_logprobs_only_when_include_requests_output_text_logprobs( - include: Sequence[ResponseIncludable], top_logprobs: int | None, chat_logprob_params: Mapping[str, object] -): - chat_request, _ = _bridged_nim_call(include, top_logprobs) - - assert { - key: chat_request[key] for key in ("logprobs", "top_logprobs") if key in chat_request - } == chat_logprob_params - - -def test_bridge_rejects_output_text_logprobs_for_a_provider_without_logprobs_with_a_400(): - with pytest.raises(litellm.UnsupportedParamsError) as exc_info: - litellm.responses( - model="anthropic/claude-haiku-4-5", - input="Say hi", - include=["message.output_text.logprobs"], - api_key="sk-unused", - ) - - assert exc_info.value.status_code == 400 - assert "logprobs" in str(exc_info.value) - - -@pytest.mark.parametrize( - ("chat_logprobs", "output_text_logprobs"), - [ - pytest.param({"content": list(_CHAT_TOKEN_LOGPROBS)}, list(_CHAT_TOKEN_LOGPROBS), id="chat-logprobs"), - pytest.param(None, [], id="no-chat-logprobs"), - ], -) -def test_bridged_output_text_carries_the_chat_token_logprobs( - chat_logprobs: Mapping[str, object] | None, output_text_logprobs: Sequence[Mapping[str, object]] -): - _, response = _bridged_nim_call(("message.output_text.logprobs",), 2, chat_logprobs) - - message: Final = response.model_dump(mode="json")["output"][0] - assert message["type"] == "message" - assert message["content"][0]["logprobs"] == output_text_logprobs diff --git a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index d5260ebcc5d..87858741908 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -12,7 +12,7 @@ spend tracking stores, so a follow-up previous_response_id still finds the conve import itertools import json -from collections.abc import Callable, Iterable, Mapping, Sequence +from collections.abc import Callable, Iterable from itertools import accumulate, chain, islice, product from typing import Final from unittest.mock import AsyncMock, MagicMock @@ -2078,103 +2078,3 @@ async def test_streamed_refusal_deltas_become_one_refusal_part_of_the_message(sy assert [event.refusal for event in refusal_done] == [expected_content[0]["refusal"]] assert [item.model_dump()["content"] for item in message_done] == [expected_content] assert [item.model_dump()["content"] for item in finished.response.output] == [expected_content] - - -_HI_LOGPROB: Final = { - "token": "Hi", - "logprob": -0.01, - "bytes": [72, 105], - "top_logprobs": [{"token": "Hi", "logprob": -0.01, "bytes": [72, 105]}], -} -_BANG_LOGPROB: Final = { - "token": "!", - "logprob": -0.2, - "bytes": [33], - "top_logprobs": [{"token": "!", "logprob": -0.2, "bytes": [33]}], -} -_THINK_LOGPROB: Final = {"token": "hmm", "logprob": -1.3, "bytes": [104, 109, 109], "top_logprobs": []} - - -def _logprobs_chunk( - delta: Delta, token_logprobs: Sequence[Mapping[str, object]], finish_reason: str | None = None -) -> ModelResponseStream: - return ModelResponseStream( - id=CHAT_COMPLETION_ID, - created=1748575031, - model="claude-haiku-4-5", - object="chat.completion.chunk", - choices=[ - StreamingChoices( - index=0, - delta=delta, - finish_reason=finish_reason, - logprobs={"content": list(token_logprobs)}, - ) - ], - ) - - -async def _wire_events(chunks: Sequence[ModelResponseStream], sync_mode: bool) -> tuple[Mapping[str, object], ...]: - events: Final = await _collect_events(_build_iterator(chunks), sync_mode) - return tuple(event.model_dump(mode="json", exclude_none=True) for event in events) - - -def _events_of_type(events: Sequence[Mapping[str, object]], event_type: str) -> tuple[Mapping[str, object], ...]: - return tuple(event for event in events if event["type"] == event_type) - - -def _streamed_full_logprobs(events: Sequence[Mapping[str, object]]) -> Mapping[str, object]: - (text_done,) = _events_of_type(events, "response.output_text.done") - (message_done,) = [ - event for event in _events_of_type(events, "response.output_item.done") if event["item"]["type"] == "message" - ] - (completed,) = _events_of_type(events, "response.completed") - (completed_message,) = [item for item in completed["response"]["output"] if item["type"] == "message"] - return { - "response.output_text.done": text_done["logprobs"], - "response.output_item.done": message_done["item"]["content"][0]["logprobs"], - "response.completed": completed_message["content"][0]["logprobs"], - } - - -def _content_part_logprobs(events: Sequence[Mapping[str, object]]) -> tuple[object, ...]: - return tuple( - event["part"]["logprobs"] - for event in events - if event["type"] in ("response.content_part.added", "response.content_part.done") - ) - - -@pytest.mark.parametrize("sync_mode", [True, False]) -@pytest.mark.asyncio -async def test_streamed_output_text_carries_each_deltas_logprobs_and_the_full_list_when_done(sync_mode: bool): - events: Final = await _wire_events( - ( - _logprobs_chunk(Delta(role="assistant", reasoning_content="hmm"), [_THINK_LOGPROB]), - _logprobs_chunk(Delta(role="assistant", content="Hi"), [_HI_LOGPROB]), - _logprobs_chunk(Delta(role="assistant", content="!"), [_BANG_LOGPROB], finish_reason="stop"), - ), - sync_mode, - ) - - assert [event["logprobs"] for event in _events_of_type(events, "response.output_text.delta")] == [ - [_HI_LOGPROB], - [_BANG_LOGPROB], - ] - assert _streamed_full_logprobs(events) == dict.fromkeys( - ("response.output_text.done", "response.output_item.done", "response.completed"), - [_HI_LOGPROB, _BANG_LOGPROB], - ) - assert _content_part_logprobs(events) == ([], []) - - -@pytest.mark.parametrize("sync_mode", [True, False]) -@pytest.mark.asyncio -async def test_streamed_output_text_without_chat_logprobs_carries_empty_logprobs(sync_mode: bool): - events: Final = await _wire_events((_chunk("Hel"), _chunk("lo", finish_reason="stop")), sync_mode) - - assert [event["logprobs"] for event in _events_of_type(events, "response.output_text.delta")] == [[], []] - assert _streamed_full_logprobs(events) == dict.fromkeys( - ("response.output_text.done", "response.output_item.done", "response.completed"), [] - ) - assert _content_part_logprobs(events) == ([], []) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index d3e29138a84..96d4c0e1bc4 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -40216,30 +40216,6 @@ export interface components { */ type: "box"; }; - /** OpenAIChatCompletionLogprobsContent */ - OpenAIChatCompletionLogprobsContent: { - /** Bytes */ - bytes?: unknown[]; - /** Logprob */ - logprob: number; - /** Token */ - token: string; - /** Top Logprobs */ - top_logprobs?: components["schemas"]["OpenAIChatCompletionLogprobsContentTopLogprobs"][]; - } & { - [key: string]: unknown; - }; - /** OpenAIChatCompletionLogprobsContentTopLogprobs */ - OpenAIChatCompletionLogprobsContentTopLogprobs: { - /** Bytes */ - bytes?: unknown[]; - /** Logprob */ - logprob: number; - /** Token */ - token: string; - } & { - [key: string]: unknown; - }; /** OpenAIChoiceAnswer */ OpenAIChoiceAnswer: { /** Choice */ @@ -40773,8 +40749,6 @@ export interface components { OutputText: { /** Annotations */ annotations: components["schemas"]["GenericResponseOutputItemContentAnnotation"][] | null; - /** Logprobs */ - logprobs?: components["schemas"]["OpenAIChatCompletionLogprobsContent"][] | null; /** Text */ text: string | null; /** Type */