From aac424df1a6959b3193ca2005cfe7a69182b8124 Mon Sep 17 00:00:00 2001 From: mateo Date: Sat, 31 Oct 2026 23:35:28 +0000 Subject: [PATCH 01/15] fix(responses): carry logprobs across the chat completions bridge A /v1/responses request with top_logprobs on a chat-bridged deployment sent top_logprobs without logprobs, which OpenAI-style chat APIs and Vertex Gemini reject, and the bridge dropped the chat choice's logprobs on the way back. The bridge now sends logprobs with top_logprobs and copies choice.logprobs into output_text.logprobs, on the response and on the output_text.delta, output_text.done and content_part.done stream events Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../logprobs.py | 34 +++++ .../streaming_iterator.py | 26 +++- .../transformation.py | 17 ++- litellm/types/llms/base.py | 32 +++- litellm/types/llms/openai.py | 15 +- litellm/types/responses/main.py | 4 +- tests/integration/proxy_config.yaml | 7 + .../translation/responses/bases/vertex_ai.py | 35 +++++ .../basic/test_responses_basic_vertex_ai.py | 2 + .../test_responses_sampling_vertex_ai.py | 113 ++++++++++++++ .../test_litellm_completion_responses.py | 109 +++++++++++++- .../test_streaming_iterator_transformation.py | 141 ++++++++++-------- 12 files changed, 451 insertions(+), 84 deletions(-) create mode 100644 litellm/responses/litellm_completion_transformation/logprobs.py create mode 100644 tests/integration/translation/responses/sampling/test_responses_sampling_vertex_ai.py diff --git a/litellm/responses/litellm_completion_transformation/logprobs.py b/litellm/responses/litellm_completion_transformation/logprobs.py new file mode 100644 index 00000000000..81953ccd602 --- /dev/null +++ b/litellm/responses/litellm_completion_transformation/logprobs.py @@ -0,0 +1,34 @@ +"""Chat ``choice.logprobs`` as Responses ``output_text.logprobs``, which share one token shape.""" + +from typing import Final + +from pydantic import TypeAdapter, ValidationError + +from litellm.types.llms.base import OutputTextLogprob, OutputTextTopLogprob +from litellm.types.utils import ChatCompletionTokenLogprob, ChoiceLogprobs, TopLogprob + +_CHOICE_LOGPROBS: Final = TypeAdapter(ChoiceLogprobs | None) + + +def _top_logprob(top: TopLogprob) -> OutputTextTopLogprob: + return {"token": top.token, "logprob": top.logprob, "bytes": top.bytes} + + +def _token_logprob(token: ChatCompletionTokenLogprob) -> OutputTextLogprob: + return { + "token": token.token, + "logprob": token.logprob, + "bytes": token.bytes, + "top_logprobs": [_top_logprob(top) for top in token.top_logprobs], + } + + +def output_text_logprobs(choice_logprobs: object) -> list[OutputTextLogprob] | None: + """The ``logprobs`` a Responses text part carries for a chat choice, ``None`` when the choice sent none.""" + try: + validated: Final = _CHOICE_LOGPROBS.validate_python(choice_logprobs) + except ValidationError: + return None + if validated is None or validated.content is None: + return None + return [_token_logprob(token) for token in validated.content] diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index ddcdc66ff20..0f84159c7dc 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -44,6 +44,7 @@ from litellm.types.llms.openai import ( OutputTextAnnotationAddedEvent, OutputTextDeltaEvent, OutputTextDoneEvent, + OutputTextLogprob, ReasoningSummaryPartDoneEvent, ReasoningSummaryTextDeltaEvent, ReasoningSummaryTextDoneEvent, @@ -71,6 +72,8 @@ from litellm.types.utils import ( TextCompletionResponse, ) +from .logprobs import output_text_logprobs + def _index_of_output_item_type(items: Sequence[object], item_type: str) -> int | None: return next( @@ -210,6 +213,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._sent_reasoning_summary_text_done_event: bool = False self._sent_reasoning_summary_part_done_event: bool = False self._reasoning_summary_text: str = "" + self._text_logprobs: tuple[OutputTextLogprob, ...] = () # -- GENERIC RESPONSE-EVENTS PENDING QUEUE as required by fix -- self._pending_response_events: list[BaseLiteLLMOpenAIResponseObject] = [] self._reasoning_active = False @@ -1014,8 +1018,14 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): output_index=self._message_index(), content_index=0, text=getattr(litellm_complete_object.choices[0].message, "content", "") or "", + logprobs=self._final_text_logprobs(litellm_complete_object), ) + def _final_text_logprobs(self, litellm_complete_object: ModelResponse) -> list[OutputTextLogprob] | None: + if self._text_logprobs: + return list(self._text_logprobs) + return output_text_logprobs(getattr(litellm_complete_object.choices[0], "logprobs", None)) + def create_output_content_part_done_event(self, litellm_complete_object: ModelResponse) -> ContentPartDoneEvent: if self._cached_item_id is None: self._cached_item_id = f"msg_{uuid.uuid4()}" @@ -1032,7 +1042,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): type="output_text", text=text, annotations=response_annotations, - logprobs=None, + logprobs=self._final_text_logprobs(litellm_complete_object), ) return ContentPartDoneEvent( @@ -1056,8 +1066,16 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): annotations=annotations ) ) + logprobs: Final = self._final_text_logprobs(litellm_complete_object) text_parts: Final = ( - ({"type": "output_text", "text": text, "annotations": response_annotations},) + ( + { + "type": "output_text", + "text": text, + "annotations": response_annotations, + **({} if logprobs is None else {"logprobs": logprobs}), + }, + ) if self.sent_content_part_added_event else () ) @@ -1505,12 +1523,16 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._queue_message_item_added_events() self._hold_tool_calls_behind_text(getattr(chunk.choices[0].delta, "tool_calls", None)) self._sequence_number += 1 + delta_logprobs: Final = output_text_logprobs(getattr(chunk.choices[0], "logprobs", None)) + if delta_logprobs is not None: + self._text_logprobs = (*self._text_logprobs, *delta_logprobs) text_delta_event: Final = OutputTextDeltaEvent( type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, item_id=item_id, output_index=self._message_index(), content_index=0, delta=delta_content, + logprobs=delta_logprobs, ) text_delta_event.__dict__["sequence_number"] = self._sequence_number return text_delta_event diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 0f4ed0611be..66750d26eaa 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -111,6 +111,7 @@ from litellm.types.llms.openai import ( OpenAIMcpServerTool, OpenAIWebSearchOptions, OpenAIWebSearchUserLocation, + OutputTextLogprob, OutputTokensDetails, Reasoning, ResponseAPIUsage, @@ -167,6 +168,7 @@ from .custom_tools import ( validated_tool_search_execution, without_malformed_tool_flags, ) +from .logprobs import output_text_logprobs from .reasoning_items import decode_thinking_blocks, encode_thinking_blocks, mint_reasoning_item_id from .tool_search import ( CompletionRoute, @@ -950,6 +952,7 @@ class LiteLLMCompletionResponsesConfig: "text", "tool_choice", "tools", + "top_logprobs", "top_p", "user", ] @@ -1289,6 +1292,8 @@ class LiteLLMCompletionResponsesConfig: ), "tools": tools, "top_p": responses_api_request.get("top_p"), + "top_logprobs": responses_api_request.get("top_logprobs"), + "logprobs": True if responses_api_request.get("top_logprobs") is not None else None, "user": responses_api_request.get("user"), "temperature": responses_api_request.get("temperature"), "parallel_tool_calls": responses_api_request.get("parallel_tool_calls"), @@ -3695,7 +3700,9 @@ class LiteLLMCompletionResponsesConfig: choice=choice, ) message_output_items.extend(image_generation_items) - elif content := LiteLLMCompletionResponsesConfig._message_output_content(choice.message): + elif content := LiteLLMCompletionResponsesConfig._message_output_content( + choice.message, output_text_logprobs(getattr(choice, "logprobs", None)) + ): message_output_items.append( GenericResponseOutputItem( type="message", @@ -3708,7 +3715,9 @@ class LiteLLMCompletionResponsesConfig: return message_output_items @staticmethod - def _message_output_content(message: Message) -> tuple[OutputText | ResponseOutputRefusal, ...]: + def _message_output_content( + message: Message, logprobs: list[OutputTextLogprob] | None = None + ) -> tuple[OutputText | ResponseOutputRefusal, ...]: refusal: Final = openai_chat_refusal_text(message) refusal_parts: Final = (ResponseOutputRefusal(type="refusal", refusal=refusal),) if refusal is not None else () has_text_part: Final = ( @@ -3717,7 +3726,7 @@ class LiteLLMCompletionResponsesConfig: or (message.content is not None and not refusal_parts) ) text_parts: Final = ( - (LiteLLMCompletionResponsesConfig._transform_chat_message_to_response_output_text(message),) + (LiteLLMCompletionResponsesConfig._transform_chat_message_to_response_output_text(message, logprobs),) if has_text_part else () ) @@ -3769,6 +3778,7 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def _transform_chat_message_to_response_output_text( message: Message, + logprobs: list[OutputTextLogprob] | None = None, ) -> OutputText: annotations: Final = getattr(message, "annotations", None) transformed_annotations: Final = ( @@ -3781,6 +3791,7 @@ class LiteLLMCompletionResponsesConfig: type="output_text", text=message.content or "", annotations=transformed_annotations, + logprobs=logprobs, ) @staticmethod diff --git a/litellm/types/llms/base.py b/litellm/types/llms/base.py index 5a2da071cec..d82b73705ef 100644 --- a/litellm/types/llms/base.py +++ b/litellm/types/llms/base.py @@ -1,7 +1,9 @@ -from typing import Final +from collections.abc import Mapping +from typing import Final, cast from openai._models import BaseModel as OpenAIObject -from pydantic import ConfigDict +from pydantic import ConfigDict, SerializerFunctionWrapHandler, model_serializer +from typing_extensions import ReadOnly, TypedDict from litellm.types.llms.base_model import LiteLLMBaseModel @@ -83,3 +85,29 @@ class CachedTokensDetails(LiteLLMBaseModel): text_tokens: int | None = None audio_tokens: int | None = None image_tokens: int | None = None + + +class OutputTextTopLogprob(TypedDict): + token: ReadOnly[str] + logprob: ReadOnly[float] + bytes: ReadOnly[list[int] | None] + + +class OutputTextLogprob(TypedDict): + token: ReadOnly[str] + logprob: ReadOnly[float] + bytes: ReadOnly[list[int] | None] + top_logprobs: ReadOnly[list[OutputTextTopLogprob]] + + +class CarriesOutputTextLogprobs(BaseLiteLLMOpenAIResponseObject): + """A Responses text payload carries ``logprobs`` only when the chat choice behind it sent them.""" + + logprobs: list[OutputTextLogprob] | None = None + + @model_serializer(mode="wrap") + def _without_absent_logprobs(self, handler: SerializerFunctionWrapHandler) -> Mapping[str, object]: + dumped: Final = cast("Mapping[str, object]", handler(self)) # cast-ok: handler dumps this model's fields + if self.logprobs is not None: + return dumped + return {key: value for key, value in dumped.items() if key != "logprobs"} diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index d7e55b57a07..c079cba3aba 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -3,7 +3,7 @@ import math from collections.abc import Iterable, Mapping, Sequence from enum import Enum from os import PathLike -from typing import IO, TYPE_CHECKING, Any, Final, Generic, Literal, Optional, TypeAlias, Union +from typing import IO, TYPE_CHECKING, Any, Final, Generic, Literal, Optional, TypeAlias, Union, cast import httpx from openai import Omit @@ -86,7 +86,12 @@ from typing_extensions import ( ) from litellm._logging import verbose_logger -from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject, LiteLLMBaseModel +from litellm.types.llms.base import ( + BaseLiteLLMOpenAIResponseObject, + CarriesOutputTextLogprobs, + LiteLLMBaseModel, + OutputTextLogprob, +) from litellm.types.responses.main import ( CustomToolCallOutputItem, GenericResponseOutputItem, @@ -1863,7 +1868,7 @@ class ContentPartDonePartOutputText(BaseLiteLLMOpenAIResponseObject): type: Literal["output_text"] text: str annotations: list[BaseLiteLLMOpenAIResponseObject] - logprobs: list[OpenAIChatCompletionLogprobsContent] | None = None + logprobs: list[OutputTextLogprob] | None = None class ContentPartDonePartRefusal(BaseLiteLLMOpenAIResponseObject): @@ -1887,7 +1892,7 @@ class ContentPartDoneEvent(BaseLiteLLMOpenAIResponseObject): part: PART_UNION_TYPES -class OutputTextDeltaEvent(BaseLiteLLMOpenAIResponseObject): +class OutputTextDeltaEvent(CarriesOutputTextLogprobs): type: Literal[ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA] item_id: str output_index: int @@ -1904,7 +1909,7 @@ class OutputTextAnnotationAddedEvent(BaseLiteLLMOpenAIResponseObject): annotation: dict -class OutputTextDoneEvent(BaseLiteLLMOpenAIResponseObject): +class OutputTextDoneEvent(CarriesOutputTextLogprobs): type: Literal[ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE] item_id: str output_index: int diff --git a/litellm/types/responses/main.py b/litellm/types/responses/main.py index 86dc857cfd0..a8c8410e723 100644 --- a/litellm/types/responses/main.py +++ b/litellm/types/responses/main.py @@ -8,7 +8,7 @@ from openai.types.responses.response_output_refusal import ResponseOutputRefusal from pydantic import ConfigDict, PrivateAttr from typing_extensions import TypedDict -from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject +from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject, CarriesOutputTextLogprobs Phase = Literal["commentary", "final_answer"] | None @@ -23,7 +23,7 @@ class GenericResponseOutputItemContentAnnotation(BaseLiteLLMOpenAIResponseObject title: str | None -class OutputText(BaseLiteLLMOpenAIResponseObject): +class OutputText(CarriesOutputTextLogprobs): """Text output content from an assistant message""" type: str | None # "output_text" diff --git a/tests/integration/proxy_config.yaml b/tests/integration/proxy_config.yaml index 4e2cab56d27..3047e53f61b 100644 --- a/tests/integration/proxy_config.yaml +++ b/tests/integration/proxy_config.yaml @@ -501,6 +501,13 @@ model_list: vertex_project: synthetic-vertex-project vertex_location: global vertex_credentials: '{"type": "external_account", "audience": "synthetic-vertex-audience", "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", "token_url": "http://127.0.0.1:8190/_oauth/token", "credential_source": {"url": "http://127.0.0.1:8190/health"}}' + - model_name: vertex_ai/gemini-2.5-flash + litellm_params: + model: vertex_ai/gemini-2.5-flash + api_base: http://127.0.0.1:8191 + vertex_project: synthetic-vertex-project + vertex_location: global + vertex_credentials: '{"type": "external_account", "audience": "synthetic-vertex-audience", "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", "token_url": "http://127.0.0.1:8190/_oauth/token", "credential_source": {"url": "http://127.0.0.1:8190/health"}}' - model_name: vertex_ai/gemini-3.8-flash litellm_params: model: vertex_ai/gemini-3.8-flash diff --git a/tests/integration/translation/responses/bases/vertex_ai.py b/tests/integration/translation/responses/bases/vertex_ai.py index 59637e380ce..3b31b995aa8 100644 --- a/tests/integration/translation/responses/bases/vertex_ai.py +++ b/tests/integration/translation/responses/bases/vertex_ai.py @@ -1,4 +1,5 @@ import json +from dataclasses import replace from typing import Final from unittest.mock import ANY @@ -1858,3 +1859,37 @@ GEMINI_4_ARGON_TEST_CASE: Final = TranslationTestCase( "provider_specific_fields": {"traffic_type": "ON_DEMAND"}, }, ) + + +GEMINI_2_5_FLASH_TEST_CASE: Final = replace( + GEMINI_3_5_FLASH_TEST_CASE, + litellm_request={**GEMINI_3_5_FLASH_TEST_CASE.litellm_request, "model": "vertex_ai/gemini-2.5-flash"}, + expected_provider_endpoint=( + "/v1/projects/synthetic-vertex-project/locations/global/publishers/google/models/gemini-2.5-flash:generateContent" + ), + mock_provider_response={ + "candidates": [{"content": {"role": "model", "parts": [{"text": "Hello."}]}, "finishReason": "STOP"}], + "usageMetadata": { + "promptTokenCount": 9, + "candidatesTokenCount": 2, + "totalTokenCount": 32, + "trafficType": "ON_DEMAND", + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 9}], + "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 2}], + "thoughtsTokenCount": 21, + }, + "modelVersion": "gemini-2.5-flash", + "createTime": "2026-10-31T23:26:23.954987Z", + "responseId": "H3nmauukOrv46tkPw5vggAo", + }, + expected_litellm_response={ + **GEMINI_3_5_FLASH_TEST_CASE.expected_litellm_response, + "model": "vertex_ai/gemini-2.5-flash", + "usage": { + **GEMINI_3_5_FLASH_TEST_CASE.expected_litellm_response["usage"], + "output_tokens": 23, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 21, "text_tokens": 2}, + "total_tokens": 32, + }, + }, +) diff --git a/tests/integration/translation/responses/basic/test_responses_basic_vertex_ai.py b/tests/integration/translation/responses/basic/test_responses_basic_vertex_ai.py index bd9118482d0..6d6e08835a3 100644 --- a/tests/integration/translation/responses/basic/test_responses_basic_vertex_ai.py +++ b/tests/integration/translation/responses/basic/test_responses_basic_vertex_ai.py @@ -11,6 +11,7 @@ from integration.translation.responses.bases.vertex_ai import ( CLAUDE_SONNET_4_6_TEST_CASE, CLAUDE_SONNET_5_5_TEST_CASE, CLAUDE_SONNET_5_TEST_CASE, + GEMINI_2_5_FLASH_TEST_CASE, GEMINI_3_1_PRO_PREVIEW_TEST_CASE, GEMINI_3_5_FLASH_TEST_CASE, GEMINI_3_8_FLASH_TEST_CASE, @@ -30,6 +31,7 @@ from integration.translation.runner import assert_translation CLAUDE_HAIKU_5_5_TEST_CASE, CLAUDE_OPUS_5_5_TEST_CASE, CLAUDE_FABLE_5_1_TEST_CASE, + GEMINI_2_5_FLASH_TEST_CASE, GEMINI_3_5_FLASH_TEST_CASE, GEMINI_3_8_FLASH_TEST_CASE, GEMINI_3_1_PRO_PREVIEW_TEST_CASE, diff --git a/tests/integration/translation/responses/sampling/test_responses_sampling_vertex_ai.py b/tests/integration/translation/responses/sampling/test_responses_sampling_vertex_ai.py new file mode 100644 index 00000000000..a89e8a867a7 --- /dev/null +++ b/tests/integration/translation/responses/sampling/test_responses_sampling_vertex_ai.py @@ -0,0 +1,113 @@ +from dataclasses import replace +from typing import Final + +import pytest +from integration._support.client import Gateway, object_value +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.responses.bases.vertex_ai import GEMINI_2_5_FLASH_TEST_CASE +from integration.translation.runner import assert_translation + +GENERATION_CONFIG: Final = object_value(GEMINI_2_5_FLASH_TEST_CASE.expected_provider_request["generationConfig"]) +USAGE: Final = object_value(GEMINI_2_5_FLASH_TEST_CASE.expected_litellm_response["usage"]) + +GEMINI_2_5_FLASH_TOP_LOGPROBS_TEST_CASE: Final = replace( + GEMINI_2_5_FLASH_TEST_CASE, + scenario="top_logprobs", + litellm_request={**GEMINI_2_5_FLASH_TEST_CASE.litellm_request, "top_logprobs": 2}, + expected_provider_request={ + **GEMINI_2_5_FLASH_TEST_CASE.expected_provider_request, + "generationConfig": {**GENERATION_CONFIG, "responseLogprobs": True, "logprobs": 2}, + }, + mock_provider_response={ + "candidates": [ + { + "content": {"role": "model", "parts": [{"text": "Hello."}]}, + "finishReason": "STOP", + "avgLogprobs": -2.20536901451851e-06, + "logprobsResult": { + "topCandidates": [ + { + "candidates": [ + {"token": "Hello", "logProbability": -1.3113013e-06, "tokenId": 9259}, + {"token": "Hi", "logProbability": -15.149978, "tokenId": 10979}, + ] + }, + { + "candidates": [ + {"token": ".", "logProbability": -3.0994368e-06, "tokenId": 236761}, + {"token": "", "logProbability": -13.81299, "tokenId": 106}, + ] + }, + ], + "chosenCandidates": [ + {"token": "Hello", "logProbability": -1.3113013e-06, "tokenId": 9259}, + {"token": ".", "logProbability": -3.0994368e-06, "tokenId": 236761}, + ], + }, + } + ], + "usageMetadata": { + "promptTokenCount": 9, + "candidatesTokenCount": 2, + "totalTokenCount": 30, + "trafficType": "ON_DEMAND", + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 9}], + "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 2}], + "thoughtsTokenCount": 19, + }, + "modelVersion": "gemini-2.5-flash", + "createTime": "2026-10-31T23:20:20.755722Z", + "responseId": "tHfmaoqQLrKZ9LsPvuuB-AU", + }, + expected_litellm_response={ + **GEMINI_2_5_FLASH_TEST_CASE.expected_litellm_response, + "output": [ + { + "type": "message", + "id": GEMINI_2_5_FLASH_TEST_CASE.expected_litellm_response["output"][0]["id"], + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Hello.", + "annotations": [], + "logprobs": [ + { + "token": "Hello", + "logprob": -1.3113013e-06, + "bytes": None, + "top_logprobs": [ + {"token": "Hello", "logprob": -1.3113013e-06, "bytes": None}, + {"token": "Hi", "logprob": -15.149978, "bytes": None}, + ], + }, + { + "token": ".", + "logprob": -3.0994368e-06, + "bytes": None, + "top_logprobs": [ + {"token": ".", "logprob": -3.0994368e-06, "bytes": None}, + {"token": "", "logprob": -13.81299, "bytes": None}, + ], + }, + ], + } + ], + "phase": None, + } + ], + "usage": { + **USAGE, + "output_tokens": 21, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 19, "text_tokens": 2}, + "total_tokens": 30, + }, + }, +) + + +@pytest.mark.parametrize("case", [GEMINI_2_5_FLASH_TOP_LOGPROBS_TEST_CASE], ids=lambda case: case.id) +def test_responses_sampling_vertex_ai(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index b4cd606374a..c446fba115c 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -18,7 +18,6 @@ from openai.types.responses.response_function_web_search import ( from pydantic import TypeAdapter, ValidationError import litellm -from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field from litellm.constants import CALLER_SESSION_ID_METADATA_KEY from litellm.integrations.anthropic_cache_control_hook import MAX_CACHE_CONTROL_BLOCKS, AnthropicCacheControlHook from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR @@ -26,6 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( _bedrock_converse_messages_pt, anthropic_messages_pt, ) +from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field from litellm.responses.litellm_completion_transformation.reasoning_items import encode_thinking_blocks from litellm.responses.litellm_completion_transformation.tool_search import tool_search_routing from litellm.responses.litellm_completion_transformation.transformation import ( @@ -38,12 +38,15 @@ from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams from litellm.types.responses.main import OutputToolSearchCall, build_web_search_call from litellm.types.utils import ( ChatCompletionMessageToolCall, + ChatCompletionTokenLogprob, + ChoiceLogprobs, Choices, CompletionTokensDetailsWrapper, Function, Message, ModelResponse, PromptTokensDetailsWrapper, + TopLogprob, Usage, ) @@ -6876,7 +6879,11 @@ def test_a_new_message_after_a_paused_search_turn_replays_only_that_turn_without [ {"role": "user", "content": "Plan a Tokyo trip."}, _reasoning_item("rs_tokyo", tokyo_thinking), - {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "Tokyo is sunny."}]}, + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "Tokyo is sunny."}], + }, _hosted_search("ws_srvtoolu_01Tokyo", "completed"), {"role": "user", "content": "Now Osaka."}, _reasoning_item("rs_osaka", _ANSWERED_THINKING), @@ -7216,7 +7223,11 @@ def test_a_tool_output_with_an_unhashable_call_id_is_a_400_naming_its_input_item ) -> None: history: Final = cast( ResponseInputParam, - [_WEATHER_ASK, _weather_call(call_id="call_weather"), {"type": output_type, "call_id": call_id, "output": "sunny"}], + [ + _WEATHER_ASK, + _weather_call(call_id="call_weather"), + {"type": output_type, "call_id": call_id, "output": "sunny"}, + ], ) with pytest.raises(litellm.BadRequestError) as raised: @@ -7320,7 +7331,11 @@ def test_a_non_string_role_still_converts_for_a_target_that_judges_it_itself() - id="dict_function_call_id", ), pytest.param( - [_WEATHER_ASK, _weather_call(call_id=7), {"type": "function_call_output", "call_id": "7", "output": "sunny"}], + [ + _WEATHER_ASK, + _weather_call(call_id=7), + {"type": "function_call_output", "call_id": "7", "output": "sunny"}, + ], "input[1].call_id", "int", id="int_function_call_id", @@ -7394,3 +7409,89 @@ def test_a_malformed_item_after_a_codex_additional_tools_item_is_named_by_its_in ) assert f"'{field_path}'" in raised.value.message + + +_HELLO_LOGPROBS: Final = ChoiceLogprobs( + content=[ + ChatCompletionTokenLogprob( + token="Hello", + logprob=-1.3113013e-06, + top_logprobs=[ + TopLogprob(token="Hello", logprob=-1.3113013e-06), + TopLogprob(token="Hi", logprob=-15.149978), + ], + ), + ChatCompletionTokenLogprob(token=".", logprob=-3.0994368e-06, bytes=[46]), + ] +) + + +def _hello_chat_response(logprobs: ChoiceLogprobs | None) -> ModelResponse: + return ModelResponse( + id="chatcmpl-hello", + created=1793489000, + model="gemini-3.5-flash", + object="chat.completion", + choices=[ + Choices( + finish_reason="stop", index=0, message=Message(content="Hello.", role="assistant"), logprobs=logprobs + ) + ], + ) + + +def test_bridged_chat_logprobs_become_output_text_logprobs() -> None: + response: Final = LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( + request_input="Say hello.", + responses_api_request={"top_logprobs": 2}, + chat_completion_response=_hello_chat_response(_HELLO_LOGPROBS), + ) + + assert response.model_dump()["output"][0]["content"] == [ + { + "type": "output_text", + "text": "Hello.", + "annotations": [], + "logprobs": [ + { + "token": "Hello", + "logprob": -1.3113013e-06, + "bytes": None, + "top_logprobs": [ + {"token": "Hello", "logprob": -1.3113013e-06, "bytes": None}, + {"token": "Hi", "logprob": -15.149978, "bytes": None}, + ], + }, + {"token": ".", "logprob": -3.0994368e-06, "bytes": [46], "top_logprobs": []}, + ], + } + ] + + +def test_bridged_chat_response_without_logprobs_emits_no_logprobs_key() -> None: + response: Final = LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( + request_input="Say hello.", responses_api_request={}, chat_completion_response=_hello_chat_response(None) + ) + + assert response.model_dump()["output"][0]["content"] == [ + {"type": "output_text", "text": "Hello.", "annotations": []} + ] + + +def test_responses_top_logprobs_bridges_to_chat_logprobs() -> None: + request: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model="gemini-3.5-flash", + input="Say hello.", + responses_api_request={"top_logprobs": 2}, + custom_llm_provider="vertex_ai", + ) + + assert (request["top_logprobs"], request["logprobs"]) == (2, True) + + +def test_responses_without_top_logprobs_bridges_without_chat_logprobs() -> None: + request: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model="gemini-3.5-flash", input="Say hello.", responses_api_request={}, custom_llm_provider="vertex_ai" + ) + + assert "top_logprobs" not in request and "logprobs" not in request diff --git a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index 684f5a89e80..4605857dcae 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -30,6 +30,8 @@ from litellm.types.llms.openai import ( ) from litellm.types.responses.main import build_web_search_call from litellm.types.utils import ( + ChatCompletionTokenLogprob, + ChoiceLogprobs, Delta, ModelResponse, ModelResponseStream, @@ -38,9 +40,7 @@ from litellm.types.utils import ( ) CHAT_COMPLETION_ID = "chatcmpl-77d33d09-effa-4cd2-9c0d-c742d4358256" -RESPONSE_ID_EVENT_TYPES = frozenset( - {"response.created", "response.in_progress", "response.completed"} -) +RESPONSE_ID_EVENT_TYPES = frozenset({"response.created", "response.in_progress", "response.completed"}) def _chunk(content: str, finish_reason: str | None = None) -> ModelResponseStream: @@ -93,11 +93,7 @@ def _build_iterator(chunks) -> LiteLLMCompletionStreamingIterator: def _response_ids(events) -> list[str]: - return [ - event.response.id - for event in events - if getattr(event, "type", None) in RESPONSE_ID_EVENT_TYPES - ] + return [event.response.id for event in events if getattr(event, "type", None) in RESPONSE_ID_EVENT_TYPES] def test_tool_call_delta_is_emitted_as_responses_events(): @@ -228,9 +224,7 @@ async def test_web_search_stream_preserves_hosted_and_client_calls(sync_mode, to content=None, provider_specific_fields={ "web_search_results": [first_result], - "web_search_calls": [ - build_web_search_call(call_ids[0], {"query": "one"}, first_result) - ] + "web_search_calls": [build_web_search_call(call_ids[0], {"query": "one"}, first_result)] if tool_type != "function" and first_result["type"] == "web_search_tool_result" else [], }, @@ -329,9 +323,7 @@ async def test_web_search_stream_preserves_hosted_and_client_calls(sync_mode, to assert {event["output_index"] for event in search_events} == {completed_indexes[item["id"]]} assert search_events[0]["item"]["status"] == "in_progress" assert search_events[-1]["item"] == item - assert item["status"] == ( - "failed" if result_kind == "error" and call_id.endswith("01Search") else "completed" - ) + assert item["status"] == ("failed" if result_kind == "error" and call_id.endswith("01Search") else "completed") assert item["action"]["type"] == "search" assert item["action"]["query"] == ("one" if call_id.endswith("01Search") else "two") assert item["action"]["queries"] == [item["action"]["query"]] @@ -440,9 +432,7 @@ def test_tool_call_arguments_are_chunked_to_match_openai_behavior(): ) # Create a chunk with a large arguments string that should be split - large_arguments = ( - '{"param1": "value1", "param2": "value2", "param3": "value3"}' # 67 chars - ) + large_arguments = '{"param1": "value1", "param2": "value2", "param3": "value3"}' # 67 chars chunk = ModelResponseStream( id="chunk-1", created=123, @@ -497,9 +487,7 @@ def test_tool_call_arguments_are_chunked_to_match_openai_behavior(): # Keep draining pending events (expected: ceil(67 / 10) = 7 delta events) while iterator._pending_tool_events: - evt = iterator._transform_chat_completion_chunk_to_response_api_chunk( - empty_chunk - ) + evt = iterator._transform_chat_completion_chunk_to_response_api_chunk(empty_chunk) if evt and evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA: delta_events.append(evt) @@ -552,20 +540,12 @@ def test_tool_call_delta_without_id_uses_index_mapping(): while iterator._pending_tool_events: all_events.append(iterator._pending_tool_events.pop(0)) - delta_events = [ - evt - for evt in all_events - if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA - ] + delta_events = [evt for evt in all_events if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA] streamed_arguments = "".join(evt.delta for evt in delta_events) assert streamed_arguments == '{"location": "New York"}' - output_item_added_events = [ - evt - for evt in all_events - if evt.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED - ] + output_item_added_events = [evt for evt in all_events if evt.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED] assert len(output_item_added_events) == 1 assert output_item_added_events[0].item.id == "fc_call_abc123" assert output_item_added_events[0].item.call_id == "call_abc123" @@ -606,18 +586,10 @@ def test_parallel_tool_calls_without_ids_use_index_mapping(): while iterator._pending_tool_events: all_events.append(iterator._pending_tool_events.pop(0)) - output_item_added_events = [ - evt - for evt in all_events - if evt.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED - ] + output_item_added_events = [evt for evt in all_events if evt.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED] assert len(output_item_added_events) == 2 - delta_events = [ - evt - for evt in all_events - if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA - ] + delta_events = [evt for evt in all_events if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA] arguments_by_call_id = {} for evt in delta_events: arguments_by_call_id.setdefault(evt.item_id, "") @@ -670,11 +642,7 @@ def test_reused_index_with_new_call_id_marks_fallback_ambiguous(): while iterator._pending_tool_events: all_events.append(iterator._pending_tool_events.pop(0)) - delta_events = [ - evt - for evt in all_events - if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA - ] + delta_events = [evt for evt in all_events if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA] arguments_by_call_id = {} for evt in delta_events: arguments_by_call_id.setdefault(evt.item_id, "") @@ -741,22 +709,17 @@ def test_sync_streaming_events_share_the_chat_completion_response_id(): assert len(response_ids) == 3 assert len(set(response_ids)) == 1 assert ( - ResponsesAPIRequestUtils.decode_responses_api_response_id(response_ids[0])["response_id"] - == CHAT_COMPLETION_ID + ResponsesAPIRequestUtils.decode_responses_api_response_id(response_ids[0])["response_id"] == CHAT_COMPLETION_ID ) @pytest.mark.asyncio async def test_streaming_emits_every_chunk_after_priming_the_response_id(): - iterator = _build_iterator( - [_chunk("Hel"), _chunk("lo"), _chunk("!", finish_reason="stop")] - ) + iterator = _build_iterator([_chunk("Hel"), _chunk("lo"), _chunk("!", finish_reason="stop")]) events = [event async for event in iterator] - deltas = "".join( - event.delta for event in events if getattr(event, "type", None) == "response.output_text.delta" - ) + deltas = "".join(event.delta for event in events if getattr(event, "type", None) == "response.output_text.delta") assert deltas == "Hello!" @@ -802,7 +765,9 @@ async def test_leading_empty_choices_chunk_does_not_kill_the_stream(): event_types = [getattr(event, "type", None) for event in events] assert event_types.count(ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED) == 1 - assert "".join(event.delta for event in events if event.type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA) == "Hello!" + assert ( + "".join(event.delta for event in events if event.type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA) == "Hello!" + ) assert event_types[-1] == ResponsesAPIStreamEvents.RESPONSE_COMPLETED @@ -986,9 +951,7 @@ def test_streamed_unrecognized_tool_choice_is_echoed_as_auto() -> None: litellm_metadata={}, ) - response_events: Final = [ - event for event in iterator if getattr(event, "type", None) in RESPONSE_ID_EVENT_TYPES - ] + response_events: Final = [event for event in iterator if getattr(event, "type", None) in RESPONSE_ID_EVENT_TYPES] assert [event.response.tool_choice for event in response_events] == ["auto", "auto", "auto"] @@ -1038,9 +1001,7 @@ def _signed_thinking_and_text_in_one_chunk() -> tuple[ModelResponseStream, ...]: chunk: Final = _chunk("Hello!", finish_reason="stop") signed_block: Final = {"type": "thinking", "thinking": "", "signature": "sig_only"} delta: Final = Delta(**{**chunk.choices[0].delta.model_dump(), "thinking_blocks": [signed_block]}) - return ( - chunk.model_copy(update={"choices": [StreamingChoices(index=0, delta=delta, finish_reason="stop")]}), - ) + return (chunk.model_copy(update={"choices": [StreamingChoices(index=0, delta=delta, finish_reason="stop")]}),) StreamChunks = Callable[[], tuple[ModelResponseStream, ...]] @@ -1560,9 +1521,7 @@ async def test_anthropic_tool_search_call_streams_as_tool_search_call_item(sync_ model="claude-sonnet-5-5", object="chat.completion.chunk", choices=[ - StreamingChoices( - index=0, delta=delta, finish_reason="tool_calls" if index == len(deltas) - 1 else None - ) + StreamingChoices(index=0, delta=delta, finish_reason="tool_calls" if index == len(deltas) - 1 else None) ], ) for index, delta in enumerate(deltas) @@ -1972,9 +1931,7 @@ def test_the_partial_response_billed_on_disconnect_carries_the_logging_objects_b custom_llm_provider="anthropic", litellm_metadata={}, ) - relayed: Final = tuple( - itertools.takewhile(lambda event: not isinstance(event, OutputTextDeltaEvent), iterator) - ) + relayed: Final = tuple(itertools.takewhile(lambda event: not isinstance(event, OutputTextDeltaEvent), iterator)) assert relayed, "the stream must relay at least one chunk before the client goes away" partial: Final = iterator.partial_response_on_disconnect() @@ -2026,3 +1983,55 @@ async def test_streamed_refusal_deltas_become_one_refusal_part_of_the_message(sy assert [event.refusal for event in refusal_done] == [expected_content[0]["refusal"]] assert [item.model_dump()["content"] for item in message_done] == [expected_content] assert [item.model_dump()["content"] for item in finished.response.output] == [expected_content] + + +def _logprobs_chunk(content: str, logprob: float, finish_reason: str | None = None) -> ModelResponseStream: + return ModelResponseStream( + id=CHAT_COMPLETION_ID, + created=1748575031, + model="gemini-3.5-flash", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + index=0, + delta=Delta(role="assistant", content=content), + finish_reason=finish_reason, + logprobs=ChoiceLogprobs(content=[ChatCompletionTokenLogprob(token=content, logprob=logprob)]), + ) + ], + ) + + +def test_streamed_chat_logprobs_reach_the_text_delta_and_done_events() -> None: + events: Final = [ + event.model_dump() + for event in _build_iterator( + [_logprobs_chunk("Hello", -0.25), _logprobs_chunk(".", -0.5, finish_reason="stop")] + ) + ] + hello: Final = {"token": "Hello", "logprob": -0.25, "bytes": None, "top_logprobs": []} + period: Final = {"token": ".", "logprob": -0.5, "bytes": None, "top_logprobs": []} + + deltas: Final = [event for event in events if event["type"] == "response.output_text.delta"] + done: Final = next(event for event in events if event["type"] == "response.output_text.done") + part_done: Final = next(event for event in events if event["type"] == "response.content_part.done") + item_done: Final = next( + event for event in events if event["type"] == "response.output_item.done" and event["item"]["type"] == "message" + ) + + assert [event["logprobs"] for event in deltas] == [[hello], [period]] + assert done["logprobs"] == [hello, period] + assert part_done["part"]["logprobs"] == [hello, period] + assert item_done["item"]["content"][0]["logprobs"] == [hello, period] + + +def test_streamed_chat_without_logprobs_emits_text_events_without_a_logprobs_key() -> None: + events: Final = [ + event.model_dump() for event in _build_iterator([_chunk("Hello"), _chunk("!", finish_reason="stop")]) + ] + + text_events: Final = [ + event for event in events if event["type"] in {"response.output_text.delta", "response.output_text.done"} + ] + + assert text_events and all("logprobs" not in event for event in text_events) From f20a6ae32920bf6d9a22e18b55eacacd13f61cdb Mon Sep 17 00:00:00 2001 From: mateo Date: Sat, 31 Oct 2026 23:39:28 +0000 Subject: [PATCH 02/15] fix(responses): keep the bridged logprobs immutable and typed for the lint gates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../logprobs.py | 23 ++++++++++++------- .../streaming_iterator.py | 4 ++-- .../transformation.py | 4 ++-- litellm/types/llms/base.py | 14 +++++------ litellm/types/llms/openai.py | 4 ++-- .../test_litellm_completion_responses.py | 4 ++-- .../test_streaming_iterator_transformation.py | 4 ++-- 7 files changed, 32 insertions(+), 25 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/logprobs.py b/litellm/responses/litellm_completion_transformation/logprobs.py index 81953ccd602..255667ae5a4 100644 --- a/litellm/responses/litellm_completion_transformation/logprobs.py +++ b/litellm/responses/litellm_completion_transformation/logprobs.py @@ -1,14 +1,13 @@ """Chat ``choice.logprobs`` as Responses ``output_text.logprobs``, which share one token shape.""" +from collections.abc import Mapping from typing import Final -from pydantic import TypeAdapter, ValidationError +from pydantic import ValidationError from litellm.types.llms.base import OutputTextLogprob, OutputTextTopLogprob from litellm.types.utils import ChatCompletionTokenLogprob, ChoiceLogprobs, TopLogprob -_CHOICE_LOGPROBS: Final = TypeAdapter(ChoiceLogprobs | None) - def _top_logprob(top: TopLogprob) -> OutputTextTopLogprob: return {"token": top.token, "logprob": top.logprob, "bytes": top.bytes} @@ -19,16 +18,24 @@ def _token_logprob(token: ChatCompletionTokenLogprob) -> OutputTextLogprob: "token": token.token, "logprob": token.logprob, "bytes": token.bytes, - "top_logprobs": [_top_logprob(top) for top in token.top_logprobs], + "top_logprobs": tuple(_top_logprob(top) for top in token.top_logprobs), } -def output_text_logprobs(choice_logprobs: object) -> list[OutputTextLogprob] | None: - """The ``logprobs`` a Responses text part carries for a chat choice, ``None`` when the choice sent none.""" +def _choice_logprobs(choice_logprobs: object) -> ChoiceLogprobs | None: + if isinstance(choice_logprobs, ChoiceLogprobs): + return choice_logprobs + if not isinstance(choice_logprobs, Mapping): + return None try: - validated: Final = _CHOICE_LOGPROBS.validate_python(choice_logprobs) + return ChoiceLogprobs.model_validate(choice_logprobs) except ValidationError: return None + + +def output_text_logprobs(choice_logprobs: object) -> tuple[OutputTextLogprob, ...] | None: + """The ``logprobs`` a Responses text part carries for a chat choice, ``None`` when the choice sent none.""" + validated: Final = _choice_logprobs(choice_logprobs) if validated is None or validated.content is None: return None - return [_token_logprob(token) for token in validated.content] + return tuple(_token_logprob(token) for token in validated.content) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 0f84159c7dc..7364eabb602 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -1021,9 +1021,9 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): logprobs=self._final_text_logprobs(litellm_complete_object), ) - def _final_text_logprobs(self, litellm_complete_object: ModelResponse) -> list[OutputTextLogprob] | None: + def _final_text_logprobs(self, litellm_complete_object: ModelResponse) -> tuple[OutputTextLogprob, ...] | None: if self._text_logprobs: - return list(self._text_logprobs) + return self._text_logprobs return output_text_logprobs(getattr(litellm_complete_object.choices[0], "logprobs", None)) def create_output_content_part_done_event(self, litellm_complete_object: ModelResponse) -> ContentPartDoneEvent: diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 66750d26eaa..7f9f2f30ed6 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -3716,7 +3716,7 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def _message_output_content( - message: Message, logprobs: list[OutputTextLogprob] | None = None + message: Message, logprobs: Sequence[OutputTextLogprob] | None = None ) -> tuple[OutputText | ResponseOutputRefusal, ...]: refusal: Final = openai_chat_refusal_text(message) refusal_parts: Final = (ResponseOutputRefusal(type="refusal", refusal=refusal),) if refusal is not None else () @@ -3778,7 +3778,7 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def _transform_chat_message_to_response_output_text( message: Message, - logprobs: list[OutputTextLogprob] | None = None, + logprobs: Sequence[OutputTextLogprob] | None = None, ) -> OutputText: annotations: Final = getattr(message, "annotations", None) transformed_annotations: Final = ( diff --git a/litellm/types/llms/base.py b/litellm/types/llms/base.py index d82b73705ef..a7cbd865384 100644 --- a/litellm/types/llms/base.py +++ b/litellm/types/llms/base.py @@ -1,5 +1,5 @@ -from collections.abc import Mapping -from typing import Final, cast +from collections.abc import Mapping, Sequence +from typing import Final from openai._models import BaseModel as OpenAIObject from pydantic import ConfigDict, SerializerFunctionWrapHandler, model_serializer @@ -90,24 +90,24 @@ class CachedTokensDetails(LiteLLMBaseModel): class OutputTextTopLogprob(TypedDict): token: ReadOnly[str] logprob: ReadOnly[float] - bytes: ReadOnly[list[int] | None] + bytes: ReadOnly[Sequence[int] | None] class OutputTextLogprob(TypedDict): token: ReadOnly[str] logprob: ReadOnly[float] - bytes: ReadOnly[list[int] | None] - top_logprobs: ReadOnly[list[OutputTextTopLogprob]] + bytes: ReadOnly[Sequence[int] | None] + top_logprobs: ReadOnly[Sequence[OutputTextTopLogprob]] class CarriesOutputTextLogprobs(BaseLiteLLMOpenAIResponseObject): """A Responses text payload carries ``logprobs`` only when the chat choice behind it sent them.""" - logprobs: list[OutputTextLogprob] | None = None + logprobs: Sequence[OutputTextLogprob] | None = None @model_serializer(mode="wrap") def _without_absent_logprobs(self, handler: SerializerFunctionWrapHandler) -> Mapping[str, object]: - dumped: Final = cast("Mapping[str, object]", handler(self)) # cast-ok: handler dumps this model's fields + dumped: Final[Mapping[str, object]] = handler(self) if self.logprobs is not None: return dumped return {key: value for key, value in dumped.items() if key != "logprobs"} diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index c079cba3aba..f5f502f8552 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -3,7 +3,7 @@ import math from collections.abc import Iterable, Mapping, Sequence from enum import Enum from os import PathLike -from typing import IO, TYPE_CHECKING, Any, Final, Generic, Literal, Optional, TypeAlias, Union, cast +from typing import IO, TYPE_CHECKING, Any, Final, Generic, Literal, Optional, TypeAlias, Union import httpx from openai import Omit @@ -1868,7 +1868,7 @@ class ContentPartDonePartOutputText(BaseLiteLLMOpenAIResponseObject): type: Literal["output_text"] text: str annotations: list[BaseLiteLLMOpenAIResponseObject] - logprobs: list[OutputTextLogprob] | None = None + logprobs: Sequence[OutputTextLogprob] | None = None class ContentPartDonePartRefusal(BaseLiteLLMOpenAIResponseObject): diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index c446fba115c..f9dba9bf994 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -7447,7 +7447,7 @@ def test_bridged_chat_logprobs_become_output_text_logprobs() -> None: chat_completion_response=_hello_chat_response(_HELLO_LOGPROBS), ) - assert response.model_dump()["output"][0]["content"] == [ + assert response.model_dump(mode="json")["output"][0]["content"] == [ { "type": "output_text", "text": "Hello.", @@ -7473,7 +7473,7 @@ def test_bridged_chat_response_without_logprobs_emits_no_logprobs_key() -> None: request_input="Say hello.", responses_api_request={}, chat_completion_response=_hello_chat_response(None) ) - assert response.model_dump()["output"][0]["content"] == [ + assert response.model_dump(mode="json")["output"][0]["content"] == [ {"type": "output_text", "text": "Hello.", "annotations": []} ] diff --git a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index 4605857dcae..fd7a91d12f6 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -2004,7 +2004,7 @@ def _logprobs_chunk(content: str, logprob: float, finish_reason: str | None = No def test_streamed_chat_logprobs_reach_the_text_delta_and_done_events() -> None: events: Final = [ - event.model_dump() + event.model_dump(mode="json") for event in _build_iterator( [_logprobs_chunk("Hello", -0.25), _logprobs_chunk(".", -0.5, finish_reason="stop")] ) @@ -2027,7 +2027,7 @@ def test_streamed_chat_logprobs_reach_the_text_delta_and_done_events() -> None: def test_streamed_chat_without_logprobs_emits_text_events_without_a_logprobs_key() -> None: events: Final = [ - event.model_dump() for event in _build_iterator([_chunk("Hello"), _chunk("!", finish_reason="stop")]) + event.model_dump(mode="json") for event in _build_iterator([_chunk("Hello"), _chunk("!", finish_reason="stop")]) ] text_events: Final = [ From fc00f737b5acb4433a79fe5e2976efda57f080ba Mon Sep 17 00:00:00 2001 From: mateo Date: Sat, 31 Oct 2026 23:49:07 +0000 Subject: [PATCH 03/15] fix(responses): read top_logprobs through a typed mapping view for the type gate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_completion_transformation/transformation.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 7f9f2f30ed6..9b2ee9b5892 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -1275,6 +1275,9 @@ class LiteLLMCompletionResponsesConfig: if malformed_field_error is not None: raise malformed_field_error + request_fields: Final[Mapping[str, object]] = responses_api_request + requested_top_logprobs: Final = request_fields.get("top_logprobs") + top_logprobs: Final = requested_top_logprobs if isinstance(requested_top_logprobs, int) else None litellm_completion_request: dict = { "messages": ( list( @@ -1292,8 +1295,8 @@ class LiteLLMCompletionResponsesConfig: ), "tools": tools, "top_p": responses_api_request.get("top_p"), - "top_logprobs": responses_api_request.get("top_logprobs"), - "logprobs": True if responses_api_request.get("top_logprobs") is not None else None, + "top_logprobs": top_logprobs, + "logprobs": True if top_logprobs is not None else None, "user": responses_api_request.get("user"), "temperature": responses_api_request.get("temperature"), "parallel_tool_calls": responses_api_request.get("parallel_tool_calls"), From 0e0b8bd254c2c78c8653cd072d55bc0b25d6bcca Mon Sep 17 00:00:00 2001 From: mateo Date: Sat, 31 Oct 2026 23:52:39 +0000 Subject: [PATCH 04/15] fix(responses): read top_logprobs through a Mapping-typed helper for the type gate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_completion_transformation/transformation.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 9b2ee9b5892..2a2f9fb0f41 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -924,6 +924,11 @@ class ChatCompletionSession(TypedDict, total=False): ########### End of Initialize Classes used for Responses API ########### +def _requested_top_logprobs(request_fields: Mapping[str, object]) -> int | None: + value: Final = request_fields.get("top_logprobs") + return value if isinstance(value, int) else None + + def _chain_session_metadata(chain_session: str, request_kwargs: Mapping[str, object]) -> Mapping[str, object]: key: Final = "litellm_metadata" if request_kwargs.get("litellm_metadata") else "metadata" metadata: Final = request_kwargs.get(key) @@ -1275,9 +1280,7 @@ class LiteLLMCompletionResponsesConfig: if malformed_field_error is not None: raise malformed_field_error - request_fields: Final[Mapping[str, object]] = responses_api_request - requested_top_logprobs: Final = request_fields.get("top_logprobs") - top_logprobs: Final = requested_top_logprobs if isinstance(requested_top_logprobs, int) else None + top_logprobs: Final = _requested_top_logprobs(responses_api_request) litellm_completion_request: dict = { "messages": ( list( From 040f5cb55a9de50ea564ab7be43afc9d9637ca07 Mon Sep 17 00:00:00 2001 From: mateo Date: Sat, 31 Oct 2026 23:56:48 +0000 Subject: [PATCH 05/15] chore(openapi): regenerate the lazy snapshot and UI API types for output_text logprobs Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/_lazy_openapi_snapshot.json | 87 +++++++++++++++-------- 1 file changed, 59 insertions(+), 28 deletions(-) diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 2bbc7f6500a..8428bd644f5 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -26474,13 +26474,16 @@ }, "OutputText": { "additionalProperties": true, - "description": "Text output content from an assistant message", + "type": "object" + }, + "OutputTextLogprob": { + "additionalProperties": true, "properties": { - "annotations": { + "bytes": { "anyOf": [ { "items": { - "$ref": "#/components/schemas/GenericResponseOutputItemContentAnnotation" + "type": "integer" }, "type": "array" }, @@ -26488,37 +26491,65 @@ "type": "null" } ], - "title": "Annotations" + "title": "Bytes" }, - "text": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Text" + "logprob": { + "title": "Logprob", + "type": "number" }, - "type": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Type" + "token": { + "title": "Token", + "type": "string" + }, + "top_logprobs": { + "items": { + "$ref": "#/components/schemas/OutputTextTopLogprob" + }, + "title": "Top Logprobs", + "type": "array" } }, "required": [ - "type", - "text", - "annotations" + "token", + "logprob", + "bytes", + "top_logprobs" ], - "title": "OutputText", + "title": "OutputTextLogprob", + "type": "object" + }, + "OutputTextTopLogprob": { + "additionalProperties": true, + "properties": { + "bytes": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Bytes" + }, + "logprob": { + "title": "Logprob", + "type": "number" + }, + "token": { + "title": "Token", + "type": "string" + } + }, + "required": [ + "token", + "logprob", + "bytes" + ], + "title": "OutputTextTopLogprob", "type": "object" }, "OutputTokensDetails": { From cd3c3385775715095e4dbd1bcff237dee9161013 Mon Sep 17 00:00:00 2001 From: mateo Date: Sat, 31 Oct 2026 23:57:23 +0000 Subject: [PATCH 06/15] chore(ui): regenerate schema.d.ts for output_text logprobs Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ui/litellm-dashboard/src/lib/http/schema.d.ts | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index a736f5a2519..49dc57f24ba 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -40723,17 +40723,30 @@ export interface components { } & { [key: string]: unknown; }; - /** - * OutputText - * @description Text output content from an assistant message - */ OutputText: { - /** Annotations */ - annotations: components["schemas"]["GenericResponseOutputItemContentAnnotation"][] | null; - /** Text */ - text: string | null; - /** Type */ - type: string | null; + [key: string]: unknown; + }; + /** OutputTextLogprob */ + OutputTextLogprob: { + /** Bytes */ + bytes: number[] | null; + /** Logprob */ + logprob: number; + /** Token */ + token: string; + /** Top Logprobs */ + top_logprobs: components["schemas"]["OutputTextTopLogprob"][]; + } & { + [key: string]: unknown; + }; + /** OutputTextTopLogprob */ + OutputTextTopLogprob: { + /** Bytes */ + bytes: number[] | null; + /** Logprob */ + logprob: number; + /** Token */ + token: string; } & { [key: string]: unknown; }; From 1988c24a4b03bec0f8bde4d9033e0c7f66424acf Mon Sep 17 00:00:00 2001 From: mateo Date: Sun, 1 Nov 2026 00:07:39 +0000 Subject: [PATCH 07/15] fix(types): keep bytes optional on output_text logprobs Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/types/llms/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/types/llms/base.py b/litellm/types/llms/base.py index a7cbd865384..6af1d4d7046 100644 --- a/litellm/types/llms/base.py +++ b/litellm/types/llms/base.py @@ -3,7 +3,7 @@ from typing import Final from openai._models import BaseModel as OpenAIObject from pydantic import ConfigDict, SerializerFunctionWrapHandler, model_serializer -from typing_extensions import ReadOnly, TypedDict +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.types.llms.base_model import LiteLLMBaseModel @@ -90,13 +90,13 @@ class CachedTokensDetails(LiteLLMBaseModel): class OutputTextTopLogprob(TypedDict): token: ReadOnly[str] logprob: ReadOnly[float] - bytes: ReadOnly[Sequence[int] | None] + bytes: NotRequired[ReadOnly[Sequence[int] | None]] class OutputTextLogprob(TypedDict): token: ReadOnly[str] logprob: ReadOnly[float] - bytes: ReadOnly[Sequence[int] | None] + bytes: NotRequired[ReadOnly[Sequence[int] | None]] top_logprobs: ReadOnly[Sequence[OutputTextTopLogprob]] From 45805384e54c10ec6f5b3cf793238b0af2b50a5b Mon Sep 17 00:00:00 2001 From: mateo Date: Sun, 1 Nov 2026 00:12:33 +0000 Subject: [PATCH 08/15] chore(openapi): regenerate the lazy snapshot and UI API types for optional logprob bytes Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/_lazy_openapi_snapshot.json | 4 +--- ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 8428bd644f5..5fd1f51cc0b 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -26512,7 +26512,6 @@ "required": [ "token", "logprob", - "bytes", "top_logprobs" ], "title": "OutputTextLogprob", @@ -26546,8 +26545,7 @@ }, "required": [ "token", - "logprob", - "bytes" + "logprob" ], "title": "OutputTextTopLogprob", "type": "object" diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 49dc57f24ba..34b0c17f53c 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -40729,7 +40729,7 @@ export interface components { /** OutputTextLogprob */ OutputTextLogprob: { /** Bytes */ - bytes: number[] | null; + bytes?: number[] | null; /** Logprob */ logprob: number; /** Token */ @@ -40742,7 +40742,7 @@ export interface components { /** OutputTextTopLogprob */ OutputTextTopLogprob: { /** Bytes */ - bytes: number[] | null; + bytes?: number[] | null; /** Logprob */ logprob: number; /** Token */ From 3588a1e87760ebec0b2f7cd4894e02d361a1c898 Mon Sep 17 00:00:00 2001 From: mateo Date: Sun, 1 Nov 2026 00:17:01 +0000 Subject: [PATCH 09/15] fix(responses): carry streamed logprobs into the completed snapshot and drop them on guardrail rewrites Greptile review follow-ups on the bridge logprobs change. The response.completed snapshot is rebuilt through stream_chunk_builder, which drops chat logprobs, so the text part now takes the logprobs the stream carried. The per-chunk accumulator appends and flattens once instead of recopying every earlier token. CarriesOutputTextLogprobs carries a frozen-ok reason since guardrails rewrite text parts and stream events in place. A guardrail rewrite of output_text (part, delta, done) now also clears the logprobs that would spell out the original text. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../guardrail_translation/handler.py | 13 ++- .../streaming_iterator.py | 37 ++++++-- litellm/types/llms/base.py | 2 +- ...test_openai_responses_guardrail_handler.py | 89 +++++++++++++++++++ .../test_streaming_iterator_transformation.py | 2 + 5 files changed, 132 insertions(+), 11 deletions(-) diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index a905bff3521..7db75645e88 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -1425,11 +1425,18 @@ class OpenAIResponsesHandler(BaseTranslation): else: setattr(event, field, value) + @staticmethod + def _write_rewritten_text(event: object, field: str, text: str) -> None: + """A rewrite replaces the text, so the logprobs that spell out the original text go with it.""" + OpenAIResponsesHandler._write_event_field(event, field, text) + if stream_item_field(event, "logprobs") is not None: + OpenAIResponsesHandler._write_event_field(event, "logprobs", None) + @staticmethod def _write_content_part_text(part: object, text: str) -> None: field: Final = _content_part_text_field(part) if _has_writable_field(part, field): - OpenAIResponsesHandler._write_event_field(part, field, text) + OpenAIResponsesHandler._write_rewritten_text(part, field, text) @staticmethod def _write_compaction_summary(item: object, summary: str) -> None: @@ -1468,9 +1475,9 @@ class OpenAIResponsesHandler(BaseTranslation): continue position = (output_index, content_index) if event_type in _TEXT_DELTA_EVENT_TYPES and position in delta_replacements: - self._write_event_field(event, "delta", next(delta_replacements[position])) + self._write_rewritten_text(event, "delta", next(delta_replacements[position])) elif event_type == "response.output_text.done" and position in rewrites_by_position: - self._write_event_field(event, "text", rewrites_by_position[position]) + self._write_rewritten_text(event, "text", rewrites_by_position[position]) elif event_type == "response.refusal.done" and position in rewrites_by_position: self._write_event_field(event, "refusal", rewrites_by_position[position]) elif event_type == "response.content_part.done" and position in rewrites_by_position: diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 7364eabb602..bf137629553 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -1,7 +1,7 @@ import time import uuid from collections.abc import Mapping, Sequence -from itertools import filterfalse +from itertools import chain, filterfalse from types import MappingProxyType from typing import Any, Final, cast @@ -63,7 +63,11 @@ from litellm.types.llms.openai import ( WebSearchCallInProgressEvent, WebSearchCallSearchingEvent, ) -from litellm.types.responses.main import GenericResponseOutputItemContentAnnotation +from litellm.types.responses.main import ( + GenericResponseOutputItem, + GenericResponseOutputItemContentAnnotation, + OutputText, +) from litellm.types.utils import Delta as ChatCompletionDelta from litellm.types.utils import ( ModelResponse, @@ -213,7 +217,9 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._sent_reasoning_summary_text_done_event: bool = False self._sent_reasoning_summary_part_done_event: bool = False self._reasoning_summary_text: str = "" - self._text_logprobs: tuple[OutputTextLogprob, ...] = () + self._text_logprob_chunks: list[ + tuple[OutputTextLogprob, ...] + ] = [] # mutable-ok: appended per chunk, flattened once at done # -- GENERIC RESPONSE-EVENTS PENDING QUEUE as required by fix -- self._pending_response_events: list[BaseLiteLLMOpenAIResponseObject] = [] self._reasoning_active = False @@ -1021,9 +1027,15 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): logprobs=self._final_text_logprobs(litellm_complete_object), ) + def _streamed_text_logprobs(self) -> tuple[OutputTextLogprob, ...] | None: + if not self._text_logprob_chunks: + return None + return tuple(chain.from_iterable(self._text_logprob_chunks)) + def _final_text_logprobs(self, litellm_complete_object: ModelResponse) -> tuple[OutputTextLogprob, ...] | None: - if self._text_logprobs: - return self._text_logprobs + streamed: Final = self._streamed_text_logprobs() + if streamed is not None: + return streamed return output_text_logprobs(getattr(litellm_complete_object.choices[0], "logprobs", None)) def create_output_content_part_done_event(self, litellm_complete_object: ModelResponse) -> ContentPartDoneEvent: @@ -1525,7 +1537,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._sequence_number += 1 delta_logprobs: Final = output_text_logprobs(getattr(chunk.choices[0], "logprobs", None)) if delta_logprobs is not None: - self._text_logprobs = (*self._text_logprobs, *delta_logprobs) + self._text_logprob_chunks.append(delta_logprobs) text_delta_event: Final = OutputTextDeltaEvent( type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, item_id=item_id, @@ -1582,7 +1594,18 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): ) compaction_aligned: Final = _output_items_with_id(reasoning_aligned, "compaction", self._compaction_item_id) streamed_items: Final = filterfalse(self._is_unstreamed_empty_message, compaction_aligned) - return tuple(sorted(streamed_items, key=self._streamed_output_position)) + return tuple(map(self._with_streamed_logprobs, sorted(streamed_items, key=self._streamed_output_position))) + + def _with_streamed_logprobs(self, item: object) -> object: + """stream_chunk_builder drops chat logprobs, so the snapshot's text part takes the ones the stream carried.""" + logprobs: Final = self._streamed_text_logprobs() + if logprobs is None or not isinstance(item, GenericResponseOutputItem): + return item + content: Final = tuple( + part.model_copy(update={"logprobs": logprobs}) if isinstance(part, OutputText) else part + for part in item.content + ) + return item.model_copy(update={"content": list(content)}) def _is_unstreamed_empty_message(self, item: object) -> bool: if getattr(item, "type", None) != "message" or self._message_output_index is not None: diff --git a/litellm/types/llms/base.py b/litellm/types/llms/base.py index 6af1d4d7046..231bfff2036 100644 --- a/litellm/types/llms/base.py +++ b/litellm/types/llms/base.py @@ -100,7 +100,7 @@ class OutputTextLogprob(TypedDict): top_logprobs: ReadOnly[Sequence[OutputTextTopLogprob]] -class CarriesOutputTextLogprobs(BaseLiteLLMOpenAIResponseObject): +class CarriesOutputTextLogprobs(BaseLiteLLMOpenAIResponseObject): # frozen-ok: guardrails rewrite these in place """A Responses text payload carries ``logprobs`` only when the chat choice behind it sent them.""" logprobs: Sequence[OutputTextLogprob] | None = None diff --git a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py index d80a5e835c9..5c8b147f842 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -36,6 +36,8 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.llms import get_guardrail_translation_mapping from litellm.llms.openai.responses.guardrail_translation.handler import ( OpenAIResponsesHandler, + stream_item_field, + stream_item_items, ) from litellm.llms.openai.responses.guardrail_translation.tool_merge import merge_guardrailed_tools from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import GenericGuardrailAPI @@ -46,8 +48,11 @@ from litellm.types.guardrails import ContentFilterAction, ContentFilterPattern from litellm.types.llms.openai import ( ChatCompletionToolCallChunk, ContentPartDoneEvent, + ContentPartDonePartOutputText, ContentPartDonePartRefusal, OutputItemDoneEvent, + OutputTextDeltaEvent, + OutputTextDoneEvent, RefusalDeltaEvent, RefusalDoneEvent, ResponseIncompleteEvent, @@ -5229,3 +5234,87 @@ class TestRefusalPartsReachTextGuardrails: ) assert (events[0].delta, events[1].refusal) == (_MASKED_REFUSAL, _MASKED_REFUSAL) + + +class _PersimmonTextMaskingGuardrail(CustomGuardrail): + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional[LiteLLMLoggingObj] = None, + ) -> GenericGuardrailAPIInputs: + return {**inputs, "texts": [text.replace("persimmon", "[MASKED]") for text in inputs.get("texts", [])]} + + +_PERSIMMON_LOGPROBS: Final = ( + {"token": "ripe", "logprob": -0.1, "bytes": None, "top_logprobs": []}, + {"token": " persimmon", "logprob": -0.2, "bytes": None, "top_logprobs": []}, +) + + +@pytest.mark.asyncio +async def test_masked_output_text_drops_the_logprobs_that_spell_the_original_text() -> None: + response = ResponsesAPIResponse.model_validate( + { + "id": "resp_1", + "created_at": 1, + "model": "gpt-5.6", + "object": "response", + "status": "completed", + "output": [ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [ + {"type": "output_text", "text": "ripe persimmon", "annotations": [], "logprobs": list(_PERSIMMON_LOGPROBS)} + ], + } + ], + } + ) + + result = await OpenAIResponsesHandler().process_output_response( + response, _PersimmonTextMaskingGuardrail(guardrail_name="mask") + ) + + part = stream_item_items(result.output[0], "content")[0] + assert (stream_item_field(part, "text"), stream_item_field(part, "logprobs")) == ("ripe [MASKED]", None) + + +def test_stream_rewrite_drops_logprobs_from_the_delta_done_and_part_events() -> None: + delta = OutputTextDeltaEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, + item_id="msg_1", + output_index=0, + content_index=0, + delta="ripe persimmon", + logprobs=_PERSIMMON_LOGPROBS, + ) + done = OutputTextDoneEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE, + item_id="msg_1", + output_index=0, + content_index=0, + text="ripe persimmon", + logprobs=_PERSIMMON_LOGPROBS, + ) + part_done = ContentPartDoneEvent( + type=ResponsesAPIStreamEvents.CONTENT_PART_DONE, + item_id="msg_1", + output_index=0, + content_index=0, + part=ContentPartDonePartOutputText( + type="output_text", text="ripe persimmon", annotations=[], logprobs=_PERSIMMON_LOGPROBS + ), + ) + + OpenAIResponsesHandler()._sync_stream_events_with_rewrites( + stream_events=(delta, done, part_done), rewrites_by_position={(0, 0): "ripe [MASKED]"} + ) + + assert (delta.delta, delta.logprobs) == ("ripe [MASKED]", None) + assert (done.text, done.logprobs) == ("ripe [MASKED]", None) + assert (part_done.part.text, part_done.part.logprobs) == ("ripe [MASKED]", None) diff --git a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index fd7a91d12f6..c0fcf509456 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -2023,6 +2023,8 @@ def test_streamed_chat_logprobs_reach_the_text_delta_and_done_events() -> None: assert done["logprobs"] == [hello, period] assert part_done["part"]["logprobs"] == [hello, period] assert item_done["item"]["content"][0]["logprobs"] == [hello, period] + completed: Final = next(event for event in events if event["type"] == "response.completed") + assert completed["response"]["output"][0]["content"][0]["logprobs"] == [hello, period] def test_streamed_chat_without_logprobs_emits_text_events_without_a_logprobs_key() -> None: From ac99d5e2d58089aa7abc2a53fa1e093b566e9404 Mon Sep 17 00:00:00 2001 From: mateo Date: Sun, 1 Nov 2026 00:24:53 +0000 Subject: [PATCH 10/15] chore(responses): drop the mutable-ok marker that suppresses nothing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_completion_transformation/streaming_iterator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index bf137629553..00a757d4d88 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -219,7 +219,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._reasoning_summary_text: str = "" self._text_logprob_chunks: list[ tuple[OutputTextLogprob, ...] - ] = [] # mutable-ok: appended per chunk, flattened once at done + ] = [] # -- GENERIC RESPONSE-EVENTS PENDING QUEUE as required by fix -- self._pending_response_events: list[BaseLiteLLMOpenAIResponseObject] = [] self._reasoning_active = False From e652f70debc848a94339322e63f52a4deb92bdf8 Mon Sep 17 00:00:00 2001 From: mateo Date: Sun, 1 Nov 2026 00:27:51 +0000 Subject: [PATCH 11/15] style(responses): ruff format streaming_iterator Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_completion_transformation/streaming_iterator.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 00a757d4d88..4ecc8f5ffac 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -217,9 +217,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._sent_reasoning_summary_text_done_event: bool = False self._sent_reasoning_summary_part_done_event: bool = False self._reasoning_summary_text: str = "" - self._text_logprob_chunks: list[ - tuple[OutputTextLogprob, ...] - ] = [] + self._text_logprob_chunks: list[tuple[OutputTextLogprob, ...]] = [] # -- GENERIC RESPONSE-EVENTS PENDING QUEUE as required by fix -- self._pending_response_events: list[BaseLiteLLMOpenAIResponseObject] = [] self._reasoning_active = False From d9b67d8f52a34d502bd3e6e1db82ea5779ee61f9 Mon Sep 17 00:00:00 2001 From: mateo Date: Sun, 1 Nov 2026 00:32:40 +0000 Subject: [PATCH 12/15] fix(responses): keep logprobs on unchanged guardrail text, off reasoning items, and in the OpenAPI schema Second Greptile round on the bridge logprobs change. A guardrail that writes the text back unchanged keeps the logprobs, only a real rewrite clears them. The response.completed snapshot attaches the streamed answer tokens to the message item only, so a reasoning item built from reasoning_content carries none. The wrap model_serializer has no return annotation anymore, since pydantic used it as the serialization schema and collapsed OutputText and the output_text events to a bare object in the OpenAPI snapshot and schema.d.ts, which are regenerated here. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../guardrail_translation/handler.py | 3 +- litellm/proxy/_lazy_openapi_snapshot.json | 59 ++++++++++++++++ .../streaming_iterator.py | 2 +- litellm/types/llms/base.py | 2 +- ...test_openai_responses_guardrail_handler.py | 69 +++++++++++++++++-- .../test_streaming_iterator_transformation.py | 16 +++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 13 ++++ 7 files changed, 156 insertions(+), 8 deletions(-) diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 7db75645e88..8756466c176 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -1428,8 +1428,9 @@ class OpenAIResponsesHandler(BaseTranslation): @staticmethod def _write_rewritten_text(event: object, field: str, text: str) -> None: """A rewrite replaces the text, so the logprobs that spell out the original text go with it.""" + changed: Final = stream_item_field(event, field) != text OpenAIResponsesHandler._write_event_field(event, field, text) - if stream_item_field(event, "logprobs") is not None: + if changed and stream_item_field(event, "logprobs") is not None: OpenAIResponsesHandler._write_event_field(event, "logprobs", None) @staticmethod diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 5fd1f51cc0b..2748b15d57f 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -26474,6 +26474,65 @@ }, "OutputText": { "additionalProperties": true, + "description": "Text output content from an assistant message", + "properties": { + "annotations": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/GenericResponseOutputItemContentAnnotation" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Annotations" + }, + "logprobs": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/OutputTextLogprob" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Logprobs" + }, + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Text" + }, + "type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Type" + } + }, + "required": [ + "type", + "text", + "annotations" + ], + "title": "OutputText", "type": "object" }, "OutputTextLogprob": { diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 4ecc8f5ffac..06eae09ed3d 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -1597,7 +1597,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): def _with_streamed_logprobs(self, item: object) -> object: """stream_chunk_builder drops chat logprobs, so the snapshot's text part takes the ones the stream carried.""" logprobs: Final = self._streamed_text_logprobs() - if logprobs is None or not isinstance(item, GenericResponseOutputItem): + if logprobs is None or not isinstance(item, GenericResponseOutputItem) or item.type != "message": return item content: Final = tuple( part.model_copy(update={"logprobs": logprobs}) if isinstance(part, OutputText) else part diff --git a/litellm/types/llms/base.py b/litellm/types/llms/base.py index 231bfff2036..21aa1ec00e6 100644 --- a/litellm/types/llms/base.py +++ b/litellm/types/llms/base.py @@ -106,7 +106,7 @@ class CarriesOutputTextLogprobs(BaseLiteLLMOpenAIResponseObject): # frozen-ok: logprobs: Sequence[OutputTextLogprob] | None = None @model_serializer(mode="wrap") - def _without_absent_logprobs(self, handler: SerializerFunctionWrapHandler) -> Mapping[str, object]: + def _without_absent_logprobs(self, handler: SerializerFunctionWrapHandler): # noqa: ANN202 # a return annotation here replaces the model's OpenAPI schema with a bare object dumped: Final[Mapping[str, object]] = handler(self) if self.logprobs is not None: return dumped diff --git a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py index 5c8b147f842..0cfc4e690b2 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -5138,7 +5138,8 @@ class TestRefusalPartsReachTextGuardrails: assert guardrail.seen_texts == [[*leading_text, _REFUSAL]] assert [ - (part["type"], part.get("text") or part.get("refusal")) for part in result.model_dump()["output"][0]["content"] + (part["type"], part.get("text") or part.get("refusal")) + for part in result.model_dump()["output"][0]["content"] ] == expected_parts @pytest.mark.asyncio @@ -5161,7 +5162,8 @@ class TestRefusalPartsReachTextGuardrails: assert guardrail.seen_texts == [["Jane asked first.", _REFUSAL]] assert [ - (part["type"], part.get("text") or part.get("refusal")) for part in result.model_dump()["output"][0]["content"] + (part["type"], part.get("text") or part.get("refusal")) + for part in result.model_dump()["output"][0]["content"] ] == [("output_text", "[NAME] asked first."), ("refusal", _MASKED_REFUSAL)] @pytest.mark.asyncio @@ -5175,7 +5177,9 @@ class TestRefusalPartsReachTextGuardrails: part=ContentPartDonePartRefusal(type="refusal", refusal=_REFUSAL), **position, ), - OutputItemDoneEvent(type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, output_index=0, item=_refused_message()), + OutputItemDoneEvent( + type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, output_index=0, item=_refused_message() + ), ResponseIncompleteEvent(type=ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, response=_refused_response()), ] @@ -5242,7 +5246,7 @@ class _PersimmonTextMaskingGuardrail(CustomGuardrail): inputs: GenericGuardrailAPIInputs, request_data: dict, input_type: Literal["request", "response"], - logging_obj: Optional[LiteLLMLoggingObj] = None, + logging_obj: LiteLLMLoggingObj | None = None, ) -> GenericGuardrailAPIInputs: return {**inputs, "texts": [text.replace("persimmon", "[MASKED]") for text in inputs.get("texts", [])]} @@ -5269,7 +5273,12 @@ async def test_masked_output_text_drops_the_logprobs_that_spell_the_original_tex "status": "completed", "role": "assistant", "content": [ - {"type": "output_text", "text": "ripe persimmon", "annotations": [], "logprobs": list(_PERSIMMON_LOGPROBS)} + { + "type": "output_text", + "text": "ripe persimmon", + "annotations": [], + "logprobs": list(_PERSIMMON_LOGPROBS), + } ], } ], @@ -5284,6 +5293,56 @@ async def test_masked_output_text_drops_the_logprobs_that_spell_the_original_tex assert (stream_item_field(part, "text"), stream_item_field(part, "logprobs")) == ("ripe [MASKED]", None) +class _PassThroughGuardrail(CustomGuardrail): + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: LiteLLMLoggingObj | None = None, + ) -> GenericGuardrailAPIInputs: + return inputs + + +@pytest.mark.asyncio +async def test_a_passing_guardrail_keeps_the_output_text_logprobs() -> None: + response = ResponsesAPIResponse.model_validate( + { + "id": "resp_1", + "created_at": 1, + "model": "gpt-5.6", + "object": "response", + "status": "completed", + "output": [ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "ripe persimmon", + "annotations": [], + "logprobs": list(_PERSIMMON_LOGPROBS), + } + ], + } + ], + } + ) + + result = await OpenAIResponsesHandler().process_output_response( + response, _PassThroughGuardrail(guardrail_name="pass") + ) + + part = stream_item_items(result.output[0], "content")[0] + assert (stream_item_field(part, "text"), stream_item_field(part, "logprobs")) == ( + "ripe persimmon", + list(_PERSIMMON_LOGPROBS), + ) + + def test_stream_rewrite_drops_logprobs_from_the_delta_done_and_part_events() -> None: delta = OutputTextDeltaEvent( type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, diff --git a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index c0fcf509456..e2ef9ad5a43 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -2027,6 +2027,22 @@ def test_streamed_chat_logprobs_reach_the_text_delta_and_done_events() -> None: assert completed["response"]["output"][0]["content"][0]["logprobs"] == [hello, period] +def test_streamed_answer_logprobs_stay_off_the_reasoning_item_in_the_completed_snapshot() -> None: + events: Final = [ + event.model_dump(mode="json") + for event in _build_iterator( + [_reasoning_chunk("let me think"), _logprobs_chunk("Hello", -0.25, finish_reason="stop")] + ) + ] + hello: Final = {"token": "Hello", "logprob": -0.25, "bytes": None, "top_logprobs": []} + + completed: Final = next(event for event in events if event["type"] == "response.completed") + by_type: Final = {item["type"]: item for item in completed["response"]["output"]} + + assert by_type["message"]["content"][0]["logprobs"] == [hello] + assert all("logprobs" not in part for part in by_type["reasoning"]["content"]) + + def test_streamed_chat_without_logprobs_emits_text_events_without_a_logprobs_key() -> None: events: Final = [ event.model_dump(mode="json") for event in _build_iterator([_chunk("Hello"), _chunk("!", finish_reason="stop")]) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 34b0c17f53c..14bd897f9e0 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -40723,7 +40723,20 @@ export interface components { } & { [key: string]: unknown; }; + /** + * OutputText + * @description Text output content from an assistant message + */ OutputText: { + /** Annotations */ + annotations: components["schemas"]["GenericResponseOutputItemContentAnnotation"][] | null; + /** Logprobs */ + logprobs?: components["schemas"]["OutputTextLogprob"][] | null; + /** Text */ + text: string | null; + /** Type */ + type: string | null; + } & { [key: string]: unknown; }; /** OutputTextLogprob */ From 968600031a7e84b64714bf3e725adf97be5923f7 Mon Sep 17 00:00:00 2001 From: mateo Date: Mon, 2 Nov 2026 19:54:32 +0000 Subject: [PATCH 13/15] fix(responses): enable chat logprobs when include names message.output_text.logprobs Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../transformation.py | 13 ++++++++++++- .../test_responses_sampling_vertex_ai.py | 17 ++++++++++++++++- .../test_litellm_completion_responses.py | 11 +++++++++++ 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 2a2f9fb0f41..fe24d45123c 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -924,11 +924,22 @@ class ChatCompletionSession(TypedDict, total=False): ########### End of Initialize Classes used for Responses API ########### +OUTPUT_TEXT_LOGPROBS_INCLUDE: Final = "message.output_text.logprobs" + + def _requested_top_logprobs(request_fields: Mapping[str, object]) -> int | None: value: Final = request_fields.get("top_logprobs") return value if isinstance(value, int) else None +def _requests_output_text_logprobs(request_fields: Mapping[str, object]) -> bool: + """``top_logprobs`` or ``include: [message.output_text.logprobs]`` each ask the chat route for logprobs.""" + include: Final = request_fields.get("include") + return _requested_top_logprobs(request_fields) is not None or ( + isinstance(include, (list, tuple)) and OUTPUT_TEXT_LOGPROBS_INCLUDE in include + ) + + def _chain_session_metadata(chain_session: str, request_kwargs: Mapping[str, object]) -> Mapping[str, object]: key: Final = "litellm_metadata" if request_kwargs.get("litellm_metadata") else "metadata" metadata: Final = request_kwargs.get(key) @@ -1299,7 +1310,7 @@ class LiteLLMCompletionResponsesConfig: "tools": tools, "top_p": responses_api_request.get("top_p"), "top_logprobs": top_logprobs, - "logprobs": True if top_logprobs is not None else None, + "logprobs": True if _requests_output_text_logprobs(responses_api_request) else None, "user": responses_api_request.get("user"), "temperature": responses_api_request.get("temperature"), "parallel_tool_calls": responses_api_request.get("parallel_tool_calls"), diff --git a/tests/integration/translation/responses/sampling/test_responses_sampling_vertex_ai.py b/tests/integration/translation/responses/sampling/test_responses_sampling_vertex_ai.py index a89e8a867a7..6fadebbe438 100644 --- a/tests/integration/translation/responses/sampling/test_responses_sampling_vertex_ai.py +++ b/tests/integration/translation/responses/sampling/test_responses_sampling_vertex_ai.py @@ -108,6 +108,21 @@ GEMINI_2_5_FLASH_TOP_LOGPROBS_TEST_CASE: Final = replace( ) -@pytest.mark.parametrize("case", [GEMINI_2_5_FLASH_TOP_LOGPROBS_TEST_CASE], ids=lambda case: case.id) +GEMINI_2_5_FLASH_INCLUDE_LOGPROBS_TEST_CASE: Final = replace( + GEMINI_2_5_FLASH_TOP_LOGPROBS_TEST_CASE, + scenario="include_output_text_logprobs", + litellm_request={**GEMINI_2_5_FLASH_TEST_CASE.litellm_request, "include": ["message.output_text.logprobs"]}, + expected_provider_request={ + **GEMINI_2_5_FLASH_TEST_CASE.expected_provider_request, + "generationConfig": {**GENERATION_CONFIG, "responseLogprobs": True}, + }, +) + + +@pytest.mark.parametrize( + "case", + [GEMINI_2_5_FLASH_TOP_LOGPROBS_TEST_CASE, GEMINI_2_5_FLASH_INCLUDE_LOGPROBS_TEST_CASE], + ids=lambda case: case.id, +) def test_responses_sampling_vertex_ai(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: assert_translation(case, gateway, provider) diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index f9dba9bf994..1a1b89f5b41 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -7489,6 +7489,17 @@ def test_responses_top_logprobs_bridges_to_chat_logprobs() -> None: assert (request["top_logprobs"], request["logprobs"]) == (2, True) +def test_responses_include_output_text_logprobs_bridges_to_chat_logprobs() -> None: + request: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model="gemini-3.5-flash", + input="Say hello.", + responses_api_request={"include": ["message.output_text.logprobs"]}, + custom_llm_provider="vertex_ai", + ) + + assert request["logprobs"] is True and "top_logprobs" not in request + + def test_responses_without_top_logprobs_bridges_without_chat_logprobs() -> None: request: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( model="gemini-3.5-flash", input="Say hello.", responses_api_request={}, custom_llm_provider="vertex_ai" From 1c7b6a3f080ed04dc86afe16b26e833de9ec3c79 Mon Sep 17 00:00:00 2001 From: mateo Date: Mon, 2 Nov 2026 20:02:37 +0000 Subject: [PATCH 14/15] test(responses): declare the guardrail logprob test locals Final Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_openai_responses_guardrail_handler.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py index 5e37e025e4b..e67d4a3b6de 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -5287,7 +5287,7 @@ _PERSIMMON_LOGPROBS: Final = ( @pytest.mark.asyncio async def test_masked_output_text_drops_the_logprobs_that_spell_the_original_text() -> None: - response = ResponsesAPIResponse.model_validate( + response: Final = ResponsesAPIResponse.model_validate( { "id": "resp_1", "created_at": 1, @@ -5313,11 +5313,11 @@ async def test_masked_output_text_drops_the_logprobs_that_spell_the_original_tex } ) - result = await OpenAIResponsesHandler().process_output_response( + result: Final = await OpenAIResponsesHandler().process_output_response( response, _PersimmonTextMaskingGuardrail(guardrail_name="mask") ) - part = stream_item_items(result.output[0], "content")[0] + part: Final = stream_item_items(result.output[0], "content")[0] assert (stream_item_field(part, "text"), stream_item_field(part, "logprobs")) == ("ripe [MASKED]", None) @@ -5334,7 +5334,7 @@ class _PassThroughGuardrail(CustomGuardrail): @pytest.mark.asyncio async def test_a_passing_guardrail_keeps_the_output_text_logprobs() -> None: - response = ResponsesAPIResponse.model_validate( + response: Final = ResponsesAPIResponse.model_validate( { "id": "resp_1", "created_at": 1, @@ -5360,11 +5360,11 @@ async def test_a_passing_guardrail_keeps_the_output_text_logprobs() -> None: } ) - result = await OpenAIResponsesHandler().process_output_response( + result: Final = await OpenAIResponsesHandler().process_output_response( response, _PassThroughGuardrail(guardrail_name="pass") ) - part = stream_item_items(result.output[0], "content")[0] + part: Final = stream_item_items(result.output[0], "content")[0] assert (stream_item_field(part, "text"), stream_item_field(part, "logprobs")) == ( "ripe persimmon", list(_PERSIMMON_LOGPROBS), @@ -5372,7 +5372,7 @@ async def test_a_passing_guardrail_keeps_the_output_text_logprobs() -> None: def test_stream_rewrite_drops_logprobs_from_the_delta_done_and_part_events() -> None: - delta = OutputTextDeltaEvent( + delta: Final = OutputTextDeltaEvent( type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, item_id="msg_1", output_index=0, @@ -5380,7 +5380,7 @@ def test_stream_rewrite_drops_logprobs_from_the_delta_done_and_part_events() -> delta="ripe persimmon", logprobs=_PERSIMMON_LOGPROBS, ) - done = OutputTextDoneEvent( + done: Final = OutputTextDoneEvent( type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE, item_id="msg_1", output_index=0, @@ -5388,7 +5388,7 @@ def test_stream_rewrite_drops_logprobs_from_the_delta_done_and_part_events() -> text="ripe persimmon", logprobs=_PERSIMMON_LOGPROBS, ) - part_done = ContentPartDoneEvent( + part_done: Final = ContentPartDoneEvent( type=ResponsesAPIStreamEvents.CONTENT_PART_DONE, item_id="msg_1", output_index=0, From 1bc94cb97269aeaa7611441a83bc67e3f694744d Mon Sep 17 00:00:00 2001 From: mateo Date: Mon, 2 Nov 2026 20:08:11 +0000 Subject: [PATCH 15/15] refactor(responses): drop docstrings that restate the logprob helpers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../responses/litellm_completion_transformation/logprobs.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/logprobs.py b/litellm/responses/litellm_completion_transformation/logprobs.py index 255667ae5a4..f0082bf48a9 100644 --- a/litellm/responses/litellm_completion_transformation/logprobs.py +++ b/litellm/responses/litellm_completion_transformation/logprobs.py @@ -1,5 +1,3 @@ -"""Chat ``choice.logprobs`` as Responses ``output_text.logprobs``, which share one token shape.""" - from collections.abc import Mapping from typing import Final @@ -34,7 +32,6 @@ def _choice_logprobs(choice_logprobs: object) -> ChoiceLogprobs | None: def output_text_logprobs(choice_logprobs: object) -> tuple[OutputTextLogprob, ...] | None: - """The ``logprobs`` a Responses text part carries for a chat choice, ``None`` when the choice sent none.""" validated: Final = _choice_logprobs(choice_logprobs) if validated is None or validated.content is None: return None