diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index a18cbe13925..b9ca4757bd1 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -1425,11 +1425,19 @@ class OpenAIResponsesHandler(BaseTranslation): else: setattr(event, field, value) + @staticmethod + def _write_rewritten_text(event: object, field: str, text: str) -> None: + """A rewrite replaces the text, so the logprobs that spell out the original text go with it.""" + changed: Final = stream_item_field(event, field) != text + OpenAIResponsesHandler._write_event_field(event, field, text) + if changed and stream_item_field(event, "logprobs") is not None: + OpenAIResponsesHandler._write_event_field(event, "logprobs", None) + @staticmethod def _write_content_part_text(part: object, text: str) -> None: field: Final = _content_part_text_field(part) if _has_writable_field(part, field): - OpenAIResponsesHandler._write_event_field(part, field, text) + OpenAIResponsesHandler._write_rewritten_text(part, field, text) @staticmethod def _write_compaction_summary(item: object, summary: str) -> None: @@ -1468,9 +1476,9 @@ class OpenAIResponsesHandler(BaseTranslation): continue position = (output_index, content_index) if event_type in _TEXT_DELTA_EVENT_TYPES and position in delta_replacements: - self._write_event_field(event, "delta", next(delta_replacements[position])) + self._write_rewritten_text(event, "delta", next(delta_replacements[position])) elif event_type == "response.output_text.done" and position in rewrites_by_position: - self._write_event_field(event, "text", rewrites_by_position[position]) + self._write_rewritten_text(event, "text", rewrites_by_position[position]) elif event_type == "response.refusal.done" and position in rewrites_by_position: self._write_event_field(event, "refusal", rewrites_by_position[position]) elif event_type == "response.content_part.done" and position in rewrites_by_position: diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 5cb63c98d56..7aa9c056a91 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -27136,6 +27136,20 @@ ], "title": "Annotations" }, + "logprobs": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/OutputTextLogprob" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Logprobs" + }, "text": { "anyOf": [ { @@ -27167,6 +27181,80 @@ "title": "OutputText", "type": "object" }, + "OutputTextLogprob": { + "additionalProperties": true, + "properties": { + "bytes": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Bytes" + }, + "logprob": { + "title": "Logprob", + "type": "number" + }, + "token": { + "title": "Token", + "type": "string" + }, + "top_logprobs": { + "items": { + "$ref": "#/components/schemas/OutputTextTopLogprob" + }, + "title": "Top Logprobs", + "type": "array" + } + }, + "required": [ + "token", + "logprob", + "top_logprobs" + ], + "title": "OutputTextLogprob", + "type": "object" + }, + "OutputTextTopLogprob": { + "additionalProperties": true, + "properties": { + "bytes": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Bytes" + }, + "logprob": { + "title": "Logprob", + "type": "number" + }, + "token": { + "title": "Token", + "type": "string" + } + }, + "required": [ + "token", + "logprob" + ], + "title": "OutputTextTopLogprob", + "type": "object" + }, "OutputTokensDetails": { "additionalProperties": true, "properties": { diff --git a/litellm/responses/litellm_completion_transformation/logprobs.py b/litellm/responses/litellm_completion_transformation/logprobs.py new file mode 100644 index 00000000000..f0082bf48a9 --- /dev/null +++ b/litellm/responses/litellm_completion_transformation/logprobs.py @@ -0,0 +1,38 @@ +from collections.abc import Mapping +from typing import Final + +from pydantic import ValidationError + +from litellm.types.llms.base import OutputTextLogprob, OutputTextTopLogprob +from litellm.types.utils import ChatCompletionTokenLogprob, ChoiceLogprobs, TopLogprob + + +def _top_logprob(top: TopLogprob) -> OutputTextTopLogprob: + return {"token": top.token, "logprob": top.logprob, "bytes": top.bytes} + + +def _token_logprob(token: ChatCompletionTokenLogprob) -> OutputTextLogprob: + return { + "token": token.token, + "logprob": token.logprob, + "bytes": token.bytes, + "top_logprobs": tuple(_top_logprob(top) for top in token.top_logprobs), + } + + +def _choice_logprobs(choice_logprobs: object) -> ChoiceLogprobs | None: + if isinstance(choice_logprobs, ChoiceLogprobs): + return choice_logprobs + if not isinstance(choice_logprobs, Mapping): + return None + try: + return ChoiceLogprobs.model_validate(choice_logprobs) + except ValidationError: + return None + + +def output_text_logprobs(choice_logprobs: object) -> tuple[OutputTextLogprob, ...] | None: + validated: Final = _choice_logprobs(choice_logprobs) + if validated is None or validated.content is None: + return None + return tuple(_token_logprob(token) for token in validated.content) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 521a698b276..0eac415c29d 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -1,7 +1,7 @@ import time import uuid from collections.abc import Mapping, Sequence -from itertools import filterfalse +from itertools import chain, filterfalse from types import MappingProxyType from typing import Any, Final, cast @@ -48,6 +48,7 @@ from litellm.types.llms.openai import ( OutputTextAnnotationAddedEvent, OutputTextDeltaEvent, OutputTextDoneEvent, + OutputTextLogprob, ReasoningSummaryPartDoneEvent, ReasoningSummaryTextDeltaEvent, ReasoningSummaryTextDoneEvent, @@ -66,7 +67,11 @@ from litellm.types.llms.openai import ( WebSearchCallInProgressEvent, WebSearchCallSearchingEvent, ) -from litellm.types.responses.main import GenericResponseOutputItemContentAnnotation +from litellm.types.responses.main import ( + GenericResponseOutputItem, + GenericResponseOutputItemContentAnnotation, + OutputText, +) from litellm.types.utils import Delta as ChatCompletionDelta from litellm.types.utils import ( ModelResponse, @@ -75,6 +80,8 @@ from litellm.types.utils import ( TextCompletionResponse, ) +from .logprobs import output_text_logprobs + def _index_of_output_item_type(items: Sequence[object], item_type: str) -> int | None: return next( @@ -214,6 +221,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._sent_reasoning_summary_text_done_event: bool = False self._sent_reasoning_summary_part_done_event: bool = False self._reasoning_summary_text: str = "" + self._text_logprob_chunks: list[tuple[OutputTextLogprob, ...]] = [] # -- GENERIC RESPONSE-EVENTS PENDING QUEUE as required by fix -- self._pending_response_events: list[BaseLiteLLMOpenAIResponseObject] = [] self._reasoning_active = False @@ -974,8 +982,20 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): output_index=self._message_index(), content_index=0, text=getattr(litellm_complete_object.choices[0].message, "content", "") or "", + logprobs=self._final_text_logprobs(litellm_complete_object), ) + def _streamed_text_logprobs(self) -> tuple[OutputTextLogprob, ...] | None: + if not self._text_logprob_chunks: + return None + return tuple(chain.from_iterable(self._text_logprob_chunks)) + + def _final_text_logprobs(self, litellm_complete_object: ModelResponse) -> tuple[OutputTextLogprob, ...] | None: + streamed: Final = self._streamed_text_logprobs() + if streamed is not None: + return streamed + return output_text_logprobs(getattr(litellm_complete_object.choices[0], "logprobs", None)) + def create_output_content_part_done_event(self, litellm_complete_object: ModelResponse) -> ContentPartDoneEvent: if self._cached_item_id is None: self._cached_item_id = f"msg_{uuid.uuid4()}" @@ -992,7 +1012,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): type="output_text", text=text, annotations=response_annotations, - logprobs=None, + logprobs=self._final_text_logprobs(litellm_complete_object), ) return ContentPartDoneEvent( @@ -1016,8 +1036,16 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): annotations=annotations ) ) + logprobs: Final = self._final_text_logprobs(litellm_complete_object) text_parts: Final = ( - ({"type": "output_text", "text": text, "annotations": response_annotations},) + ( + { + "type": "output_text", + "text": text, + "annotations": response_annotations, + **({} if logprobs is None else {"logprobs": logprobs}), + }, + ) if self.sent_content_part_added_event else () ) @@ -1468,12 +1496,16 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if not self.sent_content_part_added_event: self._queue_message_item_added_events() self._hold_tool_calls_behind_text(getattr(chunk.choices[0].delta, "tool_calls", None)) + delta_logprobs: Final = output_text_logprobs(getattr(chunk.choices[0], "logprobs", None)) + if delta_logprobs is not None: + self._text_logprob_chunks.append(delta_logprobs) return OutputTextDeltaEvent( type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, item_id=item_id, output_index=self._message_index(), content_index=0, delta=delta_content, + logprobs=delta_logprobs, ) # Priority 3: Handle tool call deltas (if any) -> queue events and emit them @@ -1521,7 +1553,18 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): ) compaction_aligned: Final = _output_items_with_id(reasoning_aligned, "compaction", self._compaction_item_id) streamed_items: Final = filterfalse(self._is_unstreamed_empty_message, compaction_aligned) - return tuple(sorted(streamed_items, key=self._streamed_output_position)) + return tuple(map(self._with_streamed_logprobs, sorted(streamed_items, key=self._streamed_output_position))) + + def _with_streamed_logprobs(self, item: object) -> object: + """stream_chunk_builder drops chat logprobs, so the snapshot's text part takes the ones the stream carried.""" + logprobs: Final = self._streamed_text_logprobs() + if logprobs is None or not isinstance(item, GenericResponseOutputItem) or item.type != "message": + return item + content: Final = tuple( + part.model_copy(update={"logprobs": logprobs}) if isinstance(part, OutputText) else part + for part in item.content + ) + return item.model_copy(update={"content": list(content)}) def _is_unstreamed_empty_message(self, item: object) -> bool: if getattr(item, "type", None) != "message" or self._message_output_index is not None: diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 3e605e926f7..50a2ef4fe7d 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -112,6 +112,7 @@ from litellm.types.llms.openai import ( OpenAIMcpServerTool, OpenAIWebSearchOptions, OpenAIWebSearchUserLocation, + OutputTextLogprob, OutputTokensDetails, Reasoning, ResponseAPIUsage, @@ -168,6 +169,7 @@ from .custom_tools import ( validated_tool_search_execution, without_malformed_tool_flags, ) +from .logprobs import output_text_logprobs from .reasoning_items import decode_thinking_blocks, encode_thinking_blocks, mint_reasoning_item_id from .tool_search import ( CompletionRoute, @@ -925,6 +927,22 @@ class ChatCompletionSession(TypedDict, total=False): ########### End of Initialize Classes used for Responses API ########### +OUTPUT_TEXT_LOGPROBS_INCLUDE: Final = "message.output_text.logprobs" + + +def _requested_top_logprobs(request_fields: Mapping[str, object]) -> int | None: + value: Final = request_fields.get("top_logprobs") + return value if isinstance(value, int) else None + + +def _requests_output_text_logprobs(request_fields: Mapping[str, object]) -> bool: + """``top_logprobs`` or ``include: [message.output_text.logprobs]`` each ask the chat route for logprobs.""" + include: Final = request_fields.get("include") + return _requested_top_logprobs(request_fields) is not None or ( + isinstance(include, (list, tuple)) and OUTPUT_TEXT_LOGPROBS_INCLUDE in include + ) + + def _chain_session_metadata(chain_session: str, request_kwargs: Mapping[str, object]) -> Mapping[str, object]: key: Final = "litellm_metadata" if request_kwargs.get("litellm_metadata") else "metadata" metadata: Final = request_kwargs.get(key) @@ -953,6 +971,7 @@ class LiteLLMCompletionResponsesConfig: "text", "tool_choice", "tools", + "top_logprobs", "top_p", "user", ] @@ -1278,6 +1297,7 @@ class LiteLLMCompletionResponsesConfig: if malformed_field_error is not None: raise malformed_field_error + top_logprobs: Final = _requested_top_logprobs(responses_api_request) litellm_completion_request: dict = { "messages": ( list( @@ -1295,6 +1315,8 @@ class LiteLLMCompletionResponsesConfig: ), "tools": tools, "top_p": responses_api_request.get("top_p"), + "top_logprobs": top_logprobs, + "logprobs": True if _requests_output_text_logprobs(responses_api_request) else None, "user": responses_api_request.get("user"), "temperature": responses_api_request.get("temperature"), "parallel_tool_calls": responses_api_request.get("parallel_tool_calls"), @@ -3704,7 +3726,9 @@ class LiteLLMCompletionResponsesConfig: choice=choice, ) message_output_items.extend(image_generation_items) - elif content := LiteLLMCompletionResponsesConfig._message_output_content(choice.message): + elif content := LiteLLMCompletionResponsesConfig._message_output_content( + choice.message, output_text_logprobs(getattr(choice, "logprobs", None)) + ): message_output_items.append( GenericResponseOutputItem( type="message", @@ -3717,7 +3741,9 @@ class LiteLLMCompletionResponsesConfig: return message_output_items @staticmethod - def _message_output_content(message: Message) -> tuple[OutputText | ResponseOutputRefusal, ...]: + def _message_output_content( + message: Message, logprobs: Sequence[OutputTextLogprob] | None = None + ) -> tuple[OutputText | ResponseOutputRefusal, ...]: refusal: Final = openai_chat_refusal_text(message) refusal_parts: Final = (ResponseOutputRefusal(type="refusal", refusal=refusal),) if refusal is not None else () has_text_part: Final = ( @@ -3726,7 +3752,7 @@ class LiteLLMCompletionResponsesConfig: or (message.content is not None and not refusal_parts) ) text_parts: Final = ( - (LiteLLMCompletionResponsesConfig._transform_chat_message_to_response_output_text(message),) + (LiteLLMCompletionResponsesConfig._transform_chat_message_to_response_output_text(message, logprobs),) if has_text_part else () ) @@ -3778,6 +3804,7 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def _transform_chat_message_to_response_output_text( message: Message, + logprobs: Sequence[OutputTextLogprob] | None = None, ) -> OutputText: annotations: Final = getattr(message, "annotations", None) transformed_annotations: Final = ( @@ -3790,6 +3817,7 @@ class LiteLLMCompletionResponsesConfig: type="output_text", text=message.content or "", annotations=transformed_annotations, + logprobs=logprobs, ) @staticmethod diff --git a/litellm/types/llms/base.py b/litellm/types/llms/base.py index 5a2da071cec..21aa1ec00e6 100644 --- a/litellm/types/llms/base.py +++ b/litellm/types/llms/base.py @@ -1,7 +1,9 @@ +from collections.abc import Mapping, Sequence from typing import Final from openai._models import BaseModel as OpenAIObject -from pydantic import ConfigDict +from pydantic import ConfigDict, SerializerFunctionWrapHandler, model_serializer +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.types.llms.base_model import LiteLLMBaseModel @@ -83,3 +85,29 @@ class CachedTokensDetails(LiteLLMBaseModel): text_tokens: int | None = None audio_tokens: int | None = None image_tokens: int | None = None + + +class OutputTextTopLogprob(TypedDict): + token: ReadOnly[str] + logprob: ReadOnly[float] + bytes: NotRequired[ReadOnly[Sequence[int] | None]] + + +class OutputTextLogprob(TypedDict): + token: ReadOnly[str] + logprob: ReadOnly[float] + bytes: NotRequired[ReadOnly[Sequence[int] | None]] + top_logprobs: ReadOnly[Sequence[OutputTextTopLogprob]] + + +class CarriesOutputTextLogprobs(BaseLiteLLMOpenAIResponseObject): # frozen-ok: guardrails rewrite these in place + """A Responses text payload carries ``logprobs`` only when the chat choice behind it sent them.""" + + logprobs: Sequence[OutputTextLogprob] | None = None + + @model_serializer(mode="wrap") + def _without_absent_logprobs(self, handler: SerializerFunctionWrapHandler): # noqa: ANN202 # a return annotation here replaces the model's OpenAPI schema with a bare object + dumped: Final[Mapping[str, object]] = handler(self) + if self.logprobs is not None: + return dumped + return {key: value for key, value in dumped.items() if key != "logprobs"} diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index c9739eee91d..94c86a10eba 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -87,7 +87,12 @@ from typing_extensions import ( ) from litellm._logging import verbose_logger -from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject, LiteLLMBaseModel +from litellm.types.llms.base import ( + BaseLiteLLMOpenAIResponseObject, + CarriesOutputTextLogprobs, + LiteLLMBaseModel, + OutputTextLogprob, +) from litellm.types.responses.main import ( CustomToolCallOutputItem, GenericResponseOutputItem, @@ -1900,7 +1905,7 @@ class ContentPartDonePartOutputText(BaseLiteLLMOpenAIResponseObject): type: Literal["output_text"] text: str annotations: list[BaseLiteLLMOpenAIResponseObject] - logprobs: list[OpenAIChatCompletionLogprobsContent] | None = None + logprobs: Sequence[OutputTextLogprob] | None = None class ContentPartDonePartRefusal(BaseLiteLLMOpenAIResponseObject): @@ -1924,7 +1929,7 @@ class ContentPartDoneEvent(BaseResponsesStreamEvent): part: PART_UNION_TYPES -class OutputTextDeltaEvent(BaseResponsesStreamEvent): +class OutputTextDeltaEvent(CarriesOutputTextLogprobs, BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA] item_id: str output_index: int @@ -1941,7 +1946,7 @@ class OutputTextAnnotationAddedEvent(BaseResponsesStreamEvent): annotation: dict -class OutputTextDoneEvent(BaseResponsesStreamEvent): +class OutputTextDoneEvent(CarriesOutputTextLogprobs, BaseResponsesStreamEvent): type: Literal[ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE] item_id: str output_index: int diff --git a/litellm/types/responses/main.py b/litellm/types/responses/main.py index 86dc857cfd0..a8c8410e723 100644 --- a/litellm/types/responses/main.py +++ b/litellm/types/responses/main.py @@ -8,7 +8,7 @@ from openai.types.responses.response_output_refusal import ResponseOutputRefusal from pydantic import ConfigDict, PrivateAttr from typing_extensions import TypedDict -from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject +from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject, CarriesOutputTextLogprobs Phase = Literal["commentary", "final_answer"] | None @@ -23,7 +23,7 @@ class GenericResponseOutputItemContentAnnotation(BaseLiteLLMOpenAIResponseObject title: str | None -class OutputText(BaseLiteLLMOpenAIResponseObject): +class OutputText(CarriesOutputTextLogprobs): """Text output content from an assistant message""" type: str | None # "output_text" diff --git a/tests/integration/proxy_config.yaml b/tests/integration/proxy_config.yaml index c552e11c22a..59742063a07 100644 --- a/tests/integration/proxy_config.yaml +++ b/tests/integration/proxy_config.yaml @@ -501,6 +501,13 @@ model_list: vertex_project: synthetic-vertex-project vertex_location: global vertex_credentials: '{"type": "external_account", "audience": "synthetic-vertex-audience", "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", "token_url": "http://127.0.0.1:8190/_oauth/token", "credential_source": {"url": "http://127.0.0.1:8190/health"}}' + - model_name: vertex_ai/gemini-2.5-flash + litellm_params: + model: vertex_ai/gemini-2.5-flash + api_base: http://127.0.0.1:8191 + vertex_project: synthetic-vertex-project + vertex_location: global + vertex_credentials: '{"type": "external_account", "audience": "synthetic-vertex-audience", "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", "token_url": "http://127.0.0.1:8190/_oauth/token", "credential_source": {"url": "http://127.0.0.1:8190/health"}}' - model_name: vertex_ai/gemini-3.8-flash litellm_params: model: vertex_ai/gemini-3.8-flash diff --git a/tests/integration/translation/responses/bases/vertex_ai.py b/tests/integration/translation/responses/bases/vertex_ai.py index 59637e380ce..3b31b995aa8 100644 --- a/tests/integration/translation/responses/bases/vertex_ai.py +++ b/tests/integration/translation/responses/bases/vertex_ai.py @@ -1,4 +1,5 @@ import json +from dataclasses import replace from typing import Final from unittest.mock import ANY @@ -1858,3 +1859,37 @@ GEMINI_4_ARGON_TEST_CASE: Final = TranslationTestCase( "provider_specific_fields": {"traffic_type": "ON_DEMAND"}, }, ) + + +GEMINI_2_5_FLASH_TEST_CASE: Final = replace( + GEMINI_3_5_FLASH_TEST_CASE, + litellm_request={**GEMINI_3_5_FLASH_TEST_CASE.litellm_request, "model": "vertex_ai/gemini-2.5-flash"}, + expected_provider_endpoint=( + "/v1/projects/synthetic-vertex-project/locations/global/publishers/google/models/gemini-2.5-flash:generateContent" + ), + mock_provider_response={ + "candidates": [{"content": {"role": "model", "parts": [{"text": "Hello."}]}, "finishReason": "STOP"}], + "usageMetadata": { + "promptTokenCount": 9, + "candidatesTokenCount": 2, + "totalTokenCount": 32, + "trafficType": "ON_DEMAND", + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 9}], + "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 2}], + "thoughtsTokenCount": 21, + }, + "modelVersion": "gemini-2.5-flash", + "createTime": "2026-10-31T23:26:23.954987Z", + "responseId": "H3nmauukOrv46tkPw5vggAo", + }, + expected_litellm_response={ + **GEMINI_3_5_FLASH_TEST_CASE.expected_litellm_response, + "model": "vertex_ai/gemini-2.5-flash", + "usage": { + **GEMINI_3_5_FLASH_TEST_CASE.expected_litellm_response["usage"], + "output_tokens": 23, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 21, "text_tokens": 2}, + "total_tokens": 32, + }, + }, +) diff --git a/tests/integration/translation/responses/basic/test_responses_basic_vertex_ai.py b/tests/integration/translation/responses/basic/test_responses_basic_vertex_ai.py index bd9118482d0..6d6e08835a3 100644 --- a/tests/integration/translation/responses/basic/test_responses_basic_vertex_ai.py +++ b/tests/integration/translation/responses/basic/test_responses_basic_vertex_ai.py @@ -11,6 +11,7 @@ from integration.translation.responses.bases.vertex_ai import ( CLAUDE_SONNET_4_6_TEST_CASE, CLAUDE_SONNET_5_5_TEST_CASE, CLAUDE_SONNET_5_TEST_CASE, + GEMINI_2_5_FLASH_TEST_CASE, GEMINI_3_1_PRO_PREVIEW_TEST_CASE, GEMINI_3_5_FLASH_TEST_CASE, GEMINI_3_8_FLASH_TEST_CASE, @@ -30,6 +31,7 @@ from integration.translation.runner import assert_translation CLAUDE_HAIKU_5_5_TEST_CASE, CLAUDE_OPUS_5_5_TEST_CASE, CLAUDE_FABLE_5_1_TEST_CASE, + GEMINI_2_5_FLASH_TEST_CASE, GEMINI_3_5_FLASH_TEST_CASE, GEMINI_3_8_FLASH_TEST_CASE, GEMINI_3_1_PRO_PREVIEW_TEST_CASE, diff --git a/tests/integration/translation/responses/sampling/test_responses_sampling_vertex_ai.py b/tests/integration/translation/responses/sampling/test_responses_sampling_vertex_ai.py new file mode 100644 index 00000000000..6fadebbe438 --- /dev/null +++ b/tests/integration/translation/responses/sampling/test_responses_sampling_vertex_ai.py @@ -0,0 +1,128 @@ +from dataclasses import replace +from typing import Final + +import pytest +from integration._support.client import Gateway, object_value +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.responses.bases.vertex_ai import GEMINI_2_5_FLASH_TEST_CASE +from integration.translation.runner import assert_translation + +GENERATION_CONFIG: Final = object_value(GEMINI_2_5_FLASH_TEST_CASE.expected_provider_request["generationConfig"]) +USAGE: Final = object_value(GEMINI_2_5_FLASH_TEST_CASE.expected_litellm_response["usage"]) + +GEMINI_2_5_FLASH_TOP_LOGPROBS_TEST_CASE: Final = replace( + GEMINI_2_5_FLASH_TEST_CASE, + scenario="top_logprobs", + litellm_request={**GEMINI_2_5_FLASH_TEST_CASE.litellm_request, "top_logprobs": 2}, + expected_provider_request={ + **GEMINI_2_5_FLASH_TEST_CASE.expected_provider_request, + "generationConfig": {**GENERATION_CONFIG, "responseLogprobs": True, "logprobs": 2}, + }, + mock_provider_response={ + "candidates": [ + { + "content": {"role": "model", "parts": [{"text": "Hello."}]}, + "finishReason": "STOP", + "avgLogprobs": -2.20536901451851e-06, + "logprobsResult": { + "topCandidates": [ + { + "candidates": [ + {"token": "Hello", "logProbability": -1.3113013e-06, "tokenId": 9259}, + {"token": "Hi", "logProbability": -15.149978, "tokenId": 10979}, + ] + }, + { + "candidates": [ + {"token": ".", "logProbability": -3.0994368e-06, "tokenId": 236761}, + {"token": "", "logProbability": -13.81299, "tokenId": 106}, + ] + }, + ], + "chosenCandidates": [ + {"token": "Hello", "logProbability": -1.3113013e-06, "tokenId": 9259}, + {"token": ".", "logProbability": -3.0994368e-06, "tokenId": 236761}, + ], + }, + } + ], + "usageMetadata": { + "promptTokenCount": 9, + "candidatesTokenCount": 2, + "totalTokenCount": 30, + "trafficType": "ON_DEMAND", + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 9}], + "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 2}], + "thoughtsTokenCount": 19, + }, + "modelVersion": "gemini-2.5-flash", + "createTime": "2026-10-31T23:20:20.755722Z", + "responseId": "tHfmaoqQLrKZ9LsPvuuB-AU", + }, + expected_litellm_response={ + **GEMINI_2_5_FLASH_TEST_CASE.expected_litellm_response, + "output": [ + { + "type": "message", + "id": GEMINI_2_5_FLASH_TEST_CASE.expected_litellm_response["output"][0]["id"], + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Hello.", + "annotations": [], + "logprobs": [ + { + "token": "Hello", + "logprob": -1.3113013e-06, + "bytes": None, + "top_logprobs": [ + {"token": "Hello", "logprob": -1.3113013e-06, "bytes": None}, + {"token": "Hi", "logprob": -15.149978, "bytes": None}, + ], + }, + { + "token": ".", + "logprob": -3.0994368e-06, + "bytes": None, + "top_logprobs": [ + {"token": ".", "logprob": -3.0994368e-06, "bytes": None}, + {"token": "", "logprob": -13.81299, "bytes": None}, + ], + }, + ], + } + ], + "phase": None, + } + ], + "usage": { + **USAGE, + "output_tokens": 21, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 19, "text_tokens": 2}, + "total_tokens": 30, + }, + }, +) + + +GEMINI_2_5_FLASH_INCLUDE_LOGPROBS_TEST_CASE: Final = replace( + GEMINI_2_5_FLASH_TOP_LOGPROBS_TEST_CASE, + scenario="include_output_text_logprobs", + litellm_request={**GEMINI_2_5_FLASH_TEST_CASE.litellm_request, "include": ["message.output_text.logprobs"]}, + expected_provider_request={ + **GEMINI_2_5_FLASH_TEST_CASE.expected_provider_request, + "generationConfig": {**GENERATION_CONFIG, "responseLogprobs": True}, + }, +) + + +@pytest.mark.parametrize( + "case", + [GEMINI_2_5_FLASH_TOP_LOGPROBS_TEST_CASE, GEMINI_2_5_FLASH_INCLUDE_LOGPROBS_TEST_CASE], + ids=lambda case: case.id, +) +def test_responses_sampling_vertex_ai(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py index d8954c4072a..e67d4a3b6de 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -36,6 +36,8 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.llms import get_guardrail_translation_mapping from litellm.llms.openai.responses.guardrail_translation.handler import ( OpenAIResponsesHandler, + stream_item_field, + stream_item_items, ) from litellm.llms.openai.responses.guardrail_translation.tool_merge import merge_guardrailed_tools from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import GenericGuardrailAPI @@ -46,8 +48,11 @@ from litellm.types.guardrails import ContentFilterAction, ContentFilterPattern from litellm.types.llms.openai import ( ChatCompletionToolCallChunk, ContentPartDoneEvent, + ContentPartDonePartOutputText, ContentPartDonePartRefusal, OutputItemDoneEvent, + OutputTextDeltaEvent, + OutputTextDoneEvent, RefusalDeltaEvent, RefusalDoneEvent, ResponseIncompleteEvent, @@ -5161,7 +5166,8 @@ class TestRefusalPartsReachTextGuardrails: assert guardrail.seen_texts == [[*leading_text, _REFUSAL]] assert [ - (part["type"], part.get("text") or part.get("refusal")) for part in result.model_dump()["output"][0]["content"] + (part["type"], part.get("text") or part.get("refusal")) + for part in result.model_dump()["output"][0]["content"] ] == expected_parts @pytest.mark.asyncio @@ -5184,7 +5190,8 @@ class TestRefusalPartsReachTextGuardrails: assert guardrail.seen_texts == [["Jane asked first.", _REFUSAL]] assert [ - (part["type"], part.get("text") or part.get("refusal")) for part in result.model_dump()["output"][0]["content"] + (part["type"], part.get("text") or part.get("refusal")) + for part in result.model_dump()["output"][0]["content"] ] == [("output_text", "[NAME] asked first."), ("refusal", _MASKED_REFUSAL)] @pytest.mark.asyncio @@ -5198,7 +5205,9 @@ class TestRefusalPartsReachTextGuardrails: part=ContentPartDonePartRefusal(type="refusal", refusal=_REFUSAL), **position, ), - OutputItemDoneEvent(type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, output_index=0, item=_refused_message()), + OutputItemDoneEvent( + type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, output_index=0, item=_refused_message() + ), ResponseIncompleteEvent(type=ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, response=_refused_response()), ] @@ -5257,3 +5266,142 @@ class TestRefusalPartsReachTextGuardrails: ) assert (events[0].delta, events[1].refusal) == (_MASKED_REFUSAL, _MASKED_REFUSAL) + + +class _PersimmonTextMaskingGuardrail(CustomGuardrail): + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: LiteLLMLoggingObj | None = None, + ) -> GenericGuardrailAPIInputs: + return {**inputs, "texts": [text.replace("persimmon", "[MASKED]") for text in inputs.get("texts", [])]} + + +_PERSIMMON_LOGPROBS: Final = ( + {"token": "ripe", "logprob": -0.1, "bytes": None, "top_logprobs": []}, + {"token": " persimmon", "logprob": -0.2, "bytes": None, "top_logprobs": []}, +) + + +@pytest.mark.asyncio +async def test_masked_output_text_drops_the_logprobs_that_spell_the_original_text() -> None: + response: Final = ResponsesAPIResponse.model_validate( + { + "id": "resp_1", + "created_at": 1, + "model": "gpt-5.6", + "object": "response", + "status": "completed", + "output": [ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "ripe persimmon", + "annotations": [], + "logprobs": list(_PERSIMMON_LOGPROBS), + } + ], + } + ], + } + ) + + result: Final = await OpenAIResponsesHandler().process_output_response( + response, _PersimmonTextMaskingGuardrail(guardrail_name="mask") + ) + + part: Final = stream_item_items(result.output[0], "content")[0] + assert (stream_item_field(part, "text"), stream_item_field(part, "logprobs")) == ("ripe [MASKED]", None) + + +class _PassThroughGuardrail(CustomGuardrail): + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: LiteLLMLoggingObj | None = None, + ) -> GenericGuardrailAPIInputs: + return inputs + + +@pytest.mark.asyncio +async def test_a_passing_guardrail_keeps_the_output_text_logprobs() -> None: + response: Final = ResponsesAPIResponse.model_validate( + { + "id": "resp_1", + "created_at": 1, + "model": "gpt-5.6", + "object": "response", + "status": "completed", + "output": [ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "ripe persimmon", + "annotations": [], + "logprobs": list(_PERSIMMON_LOGPROBS), + } + ], + } + ], + } + ) + + result: Final = await OpenAIResponsesHandler().process_output_response( + response, _PassThroughGuardrail(guardrail_name="pass") + ) + + part: Final = stream_item_items(result.output[0], "content")[0] + assert (stream_item_field(part, "text"), stream_item_field(part, "logprobs")) == ( + "ripe persimmon", + list(_PERSIMMON_LOGPROBS), + ) + + +def test_stream_rewrite_drops_logprobs_from_the_delta_done_and_part_events() -> None: + delta: Final = OutputTextDeltaEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, + item_id="msg_1", + output_index=0, + content_index=0, + delta="ripe persimmon", + logprobs=_PERSIMMON_LOGPROBS, + ) + done: Final = OutputTextDoneEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE, + item_id="msg_1", + output_index=0, + content_index=0, + text="ripe persimmon", + logprobs=_PERSIMMON_LOGPROBS, + ) + part_done: Final = ContentPartDoneEvent( + type=ResponsesAPIStreamEvents.CONTENT_PART_DONE, + item_id="msg_1", + output_index=0, + content_index=0, + part=ContentPartDonePartOutputText( + type="output_text", text="ripe persimmon", annotations=[], logprobs=_PERSIMMON_LOGPROBS + ), + ) + + OpenAIResponsesHandler()._sync_stream_events_with_rewrites( + stream_events=(delta, done, part_done), rewrites_by_position={(0, 0): "ripe [MASKED]"} + ) + + assert (delta.delta, delta.logprobs) == ("ripe [MASKED]", None) + assert (done.text, done.logprobs) == ("ripe [MASKED]", None) + assert (part_done.part.text, part_done.part.logprobs) == ("ripe [MASKED]", None) diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 2427e7eacb1..f1e60498928 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -18,7 +18,6 @@ from openai.types.responses.response_function_web_search import ( from pydantic import TypeAdapter, ValidationError import litellm -from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field from litellm.constants import CALLER_SESSION_ID_METADATA_KEY from litellm.integrations.anthropic_cache_control_hook import MAX_CACHE_CONTROL_BLOCKS, AnthropicCacheControlHook from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR @@ -26,6 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( _bedrock_converse_messages_pt, anthropic_messages_pt, ) +from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field from litellm.responses.litellm_completion_transformation.reasoning_items import encode_thinking_blocks from litellm.responses.litellm_completion_transformation.tool_search import tool_search_routing from litellm.responses.litellm_completion_transformation.transformation import ( @@ -38,12 +38,15 @@ from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams from litellm.types.responses.main import OutputToolSearchCall, build_web_search_call from litellm.types.utils import ( ChatCompletionMessageToolCall, + ChatCompletionTokenLogprob, + ChoiceLogprobs, Choices, CompletionTokensDetailsWrapper, Function, Message, ModelResponse, PromptTokensDetailsWrapper, + TopLogprob, Usage, ) @@ -7385,3 +7388,100 @@ def test_a_malformed_item_after_a_codex_additional_tools_item_is_named_by_its_in ) assert f"'{field_path}'" in raised.value.message + + +_HELLO_LOGPROBS: Final = ChoiceLogprobs( + content=[ + ChatCompletionTokenLogprob( + token="Hello", + logprob=-1.3113013e-06, + top_logprobs=[ + TopLogprob(token="Hello", logprob=-1.3113013e-06), + TopLogprob(token="Hi", logprob=-15.149978), + ], + ), + ChatCompletionTokenLogprob(token=".", logprob=-3.0994368e-06, bytes=[46]), + ] +) + + +def _hello_chat_response(logprobs: ChoiceLogprobs | None) -> ModelResponse: + return ModelResponse( + id="chatcmpl-hello", + created=1793489000, + model="gemini-3.5-flash", + object="chat.completion", + choices=[ + Choices( + finish_reason="stop", index=0, message=Message(content="Hello.", role="assistant"), logprobs=logprobs + ) + ], + ) + + +def test_bridged_chat_logprobs_become_output_text_logprobs() -> None: + response: Final = LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( + request_input="Say hello.", + responses_api_request={"top_logprobs": 2}, + chat_completion_response=_hello_chat_response(_HELLO_LOGPROBS), + ) + + assert response.model_dump(mode="json")["output"][0]["content"] == [ + { + "type": "output_text", + "text": "Hello.", + "annotations": [], + "logprobs": [ + { + "token": "Hello", + "logprob": -1.3113013e-06, + "bytes": None, + "top_logprobs": [ + {"token": "Hello", "logprob": -1.3113013e-06, "bytes": None}, + {"token": "Hi", "logprob": -15.149978, "bytes": None}, + ], + }, + {"token": ".", "logprob": -3.0994368e-06, "bytes": [46], "top_logprobs": []}, + ], + } + ] + + +def test_bridged_chat_response_without_logprobs_emits_no_logprobs_key() -> None: + response: Final = LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( + request_input="Say hello.", responses_api_request={}, chat_completion_response=_hello_chat_response(None) + ) + + assert response.model_dump(mode="json")["output"][0]["content"] == [ + {"type": "output_text", "text": "Hello.", "annotations": []} + ] + + +def test_responses_top_logprobs_bridges_to_chat_logprobs() -> None: + request: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model="gemini-3.5-flash", + input="Say hello.", + responses_api_request={"top_logprobs": 2}, + custom_llm_provider="vertex_ai", + ) + + assert (request["top_logprobs"], request["logprobs"]) == (2, True) + + +def test_responses_include_output_text_logprobs_bridges_to_chat_logprobs() -> None: + request: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model="gemini-3.5-flash", + input="Say hello.", + responses_api_request={"include": ["message.output_text.logprobs"]}, + custom_llm_provider="vertex_ai", + ) + + assert request["logprobs"] is True and "top_logprobs" not in request + + +def test_responses_without_top_logprobs_bridges_without_chat_logprobs() -> None: + request: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model="gemini-3.5-flash", input="Say hello.", responses_api_request={}, custom_llm_provider="vertex_ai" + ) + + assert "top_logprobs" not in request and "logprobs" not in request diff --git a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index 87858741908..489a2421147 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -30,6 +30,8 @@ from litellm.types.llms.openai import ( ) from litellm.types.responses.main import build_web_search_call from litellm.types.utils import ( + ChatCompletionTokenLogprob, + ChoiceLogprobs, Delta, ModelResponse, ModelResponseStream, @@ -38,9 +40,7 @@ from litellm.types.utils import ( ) CHAT_COMPLETION_ID = "chatcmpl-77d33d09-effa-4cd2-9c0d-c742d4358256" -RESPONSE_ID_EVENT_TYPES = frozenset( - {"response.created", "response.in_progress", "response.completed"} -) +RESPONSE_ID_EVENT_TYPES = frozenset({"response.created", "response.in_progress", "response.completed"}) def _chunk(content: str, finish_reason: str | None = None) -> ModelResponseStream: @@ -93,11 +93,7 @@ def _build_iterator(chunks) -> LiteLLMCompletionStreamingIterator: def _response_ids(events) -> list[str]: - return [ - event.response.id - for event in events - if getattr(event, "type", None) in RESPONSE_ID_EVENT_TYPES - ] + return [event.response.id for event in events if getattr(event, "type", None) in RESPONSE_ID_EVENT_TYPES] def test_tool_call_delta_is_emitted_as_responses_events(): @@ -228,9 +224,7 @@ async def test_web_search_stream_preserves_hosted_and_client_calls(sync_mode, to content=None, provider_specific_fields={ "web_search_results": [first_result], - "web_search_calls": [ - build_web_search_call(call_ids[0], {"query": "one"}, first_result) - ] + "web_search_calls": [build_web_search_call(call_ids[0], {"query": "one"}, first_result)] if tool_type != "function" and first_result["type"] == "web_search_tool_result" else [], }, @@ -329,9 +323,7 @@ async def test_web_search_stream_preserves_hosted_and_client_calls(sync_mode, to assert {event["output_index"] for event in search_events} == {completed_indexes[item["id"]]} assert search_events[0]["item"]["status"] == "in_progress" assert search_events[-1]["item"] == item - assert item["status"] == ( - "failed" if result_kind == "error" and call_id.endswith("01Search") else "completed" - ) + assert item["status"] == ("failed" if result_kind == "error" and call_id.endswith("01Search") else "completed") assert item["action"]["type"] == "search" assert item["action"]["query"] == ("one" if call_id.endswith("01Search") else "two") assert item["action"]["queries"] == [item["action"]["query"]] @@ -440,9 +432,7 @@ def test_tool_call_arguments_are_chunked_to_match_openai_behavior(): ) # Create a chunk with a large arguments string that should be split - large_arguments = ( - '{"param1": "value1", "param2": "value2", "param3": "value3"}' # 67 chars - ) + large_arguments = '{"param1": "value1", "param2": "value2", "param3": "value3"}' # 67 chars chunk = ModelResponseStream( id="chunk-1", created=123, @@ -496,9 +486,7 @@ def test_tool_call_arguments_are_chunked_to_match_openai_behavior(): # Keep draining pending events (expected: ceil(67 / 10) = 7 delta events) while iterator._pending_tool_events: - evt = iterator._transform_chat_completion_chunk_to_response_api_chunk( - empty_chunk - ) + evt = iterator._transform_chat_completion_chunk_to_response_api_chunk(empty_chunk) if evt and evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA: delta_events.append(evt) @@ -545,20 +533,12 @@ def test_tool_call_delta_without_id_uses_index_mapping(): while iterator._pending_tool_events: all_events.append(iterator._pending_tool_events.pop(0)) - delta_events = [ - evt - for evt in all_events - if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA - ] + delta_events = [evt for evt in all_events if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA] streamed_arguments = "".join(evt.delta for evt in delta_events) assert streamed_arguments == '{"location": "New York"}' - output_item_added_events = [ - evt - for evt in all_events - if evt.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED - ] + output_item_added_events = [evt for evt in all_events if evt.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED] assert len(output_item_added_events) == 1 assert output_item_added_events[0].item.id == "fc_call_abc123" assert output_item_added_events[0].item.call_id == "call_abc123" @@ -599,18 +579,10 @@ def test_parallel_tool_calls_without_ids_use_index_mapping(): while iterator._pending_tool_events: all_events.append(iterator._pending_tool_events.pop(0)) - output_item_added_events = [ - evt - for evt in all_events - if evt.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED - ] + output_item_added_events = [evt for evt in all_events if evt.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED] assert len(output_item_added_events) == 2 - delta_events = [ - evt - for evt in all_events - if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA - ] + delta_events = [evt for evt in all_events if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA] arguments_by_call_id = {} for evt in delta_events: arguments_by_call_id.setdefault(evt.item_id, "") @@ -663,11 +635,7 @@ def test_reused_index_with_new_call_id_marks_fallback_ambiguous(): while iterator._pending_tool_events: all_events.append(iterator._pending_tool_events.pop(0)) - delta_events = [ - evt - for evt in all_events - if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA - ] + delta_events = [evt for evt in all_events if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA] arguments_by_call_id = {} for evt in delta_events: arguments_by_call_id.setdefault(evt.item_id, "") @@ -734,22 +702,17 @@ def test_sync_streaming_events_share_the_chat_completion_response_id(): assert len(response_ids) == 3 assert len(set(response_ids)) == 1 assert ( - ResponsesAPIRequestUtils.decode_responses_api_response_id(response_ids[0])["response_id"] - == CHAT_COMPLETION_ID + ResponsesAPIRequestUtils.decode_responses_api_response_id(response_ids[0])["response_id"] == CHAT_COMPLETION_ID ) @pytest.mark.asyncio async def test_streaming_emits_every_chunk_after_priming_the_response_id(): - iterator = _build_iterator( - [_chunk("Hel"), _chunk("lo"), _chunk("!", finish_reason="stop")] - ) + iterator = _build_iterator([_chunk("Hel"), _chunk("lo"), _chunk("!", finish_reason="stop")]) events = [event async for event in iterator] - deltas = "".join( - event.delta for event in events if getattr(event, "type", None) == "response.output_text.delta" - ) + deltas = "".join(event.delta for event in events if getattr(event, "type", None) == "response.output_text.delta") assert deltas == "Hello!" @@ -795,7 +758,9 @@ async def test_leading_empty_choices_chunk_does_not_kill_the_stream(): event_types = [getattr(event, "type", None) for event in events] assert event_types.count(ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED) == 1 - assert "".join(event.delta for event in events if event.type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA) == "Hello!" + assert ( + "".join(event.delta for event in events if event.type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA) == "Hello!" + ) assert event_types[-1] == ResponsesAPIStreamEvents.RESPONSE_COMPLETED @@ -979,9 +944,7 @@ def test_streamed_unrecognized_tool_choice_is_echoed_as_auto() -> None: litellm_metadata={}, ) - response_events: Final = [ - event for event in iterator if getattr(event, "type", None) in RESPONSE_ID_EVENT_TYPES - ] + response_events: Final = [event for event in iterator if getattr(event, "type", None) in RESPONSE_ID_EVENT_TYPES] assert [event.response.tool_choice for event in response_events] == ["auto", "auto", "auto"] @@ -1031,9 +994,7 @@ def _signed_thinking_and_text_in_one_chunk() -> tuple[ModelResponseStream, ...]: chunk: Final = _chunk("Hello!", finish_reason="stop") signed_block: Final = {"type": "thinking", "thinking": "", "signature": "sig_only"} delta: Final = Delta(**{**chunk.choices[0].delta.model_dump(), "thinking_blocks": [signed_block]}) - return ( - chunk.model_copy(update={"choices": [StreamingChoices(index=0, delta=delta, finish_reason="stop")]}), - ) + return (chunk.model_copy(update={"choices": [StreamingChoices(index=0, delta=delta, finish_reason="stop")]}),) StreamChunks = Callable[[], tuple[ModelResponseStream, ...]] @@ -1553,9 +1514,7 @@ async def test_anthropic_tool_search_call_streams_as_tool_search_call_item(sync_ model="claude-sonnet-5-5", object="chat.completion.chunk", choices=[ - StreamingChoices( - index=0, delta=delta, finish_reason="tool_calls" if index == len(deltas) - 1 else None - ) + StreamingChoices(index=0, delta=delta, finish_reason="tool_calls" if index == len(deltas) - 1 else None) ], ) for index, delta in enumerate(deltas) @@ -1965,9 +1924,7 @@ def test_the_partial_response_billed_on_disconnect_carries_the_logging_objects_b custom_llm_provider="anthropic", litellm_metadata={}, ) - relayed: Final = tuple( - itertools.takewhile(lambda event: not isinstance(event, OutputTextDeltaEvent), iterator) - ) + relayed: Final = tuple(itertools.takewhile(lambda event: not isinstance(event, OutputTextDeltaEvent), iterator)) assert relayed, "the stream must relay at least one chunk before the client goes away" partial: Final = iterator.partial_response_on_disconnect() @@ -2078,3 +2035,73 @@ async def test_streamed_refusal_deltas_become_one_refusal_part_of_the_message(sy assert [event.refusal for event in refusal_done] == [expected_content[0]["refusal"]] assert [item.model_dump()["content"] for item in message_done] == [expected_content] assert [item.model_dump()["content"] for item in finished.response.output] == [expected_content] + + +def _logprobs_chunk(content: str, logprob: float, finish_reason: str | None = None) -> ModelResponseStream: + return ModelResponseStream( + id=CHAT_COMPLETION_ID, + created=1748575031, + model="gemini-3.5-flash", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + index=0, + delta=Delta(role="assistant", content=content), + finish_reason=finish_reason, + logprobs=ChoiceLogprobs(content=[ChatCompletionTokenLogprob(token=content, logprob=logprob)]), + ) + ], + ) + + +def test_streamed_chat_logprobs_reach_the_text_delta_and_done_events() -> None: + events: Final = [ + event.model_dump(mode="json") + for event in _build_iterator( + [_logprobs_chunk("Hello", -0.25), _logprobs_chunk(".", -0.5, finish_reason="stop")] + ) + ] + hello: Final = {"token": "Hello", "logprob": -0.25, "bytes": None, "top_logprobs": []} + period: Final = {"token": ".", "logprob": -0.5, "bytes": None, "top_logprobs": []} + + deltas: Final = [event for event in events if event["type"] == "response.output_text.delta"] + done: Final = next(event for event in events if event["type"] == "response.output_text.done") + part_done: Final = next(event for event in events if event["type"] == "response.content_part.done") + item_done: Final = next( + event for event in events if event["type"] == "response.output_item.done" and event["item"]["type"] == "message" + ) + + assert [event["logprobs"] for event in deltas] == [[hello], [period]] + assert done["logprobs"] == [hello, period] + assert part_done["part"]["logprobs"] == [hello, period] + assert item_done["item"]["content"][0]["logprobs"] == [hello, period] + completed: Final = next(event for event in events if event["type"] == "response.completed") + assert completed["response"]["output"][0]["content"][0]["logprobs"] == [hello, period] + + +def test_streamed_answer_logprobs_stay_off_the_reasoning_item_in_the_completed_snapshot() -> None: + events: Final = [ + event.model_dump(mode="json") + for event in _build_iterator( + [_reasoning_chunk("let me think"), _logprobs_chunk("Hello", -0.25, finish_reason="stop")] + ) + ] + hello: Final = {"token": "Hello", "logprob": -0.25, "bytes": None, "top_logprobs": []} + + completed: Final = next(event for event in events if event["type"] == "response.completed") + by_type: Final = {item["type"]: item for item in completed["response"]["output"]} + + assert by_type["message"]["content"][0]["logprobs"] == [hello] + assert all("logprobs" not in part for part in by_type["reasoning"]["content"]) + + +def test_streamed_chat_without_logprobs_emits_text_events_without_a_logprobs_key() -> None: + events: Final = [ + event.model_dump(mode="json") for event in _build_iterator([_chunk("Hello"), _chunk("!", finish_reason="stop")]) + ] + + text_events: Final = [ + event for event in events if event["type"] in {"response.output_text.delta", "response.output_text.done"} + ] + + assert text_events and all("logprobs" not in event for event in text_events) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 2272b4bea43..d6a47e6fc3f 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -41545,6 +41545,8 @@ export interface components { OutputText: { /** Annotations */ annotations: components["schemas"]["GenericResponseOutputItemContentAnnotation"][] | null; + /** Logprobs */ + logprobs?: components["schemas"]["OutputTextLogprob"][] | null; /** Text */ text: string | null; /** Type */ @@ -41552,6 +41554,30 @@ export interface components { } & { [key: string]: unknown; }; + /** OutputTextLogprob */ + OutputTextLogprob: { + /** Bytes */ + bytes?: number[] | null; + /** Logprob */ + logprob: number; + /** Token */ + token: string; + /** Top Logprobs */ + top_logprobs: components["schemas"]["OutputTextTopLogprob"][]; + } & { + [key: string]: unknown; + }; + /** OutputTextTopLogprob */ + OutputTextTopLogprob: { + /** Bytes */ + bytes?: number[] | null; + /** Logprob */ + logprob: number; + /** Token */ + token: string; + } & { + [key: string]: unknown; + }; /** OutputTokensDetails */ OutputTokensDetails: { /** Audio Tokens */