From d9b67d8f52a34d502bd3e6e1db82ea5779ee61f9 Mon Sep 17 00:00:00 2001 From: mateo Date: Sun, 1 Nov 2026 00:32:40 +0000 Subject: [PATCH] fix(responses): keep logprobs on unchanged guardrail text, off reasoning items, and in the OpenAPI schema Second Greptile round on the bridge logprobs change. A guardrail that writes the text back unchanged keeps the logprobs, only a real rewrite clears them. The response.completed snapshot attaches the streamed answer tokens to the message item only, so a reasoning item built from reasoning_content carries none. The wrap model_serializer has no return annotation anymore, since pydantic used it as the serialization schema and collapsed OutputText and the output_text events to a bare object in the OpenAPI snapshot and schema.d.ts, which are regenerated here. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../guardrail_translation/handler.py | 3 +- litellm/proxy/_lazy_openapi_snapshot.json | 59 ++++++++++++++++ .../streaming_iterator.py | 2 +- litellm/types/llms/base.py | 2 +- ...test_openai_responses_guardrail_handler.py | 69 +++++++++++++++++-- .../test_streaming_iterator_transformation.py | 16 +++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 13 ++++ 7 files changed, 156 insertions(+), 8 deletions(-) diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 7db75645e88..8756466c176 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -1428,8 +1428,9 @@ class OpenAIResponsesHandler(BaseTranslation): @staticmethod def _write_rewritten_text(event: object, field: str, text: str) -> None: """A rewrite replaces the text, so the logprobs that spell out the original text go with it.""" + changed: Final = stream_item_field(event, field) != text OpenAIResponsesHandler._write_event_field(event, field, text) - if stream_item_field(event, "logprobs") is not None: + if changed and stream_item_field(event, "logprobs") is not None: OpenAIResponsesHandler._write_event_field(event, "logprobs", None) @staticmethod diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 5fd1f51cc0b..2748b15d57f 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -26474,6 +26474,65 @@ }, "OutputText": { "additionalProperties": true, + "description": "Text output content from an assistant message", + "properties": { + "annotations": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/GenericResponseOutputItemContentAnnotation" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Annotations" + }, + "logprobs": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/OutputTextLogprob" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Logprobs" + }, + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Text" + }, + "type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Type" + } + }, + "required": [ + "type", + "text", + "annotations" + ], + "title": "OutputText", "type": "object" }, "OutputTextLogprob": { diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 4ecc8f5ffac..06eae09ed3d 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -1597,7 +1597,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): def _with_streamed_logprobs(self, item: object) -> object: """stream_chunk_builder drops chat logprobs, so the snapshot's text part takes the ones the stream carried.""" logprobs: Final = self._streamed_text_logprobs() - if logprobs is None or not isinstance(item, GenericResponseOutputItem): + if logprobs is None or not isinstance(item, GenericResponseOutputItem) or item.type != "message": return item content: Final = tuple( part.model_copy(update={"logprobs": logprobs}) if isinstance(part, OutputText) else part diff --git a/litellm/types/llms/base.py b/litellm/types/llms/base.py index 231bfff2036..21aa1ec00e6 100644 --- a/litellm/types/llms/base.py +++ b/litellm/types/llms/base.py @@ -106,7 +106,7 @@ class CarriesOutputTextLogprobs(BaseLiteLLMOpenAIResponseObject): # frozen-ok: logprobs: Sequence[OutputTextLogprob] | None = None @model_serializer(mode="wrap") - def _without_absent_logprobs(self, handler: SerializerFunctionWrapHandler) -> Mapping[str, object]: + def _without_absent_logprobs(self, handler: SerializerFunctionWrapHandler): # noqa: ANN202 # a return annotation here replaces the model's OpenAPI schema with a bare object dumped: Final[Mapping[str, object]] = handler(self) if self.logprobs is not None: return dumped diff --git a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py index 5c8b147f842..0cfc4e690b2 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -5138,7 +5138,8 @@ class TestRefusalPartsReachTextGuardrails: assert guardrail.seen_texts == [[*leading_text, _REFUSAL]] assert [ - (part["type"], part.get("text") or part.get("refusal")) for part in result.model_dump()["output"][0]["content"] + (part["type"], part.get("text") or part.get("refusal")) + for part in result.model_dump()["output"][0]["content"] ] == expected_parts @pytest.mark.asyncio @@ -5161,7 +5162,8 @@ class TestRefusalPartsReachTextGuardrails: assert guardrail.seen_texts == [["Jane asked first.", _REFUSAL]] assert [ - (part["type"], part.get("text") or part.get("refusal")) for part in result.model_dump()["output"][0]["content"] + (part["type"], part.get("text") or part.get("refusal")) + for part in result.model_dump()["output"][0]["content"] ] == [("output_text", "[NAME] asked first."), ("refusal", _MASKED_REFUSAL)] @pytest.mark.asyncio @@ -5175,7 +5177,9 @@ class TestRefusalPartsReachTextGuardrails: part=ContentPartDonePartRefusal(type="refusal", refusal=_REFUSAL), **position, ), - OutputItemDoneEvent(type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, output_index=0, item=_refused_message()), + OutputItemDoneEvent( + type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, output_index=0, item=_refused_message() + ), ResponseIncompleteEvent(type=ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, response=_refused_response()), ] @@ -5242,7 +5246,7 @@ class _PersimmonTextMaskingGuardrail(CustomGuardrail): inputs: GenericGuardrailAPIInputs, request_data: dict, input_type: Literal["request", "response"], - logging_obj: Optional[LiteLLMLoggingObj] = None, + logging_obj: LiteLLMLoggingObj | None = None, ) -> GenericGuardrailAPIInputs: return {**inputs, "texts": [text.replace("persimmon", "[MASKED]") for text in inputs.get("texts", [])]} @@ -5269,7 +5273,12 @@ async def test_masked_output_text_drops_the_logprobs_that_spell_the_original_tex "status": "completed", "role": "assistant", "content": [ - {"type": "output_text", "text": "ripe persimmon", "annotations": [], "logprobs": list(_PERSIMMON_LOGPROBS)} + { + "type": "output_text", + "text": "ripe persimmon", + "annotations": [], + "logprobs": list(_PERSIMMON_LOGPROBS), + } ], } ], @@ -5284,6 +5293,56 @@ async def test_masked_output_text_drops_the_logprobs_that_spell_the_original_tex assert (stream_item_field(part, "text"), stream_item_field(part, "logprobs")) == ("ripe [MASKED]", None) +class _PassThroughGuardrail(CustomGuardrail): + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: LiteLLMLoggingObj | None = None, + ) -> GenericGuardrailAPIInputs: + return inputs + + +@pytest.mark.asyncio +async def test_a_passing_guardrail_keeps_the_output_text_logprobs() -> None: + response = ResponsesAPIResponse.model_validate( + { + "id": "resp_1", + "created_at": 1, + "model": "gpt-5.6", + "object": "response", + "status": "completed", + "output": [ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "ripe persimmon", + "annotations": [], + "logprobs": list(_PERSIMMON_LOGPROBS), + } + ], + } + ], + } + ) + + result = await OpenAIResponsesHandler().process_output_response( + response, _PassThroughGuardrail(guardrail_name="pass") + ) + + part = stream_item_items(result.output[0], "content")[0] + assert (stream_item_field(part, "text"), stream_item_field(part, "logprobs")) == ( + "ripe persimmon", + list(_PERSIMMON_LOGPROBS), + ) + + def test_stream_rewrite_drops_logprobs_from_the_delta_done_and_part_events() -> None: delta = OutputTextDeltaEvent( type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, diff --git a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index c0fcf509456..e2ef9ad5a43 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -2027,6 +2027,22 @@ def test_streamed_chat_logprobs_reach_the_text_delta_and_done_events() -> None: assert completed["response"]["output"][0]["content"][0]["logprobs"] == [hello, period] +def test_streamed_answer_logprobs_stay_off_the_reasoning_item_in_the_completed_snapshot() -> None: + events: Final = [ + event.model_dump(mode="json") + for event in _build_iterator( + [_reasoning_chunk("let me think"), _logprobs_chunk("Hello", -0.25, finish_reason="stop")] + ) + ] + hello: Final = {"token": "Hello", "logprob": -0.25, "bytes": None, "top_logprobs": []} + + completed: Final = next(event for event in events if event["type"] == "response.completed") + by_type: Final = {item["type"]: item for item in completed["response"]["output"]} + + assert by_type["message"]["content"][0]["logprobs"] == [hello] + assert all("logprobs" not in part for part in by_type["reasoning"]["content"]) + + def test_streamed_chat_without_logprobs_emits_text_events_without_a_logprobs_key() -> None: events: Final = [ event.model_dump(mode="json") for event in _build_iterator([_chunk("Hello"), _chunk("!", finish_reason="stop")]) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 34b0c17f53c..14bd897f9e0 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -40723,7 +40723,20 @@ export interface components { } & { [key: string]: unknown; }; + /** + * OutputText + * @description Text output content from an assistant message + */ OutputText: { + /** Annotations */ + annotations: components["schemas"]["GenericResponseOutputItemContentAnnotation"][] | null; + /** Logprobs */ + logprobs?: components["schemas"]["OutputTextLogprob"][] | null; + /** Text */ + text: string | null; + /** Type */ + type: string | null; + } & { [key: string]: unknown; }; /** OutputTextLogprob */