From 015ee22f1470e44e794f18a1787d330b75a40fa2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 3 Nov 2026 17:18:14 -0800 Subject: [PATCH] fix(responses): accept SDK logprob shapes and redact Responses logprobs and untyped output items Message redaction now drops output_text logprobs and also redacts output items that stay plain dicts, and the allowed-params choice for bridged calls moves into one helper --- .../transformation.py | 2 +- litellm/litellm_core_utils/redact_messages.py | 28 +++++++---- litellm/main.py | 31 +++++-------- litellm/proxy/_lazy_openapi_snapshot.json | 46 +++++++++++++------ litellm/responses/bridge_check.py | 21 +++++++++ litellm/types/llms/base.py | 6 +-- .../test_redact_messages.py | 29 ++++++++++++ .../unit/types/llms/test_types_llms_openai.py | 30 ++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 6 +-- 9 files changed, 150 insertions(+), 49 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index b4c8270fe76..5e1f8a07381 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -1902,7 +1902,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): tools = [] responses_api_request["tools"] = tools - typed_options: Final = ( + typed_options: Final[Mapping[str, object]] = ( cast(Mapping[str, object], web_search_options) # cast-ok: caller-supplied dict if isinstance(web_search_options, dict) else {} diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index 9da0e35bdda..24f2476391b 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -308,6 +308,7 @@ def _redact_responses_api_output(output_items): for content_part in stream_item_items(output_item, "content"): _redact_field_when_present(content_part, "text") _redact_field_when_present(content_part, "refusal") + _drop_field_when_present(content_part, "logprobs") if output_type == "reasoning": for summary_item in stream_item_items(output_item, "summary"): @@ -324,10 +325,16 @@ def _redact_responses_api_output(output_items): _RESPONSE_ITEMS: Final = TypeAdapter(tuple[object, ...]) -def _redacted_responses_api_output_part(part: object, fields: tuple[str, ...], redacted_str: str) -> object: +def _redacted_responses_api_output_part( + part: object, fields: tuple[str, ...], redacted_str: str, *, dropped: tuple[str, ...] = () +) -> object: if (part_mapping := _as_mapping(part)) is None: return part - return {key: redacted_str if key in fields and value is not None else value for key, value in part_mapping.items()} + return { + key: redacted_str if key in fields and value is not None else value + for key, value in part_mapping.items() + if key not in dropped + } def _redacted_responses_api_output_item(item: object, redacted_str: str) -> object: @@ -341,7 +348,7 @@ def _redacted_responses_api_output_item(item: object, redacted_str: str) -> obje **( { "content": [ - _redacted_responses_api_output_part(part, ("text", "refusal"), redacted_str) + _redacted_responses_api_output_part(part, ("text", "refusal"), redacted_str, dropped=("logprobs",)) for part in _RESPONSE_ITEMS.validate_python(content) ] } @@ -607,15 +614,20 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons elif isinstance(_result, dict) and "output" in _result: return _redacted_responses_api_response(_result, redact_instructions=redact_streaming_responses) elif isinstance(_result, litellm.ResponsesAPIResponse): - if hasattr(_result, "output"): - _redact_responses_api_output(_result.output) + output: Final = _RESPONSE_ITEMS.validate_python(_result.output) + _redact_responses_api_output(output) # Redact reasoning field in ResponsesAPIResponse if hasattr(_result, "reasoning") and _result.reasoning is not None: _result.reasoning = None return _result.model_copy( - update={"instructions": REDACTED_BY_LITELLM} - if redact_streaming_responses and _result.instructions is not None - else {} + update={ + "output": _redact_responses_api_output_dict(output, REDACTED_BY_LITELLM), + **( + {"instructions": REDACTED_BY_LITELLM} + if redact_streaming_responses and _result.instructions is not None + else {} + ), + } ) elif isinstance(_result, litellm.EmbeddingResponse): if hasattr(_result, "data") and _result.data is not None: diff --git a/litellm/main.py b/litellm/main.py index 6c3992e6bbd..dc2c9aa8fc3 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -121,7 +121,6 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header from litellm.litellm_core_utils.reasoning_effort_capability import ( model_group_from_metadata, - reasoning_effort_needs_allowing, resolve_chat_reasoning, ) from litellm.litellm_core_utils.request_timeout_resolver import ( @@ -1318,16 +1317,16 @@ def mock_completion( raise Exception(f"Mock completion response failed - {e}") +from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the old definition site + _resolve_openai_api_base as _resolve_openai_api_base, # pyright: ignore[reportPrivateUsage] # noqa: PLC0414 # re-export of moved helper +) from litellm.responses.bridge_check import ( - RESPONSES_WEB_SEARCH_PROVIDERS, + bridge_allowed_params, bridge_check_request_params, bridge_reasoning_effort, keep_registered_mode, str_keyed_mapping, ) -from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the old definition site - _resolve_openai_api_base as _resolve_openai_api_base, # pyright: ignore[reportPrivateUsage] # noqa: PLC0414 # re-export of moved helper -) from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the old definition site responses_api_bridge_check as responses_api_bridge_check, # pyright: ignore[reportUnknownVariableType] # noqa: PLC0414 # re-export of moved function ) @@ -6078,22 +6077,16 @@ def completion( bridges_to_responses_api: Final = ( responses_api_model_info.get("mode") == "responses" and not skip_responses_api_bridge ) - bridge_allowed_params: Final[tuple[str, ...]] = ( - ("reasoning_effort", "web_search_options") - if bridges_to_responses_api and custom_llm_provider in RESPONSES_WEB_SEARCH_PROVIDERS - else ("reasoning_effort",) - if bridges_to_responses_api - or reasoning_effort_needs_allowing( - chat_reasoning.effort, - model=model, - custom_llm_provider=custom_llm_provider, - base_model=base_model if isinstance(base_model, str) else None, - ) - else () + bridge_params: Final = bridge_allowed_params( + bridges_to_responses_api=bridges_to_responses_api, + chat_reasoning_effort=chat_reasoning.effort, + model=model, + custom_llm_provider=custom_llm_provider, + base_model=base_model if isinstance(base_model, str) else None, ) allowed_openai_params: Final[list[str] | None] = ( - [*(kwargs.get("allowed_openai_params") or []), *bridge_allowed_params] - if bridge_allowed_params + [*(kwargs.get("allowed_openai_params") or []), *bridge_params] + if bridge_params else kwargs.get("allowed_openai_params") ) optional_param_args: Final = { diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 7129a8bcc44..99924558da4 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -26669,17 +26669,23 @@ "type": "string" }, "top_logprobs": { - "items": { - "$ref": "#/components/schemas/OutputTextTopLogprob" - }, - "title": "Top Logprobs", - "type": "array" + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/OutputTextTopLogprob" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Top Logprobs" } }, "required": [ "token", - "logprob", - "top_logprobs" + "logprob" ], "title": "OutputTextLogprob", "type": "object" @@ -26702,18 +26708,28 @@ "title": "Bytes" }, "logprob": { - "title": "Logprob", - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Logprob" }, "token": { - "title": "Token", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Token" } }, - "required": [ - "token", - "logprob" - ], "title": "OutputTextTopLogprob", "type": "object" }, diff --git a/litellm/responses/bridge_check.py b/litellm/responses/bridge_check.py index 1b534882b60..159acc5a69b 100644 --- a/litellm/responses/bridge_check.py +++ b/litellm/responses/bridge_check.py @@ -7,6 +7,10 @@ import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value, is_nested_path from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.litellm_core_utils.reasoning_effort_capability import ( + DeploymentReasoningEffort, + reasoning_effort_needs_allowing, +) from litellm.llms.azure_ai.chat.transformation import AzureAIGPT5Config from litellm.llms.azure_ai.common_utils import ( azure_ai_supports_native_responses, @@ -126,6 +130,23 @@ def bridge_check_request_params(request_params: Mapping[str, object]) -> Mapping return {**kept, **reduce(delete_nested_value, nested_paths, {"extra_body": kept_extra_body})} +def bridge_allowed_params( + *, + bridges_to_responses_api: bool, + chat_reasoning_effort: DeploymentReasoningEffort, + model: str, + custom_llm_provider: str, + base_model: str | None, +) -> tuple[str, ...]: + if bridges_to_responses_api and custom_llm_provider in RESPONSES_WEB_SEARCH_PROVIDERS: + return ("reasoning_effort", "web_search_options") + if bridges_to_responses_api or reasoning_effort_needs_allowing( + chat_reasoning_effort, model=model, custom_llm_provider=custom_llm_provider, base_model=base_model + ): + return ("reasoning_effort",) + return () + + def bridge_reasoning_effort( reasoning_effort: str | Mapping[str, object] | None, extra_body: object ) -> str | Mapping[str, object] | None: diff --git a/litellm/types/llms/base.py b/litellm/types/llms/base.py index 21aa1ec00e6..b4abf15a1de 100644 --- a/litellm/types/llms/base.py +++ b/litellm/types/llms/base.py @@ -88,8 +88,8 @@ class CachedTokensDetails(LiteLLMBaseModel): class OutputTextTopLogprob(TypedDict): - token: ReadOnly[str] - logprob: ReadOnly[float] + token: NotRequired[ReadOnly[str | None]] + logprob: NotRequired[ReadOnly[float | None]] bytes: NotRequired[ReadOnly[Sequence[int] | None]] @@ -97,7 +97,7 @@ class OutputTextLogprob(TypedDict): token: ReadOnly[str] logprob: ReadOnly[float] bytes: NotRequired[ReadOnly[Sequence[int] | None]] - top_logprobs: ReadOnly[Sequence[OutputTextTopLogprob]] + top_logprobs: NotRequired[ReadOnly[Sequence[OutputTextTopLogprob] | None]] class CarriesOutputTextLogprobs(BaseLiteLLMOpenAIResponseObject): # frozen-ok: guardrails rewrite these in place diff --git a/tests/unit/litellm_core_utils/test_redact_messages.py b/tests/unit/litellm_core_utils/test_redact_messages.py index 2fd3f16ab17..dc3fdc26612 100644 --- a/tests/unit/litellm_core_utils/test_redact_messages.py +++ b/tests/unit/litellm_core_utils/test_redact_messages.py @@ -244,6 +244,35 @@ def test_redaction_hides_a_compaction_summary_in_responses_api_output(shape: str assert SECRET in str(result) +@pytest.mark.parametrize("token_bytes", [list(SECRET.encode()), None], ids=["utf8_bytes", "null_bytes"]) +@pytest.mark.parametrize("shape", ["object", "dict"]) +def test_redaction_hides_the_answer_tokens_in_responses_api_logprobs(shape: str, token_bytes: list[int] | None) -> None: + top: Final = {"token": SECRET, "bytes": token_bytes, "logprob": -0.2} + logprob: Final = {"token": SECRET, "bytes": token_bytes, "logprob": -0.1, "top_logprobs": [top]} + response: Final = ResponsesAPIResponse( + id="resp_1", + created_at=1, + model="gpt-5.6-sol", + object="response", + status="completed", + output=[ + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": SECRET, "annotations": [], "logprobs": [logprob]}], + } + ], + ) + result: Final = response if shape == "object" else response.model_dump() + + redacted: Final = perform_redaction({}, result) + + assert SECRET not in str(redacted), redacted + assert SECRET in str(result) + + @pytest.mark.parametrize("surface", ("typed", "dict", "standard", "callback", "helper")) def test_responses_redaction_removes_instructions_without_changing_the_response(surface: str) -> None: response: Final = litellm.ResponsesAPIResponse.model_validate( diff --git a/tests/unit/types/llms/test_types_llms_openai.py b/tests/unit/types/llms/test_types_llms_openai.py index 8c37e86500b..65aa02c122e 100644 --- a/tests/unit/types/llms/test_types_llms_openai.py +++ b/tests/unit/types/llms/test_types_llms_openai.py @@ -17,6 +17,7 @@ from litellm.types.llms.openai import ( OpenAIModerationResponse, OpenAIVideoObject, OutputTextDeltaEvent, + OutputTextDoneEvent, ResponseCompletedEvent, ResponsesAPIResponse, ResponsesAPIStreamEvents, @@ -115,6 +116,35 @@ def test_content_part_done_preserves_typed_text_with_optional_logprobs( assert event.part.logprobs == logprobs +@pytest.mark.parametrize( + "logprob", + ( + {"token": "hi", "logprob": -0.1}, + {"token": "hi", "logprob": -0.1, "top_logprobs": None}, + {"token": "hi", "logprob": -0.1, "top_logprobs": [{"token": None, "logprob": None}]}, + ), + ids=("no_top_logprobs", "null_top_logprobs", "null_top_entry"), +) +@pytest.mark.parametrize( + ("event_type", "text_fields"), + ( + (OutputTextDeltaEvent, {"type": "response.output_text.delta", "delta": "hi"}), + (OutputTextDoneEvent, {"type": "response.output_text.done", "text": "hi"}), + ), + ids=("delta", "done"), +) +def test_text_events_accept_every_logprob_shape_the_openai_sdk_allows( + event_type: type[OutputTextDeltaEvent] | type[OutputTextDoneEvent], + text_fields: Mapping[str, str], + logprob: Mapping[str, object], +) -> None: + event: Final = event_type.model_validate( + {**text_fields, "item_id": "msg_1", "output_index": 0, "content_index": 0, "logprobs": [logprob]} + ) + + assert event.model_dump()["logprobs"] == [logprob] + + class TestResponsesAPIResponseOutputText: """Tests for the output_text property on ResponsesAPIResponse""" diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 54a52fb3ffb..3be5bebb190 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -40910,7 +40910,7 @@ export interface components { /** Token */ token: string; /** Top Logprobs */ - top_logprobs: components["schemas"]["OutputTextTopLogprob"][]; + top_logprobs?: components["schemas"]["OutputTextTopLogprob"][] | null; } & { [key: string]: unknown; }; @@ -40919,9 +40919,9 @@ export interface components { /** Bytes */ bytes?: number[] | null; /** Logprob */ - logprob: number; + logprob?: number | null; /** Token */ - token: string; + token?: string | null; } & { [key: string]: unknown; };