mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
fix(responses): accept SDK logprob shapes and redact Responses logprobs and untyped output items
Message redaction now drops output_text logprobs and also redacts output items that stay plain dicts, and the allowed-params choice for bridged calls moves into one helper
This commit is contained in:
parent
ded0004d31
commit
015ee22f14
9 changed files with 150 additions and 49 deletions
|
|
@ -1902,7 +1902,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
tools = []
|
||||
responses_api_request["tools"] = tools
|
||||
|
||||
typed_options: Final = (
|
||||
typed_options: Final[Mapping[str, object]] = (
|
||||
cast(Mapping[str, object], web_search_options) # cast-ok: caller-supplied dict
|
||||
if isinstance(web_search_options, dict)
|
||||
else {}
|
||||
|
|
|
|||
|
|
@ -308,6 +308,7 @@ def _redact_responses_api_output(output_items):
|
|||
for content_part in stream_item_items(output_item, "content"):
|
||||
_redact_field_when_present(content_part, "text")
|
||||
_redact_field_when_present(content_part, "refusal")
|
||||
_drop_field_when_present(content_part, "logprobs")
|
||||
|
||||
if output_type == "reasoning":
|
||||
for summary_item in stream_item_items(output_item, "summary"):
|
||||
|
|
@ -324,10 +325,16 @@ def _redact_responses_api_output(output_items):
|
|||
_RESPONSE_ITEMS: Final = TypeAdapter(tuple[object, ...])
|
||||
|
||||
|
||||
def _redacted_responses_api_output_part(part: object, fields: tuple[str, ...], redacted_str: str) -> object:
|
||||
def _redacted_responses_api_output_part(
|
||||
part: object, fields: tuple[str, ...], redacted_str: str, *, dropped: tuple[str, ...] = ()
|
||||
) -> object:
|
||||
if (part_mapping := _as_mapping(part)) is None:
|
||||
return part
|
||||
return {key: redacted_str if key in fields and value is not None else value for key, value in part_mapping.items()}
|
||||
return {
|
||||
key: redacted_str if key in fields and value is not None else value
|
||||
for key, value in part_mapping.items()
|
||||
if key not in dropped
|
||||
}
|
||||
|
||||
|
||||
def _redacted_responses_api_output_item(item: object, redacted_str: str) -> object:
|
||||
|
|
@ -341,7 +348,7 @@ def _redacted_responses_api_output_item(item: object, redacted_str: str) -> obje
|
|||
**(
|
||||
{
|
||||
"content": [
|
||||
_redacted_responses_api_output_part(part, ("text", "refusal"), redacted_str)
|
||||
_redacted_responses_api_output_part(part, ("text", "refusal"), redacted_str, dropped=("logprobs",))
|
||||
for part in _RESPONSE_ITEMS.validate_python(content)
|
||||
]
|
||||
}
|
||||
|
|
@ -607,15 +614,20 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
|
|||
elif isinstance(_result, dict) and "output" in _result:
|
||||
return _redacted_responses_api_response(_result, redact_instructions=redact_streaming_responses)
|
||||
elif isinstance(_result, litellm.ResponsesAPIResponse):
|
||||
if hasattr(_result, "output"):
|
||||
_redact_responses_api_output(_result.output)
|
||||
output: Final = _RESPONSE_ITEMS.validate_python(_result.output)
|
||||
_redact_responses_api_output(output)
|
||||
# Redact reasoning field in ResponsesAPIResponse
|
||||
if hasattr(_result, "reasoning") and _result.reasoning is not None:
|
||||
_result.reasoning = None
|
||||
return _result.model_copy(
|
||||
update={"instructions": REDACTED_BY_LITELLM}
|
||||
if redact_streaming_responses and _result.instructions is not None
|
||||
else {}
|
||||
update={
|
||||
"output": _redact_responses_api_output_dict(output, REDACTED_BY_LITELLM),
|
||||
**(
|
||||
{"instructions": REDACTED_BY_LITELLM}
|
||||
if redact_streaming_responses and _result.instructions is not None
|
||||
else {}
|
||||
),
|
||||
}
|
||||
)
|
||||
elif isinstance(_result, litellm.EmbeddingResponse):
|
||||
if hasattr(_result, "data") and _result.data is not None:
|
||||
|
|
|
|||
|
|
@ -121,7 +121,6 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header
|
||||
from litellm.litellm_core_utils.reasoning_effort_capability import (
|
||||
model_group_from_metadata,
|
||||
reasoning_effort_needs_allowing,
|
||||
resolve_chat_reasoning,
|
||||
)
|
||||
from litellm.litellm_core_utils.request_timeout_resolver import (
|
||||
|
|
@ -1318,16 +1317,16 @@ def mock_completion(
|
|||
raise Exception(f"Mock completion response failed - {e}")
|
||||
|
||||
|
||||
from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the old definition site
|
||||
_resolve_openai_api_base as _resolve_openai_api_base, # pyright: ignore[reportPrivateUsage] # noqa: PLC0414 # re-export of moved helper
|
||||
)
|
||||
from litellm.responses.bridge_check import (
|
||||
RESPONSES_WEB_SEARCH_PROVIDERS,
|
||||
bridge_allowed_params,
|
||||
bridge_check_request_params,
|
||||
bridge_reasoning_effort,
|
||||
keep_registered_mode,
|
||||
str_keyed_mapping,
|
||||
)
|
||||
from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the old definition site
|
||||
_resolve_openai_api_base as _resolve_openai_api_base, # pyright: ignore[reportPrivateUsage] # noqa: PLC0414 # re-export of moved helper
|
||||
)
|
||||
from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the old definition site
|
||||
responses_api_bridge_check as responses_api_bridge_check, # pyright: ignore[reportUnknownVariableType] # noqa: PLC0414 # re-export of moved function
|
||||
)
|
||||
|
|
@ -6078,22 +6077,16 @@ def completion(
|
|||
bridges_to_responses_api: Final = (
|
||||
responses_api_model_info.get("mode") == "responses" and not skip_responses_api_bridge
|
||||
)
|
||||
bridge_allowed_params: Final[tuple[str, ...]] = (
|
||||
("reasoning_effort", "web_search_options")
|
||||
if bridges_to_responses_api and custom_llm_provider in RESPONSES_WEB_SEARCH_PROVIDERS
|
||||
else ("reasoning_effort",)
|
||||
if bridges_to_responses_api
|
||||
or reasoning_effort_needs_allowing(
|
||||
chat_reasoning.effort,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
base_model=base_model if isinstance(base_model, str) else None,
|
||||
)
|
||||
else ()
|
||||
bridge_params: Final = bridge_allowed_params(
|
||||
bridges_to_responses_api=bridges_to_responses_api,
|
||||
chat_reasoning_effort=chat_reasoning.effort,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
base_model=base_model if isinstance(base_model, str) else None,
|
||||
)
|
||||
allowed_openai_params: Final[list[str] | None] = (
|
||||
[*(kwargs.get("allowed_openai_params") or []), *bridge_allowed_params]
|
||||
if bridge_allowed_params
|
||||
[*(kwargs.get("allowed_openai_params") or []), *bridge_params]
|
||||
if bridge_params
|
||||
else kwargs.get("allowed_openai_params")
|
||||
)
|
||||
optional_param_args: Final = {
|
||||
|
|
|
|||
|
|
@ -26669,17 +26669,23 @@
|
|||
"type": "string"
|
||||
},
|
||||
"top_logprobs": {
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/OutputTextTopLogprob"
|
||||
},
|
||||
"title": "Top Logprobs",
|
||||
"type": "array"
|
||||
"anyOf": [
|
||||
{
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/OutputTextTopLogprob"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Top Logprobs"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"token",
|
||||
"logprob",
|
||||
"top_logprobs"
|
||||
"logprob"
|
||||
],
|
||||
"title": "OutputTextLogprob",
|
||||
"type": "object"
|
||||
|
|
@ -26702,18 +26708,28 @@
|
|||
"title": "Bytes"
|
||||
},
|
||||
"logprob": {
|
||||
"title": "Logprob",
|
||||
"type": "number"
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Logprob"
|
||||
},
|
||||
"token": {
|
||||
"title": "Token",
|
||||
"type": "string"
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Token"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"token",
|
||||
"logprob"
|
||||
],
|
||||
"title": "OutputTextTopLogprob",
|
||||
"type": "object"
|
||||
},
|
||||
|
|
|
|||
|
|
@ -7,6 +7,10 @@ import litellm
|
|||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value, is_nested_path
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.litellm_core_utils.reasoning_effort_capability import (
|
||||
DeploymentReasoningEffort,
|
||||
reasoning_effort_needs_allowing,
|
||||
)
|
||||
from litellm.llms.azure_ai.chat.transformation import AzureAIGPT5Config
|
||||
from litellm.llms.azure_ai.common_utils import (
|
||||
azure_ai_supports_native_responses,
|
||||
|
|
@ -126,6 +130,23 @@ def bridge_check_request_params(request_params: Mapping[str, object]) -> Mapping
|
|||
return {**kept, **reduce(delete_nested_value, nested_paths, {"extra_body": kept_extra_body})}
|
||||
|
||||
|
||||
def bridge_allowed_params(
|
||||
*,
|
||||
bridges_to_responses_api: bool,
|
||||
chat_reasoning_effort: DeploymentReasoningEffort,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
base_model: str | None,
|
||||
) -> tuple[str, ...]:
|
||||
if bridges_to_responses_api and custom_llm_provider in RESPONSES_WEB_SEARCH_PROVIDERS:
|
||||
return ("reasoning_effort", "web_search_options")
|
||||
if bridges_to_responses_api or reasoning_effort_needs_allowing(
|
||||
chat_reasoning_effort, model=model, custom_llm_provider=custom_llm_provider, base_model=base_model
|
||||
):
|
||||
return ("reasoning_effort",)
|
||||
return ()
|
||||
|
||||
|
||||
def bridge_reasoning_effort(
|
||||
reasoning_effort: str | Mapping[str, object] | None, extra_body: object
|
||||
) -> str | Mapping[str, object] | None:
|
||||
|
|
|
|||
|
|
@ -88,8 +88,8 @@ class CachedTokensDetails(LiteLLMBaseModel):
|
|||
|
||||
|
||||
class OutputTextTopLogprob(TypedDict):
|
||||
token: ReadOnly[str]
|
||||
logprob: ReadOnly[float]
|
||||
token: NotRequired[ReadOnly[str | None]]
|
||||
logprob: NotRequired[ReadOnly[float | None]]
|
||||
bytes: NotRequired[ReadOnly[Sequence[int] | None]]
|
||||
|
||||
|
||||
|
|
@ -97,7 +97,7 @@ class OutputTextLogprob(TypedDict):
|
|||
token: ReadOnly[str]
|
||||
logprob: ReadOnly[float]
|
||||
bytes: NotRequired[ReadOnly[Sequence[int] | None]]
|
||||
top_logprobs: ReadOnly[Sequence[OutputTextTopLogprob]]
|
||||
top_logprobs: NotRequired[ReadOnly[Sequence[OutputTextTopLogprob] | None]]
|
||||
|
||||
|
||||
class CarriesOutputTextLogprobs(BaseLiteLLMOpenAIResponseObject): # frozen-ok: guardrails rewrite these in place
|
||||
|
|
|
|||
|
|
@ -244,6 +244,35 @@ def test_redaction_hides_a_compaction_summary_in_responses_api_output(shape: str
|
|||
assert SECRET in str(result)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("token_bytes", [list(SECRET.encode()), None], ids=["utf8_bytes", "null_bytes"])
|
||||
@pytest.mark.parametrize("shape", ["object", "dict"])
|
||||
def test_redaction_hides_the_answer_tokens_in_responses_api_logprobs(shape: str, token_bytes: list[int] | None) -> None:
|
||||
top: Final = {"token": SECRET, "bytes": token_bytes, "logprob": -0.2}
|
||||
logprob: Final = {"token": SECRET, "bytes": token_bytes, "logprob": -0.1, "top_logprobs": [top]}
|
||||
response: Final = ResponsesAPIResponse(
|
||||
id="resp_1",
|
||||
created_at=1,
|
||||
model="gpt-5.6-sol",
|
||||
object="response",
|
||||
status="completed",
|
||||
output=[
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_1",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{"type": "output_text", "text": SECRET, "annotations": [], "logprobs": [logprob]}],
|
||||
}
|
||||
],
|
||||
)
|
||||
result: Final = response if shape == "object" else response.model_dump()
|
||||
|
||||
redacted: Final = perform_redaction({}, result)
|
||||
|
||||
assert SECRET not in str(redacted), redacted
|
||||
assert SECRET in str(result)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("surface", ("typed", "dict", "standard", "callback", "helper"))
|
||||
def test_responses_redaction_removes_instructions_without_changing_the_response(surface: str) -> None:
|
||||
response: Final = litellm.ResponsesAPIResponse.model_validate(
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from litellm.types.llms.openai import (
|
|||
OpenAIModerationResponse,
|
||||
OpenAIVideoObject,
|
||||
OutputTextDeltaEvent,
|
||||
OutputTextDoneEvent,
|
||||
ResponseCompletedEvent,
|
||||
ResponsesAPIResponse,
|
||||
ResponsesAPIStreamEvents,
|
||||
|
|
@ -115,6 +116,35 @@ def test_content_part_done_preserves_typed_text_with_optional_logprobs(
|
|||
assert event.part.logprobs == logprobs
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"logprob",
|
||||
(
|
||||
{"token": "hi", "logprob": -0.1},
|
||||
{"token": "hi", "logprob": -0.1, "top_logprobs": None},
|
||||
{"token": "hi", "logprob": -0.1, "top_logprobs": [{"token": None, "logprob": None}]},
|
||||
),
|
||||
ids=("no_top_logprobs", "null_top_logprobs", "null_top_entry"),
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("event_type", "text_fields"),
|
||||
(
|
||||
(OutputTextDeltaEvent, {"type": "response.output_text.delta", "delta": "hi"}),
|
||||
(OutputTextDoneEvent, {"type": "response.output_text.done", "text": "hi"}),
|
||||
),
|
||||
ids=("delta", "done"),
|
||||
)
|
||||
def test_text_events_accept_every_logprob_shape_the_openai_sdk_allows(
|
||||
event_type: type[OutputTextDeltaEvent] | type[OutputTextDoneEvent],
|
||||
text_fields: Mapping[str, str],
|
||||
logprob: Mapping[str, object],
|
||||
) -> None:
|
||||
event: Final = event_type.model_validate(
|
||||
{**text_fields, "item_id": "msg_1", "output_index": 0, "content_index": 0, "logprobs": [logprob]}
|
||||
)
|
||||
|
||||
assert event.model_dump()["logprobs"] == [logprob]
|
||||
|
||||
|
||||
class TestResponsesAPIResponseOutputText:
|
||||
"""Tests for the output_text property on ResponsesAPIResponse"""
|
||||
|
||||
|
|
|
|||
6
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
6
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -40910,7 +40910,7 @@ export interface components {
|
|||
/** Token */
|
||||
token: string;
|
||||
/** Top Logprobs */
|
||||
top_logprobs: components["schemas"]["OutputTextTopLogprob"][];
|
||||
top_logprobs?: components["schemas"]["OutputTextTopLogprob"][] | null;
|
||||
} & {
|
||||
[key: string]: unknown;
|
||||
};
|
||||
|
|
@ -40919,9 +40919,9 @@ export interface components {
|
|||
/** Bytes */
|
||||
bytes?: number[] | null;
|
||||
/** Logprob */
|
||||
logprob: number;
|
||||
logprob?: number | null;
|
||||
/** Token */
|
||||
token: string;
|
||||
token?: string | null;
|
||||
} & {
|
||||
[key: string]: unknown;
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue