mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(responses): preserve multi-turn assistant message history across transformations and guardrails (#47918)
This commit is contained in:
parent
f2118fc767
commit
98fa0dc010
7 changed files with 205 additions and 33 deletions
|
|
@ -1572,9 +1572,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
result = [self._convert_content_str_to_input_text(content, role)]
|
||||
verbose_logger.debug("Chat provider: String content -> %s", result)
|
||||
return result
|
||||
elif isinstance(content, list):
|
||||
elif isinstance(content, Iterable) and not isinstance(content, (bytes, bytearray, Mapping)):
|
||||
result = []
|
||||
for i, item in enumerate(content):
|
||||
for i, raw_item in enumerate(content):
|
||||
item = raw_item.model_dump(exclude_none=True) if isinstance(raw_item, BaseModel) else raw_item
|
||||
verbose_logger.debug("Chat provider: Processing content item %s: %s = %s", i, type(item), item)
|
||||
if isinstance(item, str):
|
||||
converted = self._convert_content_str_to_input_text(item, role)
|
||||
|
|
|
|||
|
|
@ -74,6 +74,7 @@ from litellm.responses.litellm_completion_transformation.custom_tools import wit
|
|||
from litellm.responses.litellm_completion_transformation.tool_search import is_server_tool_search_tool
|
||||
from litellm.responses.litellm_completion_transformation.transformation import (
|
||||
LiteLLMCompletionResponsesConfig,
|
||||
with_compaction_items_as_params,
|
||||
)
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.llms.openai import (
|
||||
|
|
@ -631,9 +632,12 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
|
||||
Handles both string input and list of message objects.
|
||||
"""
|
||||
input_data: Final[str | ResponseInputParam | None] = data.get("input")
|
||||
if not isinstance(input_data, (str, list)):
|
||||
raw_input_data: Final[str | ResponseInputParam | None] = data.get("input")
|
||||
if not isinstance(raw_input_data, (str, list)):
|
||||
return data
|
||||
input_data: Final = with_compaction_items_as_params(raw_input_data)
|
||||
if input_data is not raw_input_data:
|
||||
data["input"] = input_data # rebind-ok: materializes SDK output models and iterators in place
|
||||
skip_system: Final = effective_skip_system_message_for_guardrail(guardrail_to_apply)
|
||||
structured_messages: Final = self.get_structured_messages(data)
|
||||
scoped_indices: Final = scoped_structured_message_indices(
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
from collections.abc import Callable, Collection, Mapping, Sequence
|
||||
from collections.abc import Callable, Collection, Iterable, Mapping, Sequence
|
||||
from functools import lru_cache
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Protocol, cast, get_type_hints
|
||||
|
|
@ -747,6 +747,26 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
]
|
||||
return tools if all(new is old for new, old in zip(sanitized, tools, strict=True)) else sanitized
|
||||
|
||||
@staticmethod
|
||||
def _materialize_input_value(value: object) -> object:
|
||||
if isinstance(value, BaseModel):
|
||||
return value.model_dump(exclude_none=True)
|
||||
if isinstance(value, dict):
|
||||
materialized_dict: Final = {
|
||||
key: OpenAIResponsesAPIConfig._materialize_input_value(val) for key, val in value.items()
|
||||
}
|
||||
return value if all(materialized_dict[key] is value[key] for key in value) else materialized_dict
|
||||
if isinstance(value, list):
|
||||
materialized_list: Final = [OpenAIResponsesAPIConfig._materialize_input_value(part) for part in value]
|
||||
return (
|
||||
value
|
||||
if all(new is old for new, old in zip(materialized_list, value, strict=True))
|
||||
else materialized_list
|
||||
)
|
||||
if isinstance(value, Iterable) and not isinstance(value, (str, bytes, bytearray, Mapping)):
|
||||
return [OpenAIResponsesAPIConfig._materialize_input_value(part) for part in value]
|
||||
return value
|
||||
|
||||
def _validate_input_param(self, input: str | ResponseInputParam) -> str | ResponseInputParam:
|
||||
"""
|
||||
Ensure all input fields if pydantic are converted to dict
|
||||
|
|
@ -756,7 +776,8 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
"""
|
||||
if isinstance(input, list):
|
||||
validated_input: Final = []
|
||||
for item in input:
|
||||
for raw_item in input:
|
||||
item = self._materialize_input_value(raw_item)
|
||||
# if it's pydantic, convert to dict
|
||||
if isinstance(item, BaseModel):
|
||||
validated_input.append(item.model_dump(exclude_none=True))
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
import fnmatch
|
||||
import os
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Final, Literal, Optional
|
||||
from typing import TYPE_CHECKING, Final, Literal, Optional, cast
|
||||
|
||||
import httpx
|
||||
from pydantic import JsonValue, TypeAdapter, ValidationError
|
||||
|
|
@ -590,11 +590,20 @@ class GenericGuardrailAPI(CustomGuardrail):
|
|||
)
|
||||
|
||||
headers: Final = self._build_request_headers()
|
||||
request_payload: Final = self._request_body(guardrail_request, structured_messages)
|
||||
shown_messages_raw: Final = request_payload.get("structured_messages")
|
||||
shown_messages: Final = (
|
||||
cast( # cast-ok: JSON-serialized structured_messages from GenericGuardrailAPIRequest
|
||||
Sequence[AllMessageValues], shown_messages_raw
|
||||
)
|
||||
if isinstance(shown_messages_raw, list)
|
||||
else guardrail_request.structured_messages
|
||||
)
|
||||
|
||||
# Make the API request
|
||||
# Use mode="json" to ensure all iterables are converted to lists
|
||||
response: Final = await self._post_check(
|
||||
self._request_body(guardrail_request, structured_messages),
|
||||
request_payload,
|
||||
headers,
|
||||
self.num_retries,
|
||||
)
|
||||
|
|
@ -623,7 +632,7 @@ class GenericGuardrailAPI(CustomGuardrail):
|
|||
images=images,
|
||||
tools=tools,
|
||||
structured_messages=structured_messages,
|
||||
shown_messages=guardrail_request.structured_messages,
|
||||
shown_messages=shown_messages,
|
||||
guardrail_response=guardrail_response,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ from openai.types.responses.response_create_params import ResponseInputParam
|
|||
from openai.types.responses.tool_choice_custom_param import ToolChoiceCustomParam
|
||||
from openai.types.responses.tool_choice_function_param import ToolChoiceFunctionParam
|
||||
from openai.types.responses.tool_param import FunctionToolParam
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
from pydantic import BaseModel, TypeAdapter, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
|
|
@ -474,7 +474,17 @@ def without_item_cache_control(input: str | ResponseInputParam) -> str | Respons
|
|||
|
||||
|
||||
def _as_input_param(item: object) -> object:
|
||||
return item.model_dump(exclude_none=True) if isinstance(item, ResponseCompactionItem) else item
|
||||
if isinstance(item, BaseModel):
|
||||
return item.model_dump(exclude_none=True)
|
||||
if isinstance(item, dict):
|
||||
materialized_dict: Final = {key: _as_input_param(value) for key, value in item.items()}
|
||||
return item if all(materialized_dict[key] is item[key] for key in item) else materialized_dict
|
||||
if isinstance(item, list):
|
||||
materialized_list: Final = [_as_input_param(part) for part in item]
|
||||
return item if all(new is old for new, old in zip(materialized_list, item, strict=True)) else materialized_list
|
||||
if isinstance(item, Iterable) and not isinstance(item, (str, bytes, bytearray, Mapping)):
|
||||
return [_as_input_param(part) for part in item]
|
||||
return item
|
||||
|
||||
|
||||
def with_compaction_items_as_params(input: str | ResponseInputParam) -> str | ResponseInputParam:
|
||||
|
|
@ -1339,7 +1349,10 @@ class LiteLLMCompletionResponsesConfig:
|
|||
messages.append(ChatCompletionUserMessage(role="user", content=input))
|
||||
elif isinstance(input, list):
|
||||
existing_tool_call_ids: Final[set[str]] = set()
|
||||
for _input in input:
|
||||
for _raw_input in input:
|
||||
_input = cast( # cast-ok: _as_input_param converts SDK output models and iterators to param dicts
|
||||
Mapping[str, object], _as_input_param(_raw_input)
|
||||
)
|
||||
chat_completion_messages = (
|
||||
LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message(
|
||||
input_item=_input,
|
||||
|
|
@ -2061,35 +2074,38 @@ class LiteLLMCompletionResponsesConfig:
|
|||
- ResponseReasoningItemParam
|
||||
- ItemReference
|
||||
"""
|
||||
if input_item.get("type") == "web_search_call":
|
||||
search: Final = ResponseFunctionWebSearch.model_validate(input_item)
|
||||
normalized_item: Final = cast( # cast-ok: _as_input_param converts SDK output models to param dicts
|
||||
Mapping[str, object], _as_input_param(input_item)
|
||||
)
|
||||
if normalized_item.get("type") == "web_search_call":
|
||||
search: Final = ResponseFunctionWebSearch.model_validate(normalized_item)
|
||||
return [
|
||||
GenericChatCompletionMessage(
|
||||
role="assistant",
|
||||
content="Hosted web search: " + search.model_dump_json(exclude_none=True),
|
||||
)
|
||||
]
|
||||
if LiteLLMCompletionResponsesConfig._is_input_item_tool_call_output(input_item):
|
||||
if LiteLLMCompletionResponsesConfig._is_input_item_tool_call_output(normalized_item):
|
||||
# handle executed tool call results
|
||||
return (
|
||||
LiteLLMCompletionResponsesConfig._transform_responses_api_tool_call_output_to_chat_completion_message(
|
||||
tool_call_output=input_item
|
||||
tool_call_output=normalized_item
|
||||
)
|
||||
)
|
||||
elif LiteLLMCompletionResponsesConfig._is_input_item_function_call(input_item):
|
||||
elif LiteLLMCompletionResponsesConfig._is_input_item_function_call(normalized_item):
|
||||
# handle function call input items
|
||||
return LiteLLMCompletionResponsesConfig._transform_responses_api_function_call_to_chat_completion_message(
|
||||
function_call=cast( # cast-ok: callee coerces every field it reads with `or ""` / str()
|
||||
Mapping[str, str], input_item
|
||||
Mapping[str, str], normalized_item
|
||||
)
|
||||
)
|
||||
elif input_item.get("type") == "compaction":
|
||||
elif normalized_item.get("type") == "compaction":
|
||||
return list(
|
||||
LiteLLMCompletionResponsesConfig._transform_compaction_item_to_chat_completion_messages(
|
||||
input_item, native_compaction
|
||||
normalized_item, native_compaction
|
||||
)
|
||||
)
|
||||
elif input_item.get("type") == "reasoning":
|
||||
elif normalized_item.get("type") == "reasoning":
|
||||
# A ResponseReasoningItemParam carries the prior-turn chain-of-thought.
|
||||
# Chat-completions providers (DeepSeek V4, Kimi K2.6, ...) expect this
|
||||
# to be replayed as `reasoning_content` on an assistant message, not as
|
||||
|
|
@ -2103,26 +2119,26 @@ class LiteLLMCompletionResponsesConfig:
|
|||
# would replay; an empty or block-only `content` falls back to
|
||||
# the summary text, which is what that branch replays instead.
|
||||
inspectable: Final[object] = (
|
||||
input_item.get("content")
|
||||
if LiteLLMCompletionResponsesConfig._reasoning_text_from_content(input_item) is not None
|
||||
else LiteLLMCompletionResponsesConfig._reasoning_text_from_summary(input_item)
|
||||
or input_item.get("content")
|
||||
normalized_item.get("content")
|
||||
if LiteLLMCompletionResponsesConfig._reasoning_text_from_content(normalized_item) is not None
|
||||
else LiteLLMCompletionResponsesConfig._reasoning_text_from_summary(normalized_item)
|
||||
or normalized_item.get("content")
|
||||
)
|
||||
if inspectable is None:
|
||||
return []
|
||||
return [
|
||||
GenericChatCompletionMessage(
|
||||
role=_input_item_role(input_item),
|
||||
role=_input_item_role(normalized_item),
|
||||
content=LiteLLMCompletionResponsesConfig._transform_responses_api_content_to_chat_completion_content(
|
||||
inspectable
|
||||
),
|
||||
)
|
||||
]
|
||||
reasoning_text = LiteLLMCompletionResponsesConfig._extract_reasoning_text_from_input_item( # rebind-ok: extraction result
|
||||
input_item
|
||||
normalized_item
|
||||
)
|
||||
thinking_blocks = LiteLLMCompletionResponsesConfig._decode_thinking_blocks_from_input_item( # rebind-ok: extraction result
|
||||
input_item
|
||||
normalized_item
|
||||
)
|
||||
if not reasoning_text and not thinking_blocks:
|
||||
return []
|
||||
|
|
@ -2133,12 +2149,12 @@ class LiteLLMCompletionResponsesConfig:
|
|||
)
|
||||
]
|
||||
else:
|
||||
content: Final[object] = input_item.get("content")
|
||||
content: Final[object] = normalized_item.get("content")
|
||||
# Handle None content: Responses API allows None content, but GenericChatCompletionMessage requires content
|
||||
# Since guardrails skip None content anyway, we return empty list to exclude it from structured messages
|
||||
if content is None:
|
||||
return []
|
||||
return [LiteLLMCompletionResponsesConfig._transform_responses_api_message_item(input_item, content)]
|
||||
return [LiteLLMCompletionResponsesConfig._transform_responses_api_message_item(normalized_item, content)]
|
||||
|
||||
@staticmethod
|
||||
def _transform_compaction_item_to_chat_completion_messages(
|
||||
|
|
@ -2535,9 +2551,10 @@ class LiteLLMCompletionResponsesConfig:
|
|||
return ""
|
||||
elif isinstance(content, str):
|
||||
return content
|
||||
elif isinstance(content, list):
|
||||
elif isinstance(content, Iterable) and not isinstance(content, (bytes, bytearray, Mapping)):
|
||||
content_list: Final[list[str | dict[str, object]]] = []
|
||||
for item in content:
|
||||
for raw_item in content:
|
||||
item = raw_item.model_dump(exclude_none=True) if isinstance(raw_item, BaseModel) else raw_item
|
||||
if isinstance(item, str):
|
||||
content_list.append(item)
|
||||
elif isinstance(item, dict):
|
||||
|
|
@ -3523,7 +3540,11 @@ class LiteLLMCompletionResponsesConfig:
|
|||
messages: Final[list[AllMessageValues | GenericChatCompletionMessage | ChatCompletionMessageToolCall]] = []
|
||||
output_items: Final = responses_api_output.output
|
||||
for _output_item in output_items:
|
||||
output_item: dict = dict(_output_item)
|
||||
output_item: dict = (
|
||||
_output_item.model_dump(exclude_none=True)
|
||||
if isinstance(_output_item, BaseModel)
|
||||
else dict(_output_item)
|
||||
)
|
||||
if output_item.get("type") == "function_call":
|
||||
# handle function call output
|
||||
messages.append(
|
||||
|
|
|
|||
|
|
@ -258,3 +258,65 @@ def test_translate_responses_chunk_passthrough_chat_completion_chunk():
|
|||
assert result.choices[0].delta.content == "Hi! How can I help?"
|
||||
assert result.choices[0].finish_reason is None
|
||||
|
||||
|
||||
def test_multi_turn_responses_input_preserves_assistant_turns_for_models_and_validated_params():
|
||||
from openai.types.responses import ResponseInputParam, ResponseOutputMessage, ResponseOutputText
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.responses.litellm_completion_transformation.transformation import (
|
||||
LiteLLMCompletionResponsesConfig,
|
||||
)
|
||||
from litellm.types.responses.main import GenericResponseOutputItem, OutputText
|
||||
|
||||
validated_input = TypeAdapter(ResponseInputParam).validate_python(
|
||||
[
|
||||
{"role": "user", "content": [{"type": "input_text", "text": "Turn 1 user"}]},
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_1",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "Turn 1 assistant", "annotations": []}],
|
||||
},
|
||||
{"role": "user", "content": [{"type": "input_text", "text": "Turn 2 user"}]},
|
||||
]
|
||||
)
|
||||
messages_from_validated = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
|
||||
input=validated_input,
|
||||
responses_api_request={},
|
||||
)
|
||||
assert messages_from_validated == [
|
||||
{"role": "user", "content": [{"type": "text", "text": "Turn 1 user"}]},
|
||||
{"role": "assistant", "content": [{"type": "text", "text": "Turn 1 assistant"}]},
|
||||
{"role": "user", "content": [{"type": "text", "text": "Turn 2 user"}]},
|
||||
]
|
||||
|
||||
messages_from_sdk_items = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
|
||||
input=[
|
||||
{"role": "user", "content": "Turn 1 user"},
|
||||
GenericResponseOutputItem(
|
||||
type="message",
|
||||
id="msg_1",
|
||||
status="completed",
|
||||
role="assistant",
|
||||
content=[OutputText(type="output_text", text="Turn 1 assistant", annotations=[])],
|
||||
),
|
||||
ResponseOutputMessage(
|
||||
id="msg_2",
|
||||
type="message",
|
||||
role="assistant",
|
||||
status="completed",
|
||||
content=[ResponseOutputText(type="output_text", text="Turn 1b assistant", annotations=[])],
|
||||
),
|
||||
{"role": "user", "content": "Turn 2 user"},
|
||||
],
|
||||
responses_api_request={},
|
||||
)
|
||||
assert messages_from_sdk_items == [
|
||||
{"role": "user", "content": "Turn 1 user"},
|
||||
{"role": "assistant", "content": [{"type": "text", "text": "Turn 1 assistant"}]},
|
||||
{"role": "assistant", "content": [{"type": "text", "text": "Turn 1b assistant"}]},
|
||||
{"role": "user", "content": "Turn 2 user"},
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -2621,3 +2621,57 @@ class TestNumRetries:
|
|||
|
||||
assert sent.qsize() == num_retries + 1
|
||||
assert result == {"texts": ["hi"]}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_echoed_multi_turn_block_content_assistant_row_is_recognized_as_shown(self) -> None:
|
||||
from litellm.llms.openai.responses.guardrail_translation.handler import OpenAIResponsesHandler
|
||||
from litellm.types.responses.main import GenericResponseOutputItem, OutputText
|
||||
|
||||
assistant_item: Final = GenericResponseOutputItem(
|
||||
type="message",
|
||||
id="msg_1",
|
||||
status="completed",
|
||||
role="assistant",
|
||||
content=[OutputText(type="output_text", text="Turn 1 assistant", annotations=[])],
|
||||
)
|
||||
echoed_structured_messages: Final = [
|
||||
{"role": "user", "content": "Turn 1 user"},
|
||||
{"role": "assistant", "content": [{"type": "text", "text": "Turn 1 assistant"}]},
|
||||
{"role": "user", "content": "Turn 2 user"},
|
||||
]
|
||||
transport, sent = _scripted(
|
||||
partial(
|
||||
httpx.Response,
|
||||
200,
|
||||
json={
|
||||
"action": "GUARDRAIL_INTERVENED",
|
||||
"texts": ["Turn 1 user", "Turn 1 assistant", "Turn 2 user redacted"],
|
||||
"structured_messages": echoed_structured_messages,
|
||||
},
|
||||
)
|
||||
)
|
||||
guardrail: Final = _guardrail_on(transport, num_retries=0)
|
||||
handler: Final = OpenAIResponsesHandler()
|
||||
data: Final[dict[str, object]] = {
|
||||
"model": "gpt-4o",
|
||||
"input": [
|
||||
{"role": "user", "content": "Turn 1 user"},
|
||||
assistant_item,
|
||||
{"role": "user", "content": "Turn 2 user"},
|
||||
],
|
||||
}
|
||||
|
||||
processed: Final = await handler.process_input_messages(data=data, guardrail_to_apply=guardrail)
|
||||
assert sent.qsize() == 1
|
||||
assert processed["input"] == [
|
||||
{"role": "user", "content": "Turn 1 user"},
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_1",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "Turn 1 assistant", "annotations": []}],
|
||||
},
|
||||
{"role": "user", "content": "Turn 2 user redacted"},
|
||||
]
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue