fix(responses): preserve multi-turn assistant message history across transformations and guardrails (#47918)

This commit is contained in:
ryder777777 2026-10-22 14:11:16 +00:00
parent f2118fc767
commit 98fa0dc010
7 changed files with 205 additions and 33 deletions

View file

@ -1572,9 +1572,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
result = [self._convert_content_str_to_input_text(content, role)]
verbose_logger.debug("Chat provider: String content -> %s", result)
return result
elif isinstance(content, list):
elif isinstance(content, Iterable) and not isinstance(content, (bytes, bytearray, Mapping)):
result = []
for i, item in enumerate(content):
for i, raw_item in enumerate(content):
item = raw_item.model_dump(exclude_none=True) if isinstance(raw_item, BaseModel) else raw_item
verbose_logger.debug("Chat provider: Processing content item %s: %s = %s", i, type(item), item)
if isinstance(item, str):
converted = self._convert_content_str_to_input_text(item, role)

View file

@ -74,6 +74,7 @@ from litellm.responses.litellm_completion_transformation.custom_tools import wit
from litellm.responses.litellm_completion_transformation.tool_search import is_server_tool_search_tool
from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
with_compaction_items_as_params,
)
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.openai import (
@ -631,9 +632,12 @@ class OpenAIResponsesHandler(BaseTranslation):
Handles both string input and list of message objects.
"""
input_data: Final[str | ResponseInputParam | None] = data.get("input")
if not isinstance(input_data, (str, list)):
raw_input_data: Final[str | ResponseInputParam | None] = data.get("input")
if not isinstance(raw_input_data, (str, list)):
return data
input_data: Final = with_compaction_items_as_params(raw_input_data)
if input_data is not raw_input_data:
data["input"] = input_data # rebind-ok: materializes SDK output models and iterators in place
skip_system: Final = effective_skip_system_message_for_guardrail(guardrail_to_apply)
structured_messages: Final = self.get_structured_messages(data)
scoped_indices: Final = scoped_structured_message_indices(

View file

@ -1,4 +1,4 @@
from collections.abc import Callable, Collection, Mapping, Sequence
from collections.abc import Callable, Collection, Iterable, Mapping, Sequence
from functools import lru_cache
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Protocol, cast, get_type_hints
@ -747,6 +747,26 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
]
return tools if all(new is old for new, old in zip(sanitized, tools, strict=True)) else sanitized
@staticmethod
def _materialize_input_value(value: object) -> object:
if isinstance(value, BaseModel):
return value.model_dump(exclude_none=True)
if isinstance(value, dict):
materialized_dict: Final = {
key: OpenAIResponsesAPIConfig._materialize_input_value(val) for key, val in value.items()
}
return value if all(materialized_dict[key] is value[key] for key in value) else materialized_dict
if isinstance(value, list):
materialized_list: Final = [OpenAIResponsesAPIConfig._materialize_input_value(part) for part in value]
return (
value
if all(new is old for new, old in zip(materialized_list, value, strict=True))
else materialized_list
)
if isinstance(value, Iterable) and not isinstance(value, (str, bytes, bytearray, Mapping)):
return [OpenAIResponsesAPIConfig._materialize_input_value(part) for part in value]
return value
def _validate_input_param(self, input: str | ResponseInputParam) -> str | ResponseInputParam:
"""
Ensure all input fields if pydantic are converted to dict
@ -756,7 +776,8 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
"""
if isinstance(input, list):
validated_input: Final = []
for item in input:
for raw_item in input:
item = self._materialize_input_value(raw_item)
# if it's pydantic, convert to dict
if isinstance(item, BaseModel):
validated_input.append(item.model_dump(exclude_none=True))

View file

@ -8,7 +8,7 @@
import fnmatch
import os
from collections.abc import Callable, Mapping, Sequence
from typing import TYPE_CHECKING, Final, Literal, Optional
from typing import TYPE_CHECKING, Final, Literal, Optional, cast
import httpx
from pydantic import JsonValue, TypeAdapter, ValidationError
@ -590,11 +590,20 @@ class GenericGuardrailAPI(CustomGuardrail):
)
headers: Final = self._build_request_headers()
request_payload: Final = self._request_body(guardrail_request, structured_messages)
shown_messages_raw: Final = request_payload.get("structured_messages")
shown_messages: Final = (
cast( # cast-ok: JSON-serialized structured_messages from GenericGuardrailAPIRequest
Sequence[AllMessageValues], shown_messages_raw
)
if isinstance(shown_messages_raw, list)
else guardrail_request.structured_messages
)
# Make the API request
# Use mode="json" to ensure all iterables are converted to lists
response: Final = await self._post_check(
self._request_body(guardrail_request, structured_messages),
request_payload,
headers,
self.num_retries,
)
@ -623,7 +632,7 @@ class GenericGuardrailAPI(CustomGuardrail):
images=images,
tools=tools,
structured_messages=structured_messages,
shown_messages=guardrail_request.structured_messages,
shown_messages=shown_messages,
guardrail_response=guardrail_response,
)

View file

@ -34,7 +34,7 @@ from openai.types.responses.response_create_params import ResponseInputParam
from openai.types.responses.tool_choice_custom_param import ToolChoiceCustomParam
from openai.types.responses.tool_choice_function_param import ToolChoiceFunctionParam
from openai.types.responses.tool_param import FunctionToolParam
from pydantic import TypeAdapter, ValidationError
from pydantic import BaseModel, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
import litellm
@ -474,7 +474,17 @@ def without_item_cache_control(input: str | ResponseInputParam) -> str | Respons
def _as_input_param(item: object) -> object:
return item.model_dump(exclude_none=True) if isinstance(item, ResponseCompactionItem) else item
if isinstance(item, BaseModel):
return item.model_dump(exclude_none=True)
if isinstance(item, dict):
materialized_dict: Final = {key: _as_input_param(value) for key, value in item.items()}
return item if all(materialized_dict[key] is item[key] for key in item) else materialized_dict
if isinstance(item, list):
materialized_list: Final = [_as_input_param(part) for part in item]
return item if all(new is old for new, old in zip(materialized_list, item, strict=True)) else materialized_list
if isinstance(item, Iterable) and not isinstance(item, (str, bytes, bytearray, Mapping)):
return [_as_input_param(part) for part in item]
return item
def with_compaction_items_as_params(input: str | ResponseInputParam) -> str | ResponseInputParam:
@ -1339,7 +1349,10 @@ class LiteLLMCompletionResponsesConfig:
messages.append(ChatCompletionUserMessage(role="user", content=input))
elif isinstance(input, list):
existing_tool_call_ids: Final[set[str]] = set()
for _input in input:
for _raw_input in input:
_input = cast( # cast-ok: _as_input_param converts SDK output models and iterators to param dicts
Mapping[str, object], _as_input_param(_raw_input)
)
chat_completion_messages = (
LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message(
input_item=_input,
@ -2061,35 +2074,38 @@ class LiteLLMCompletionResponsesConfig:
- ResponseReasoningItemParam
- ItemReference
"""
if input_item.get("type") == "web_search_call":
search: Final = ResponseFunctionWebSearch.model_validate(input_item)
normalized_item: Final = cast( # cast-ok: _as_input_param converts SDK output models to param dicts
Mapping[str, object], _as_input_param(input_item)
)
if normalized_item.get("type") == "web_search_call":
search: Final = ResponseFunctionWebSearch.model_validate(normalized_item)
return [
GenericChatCompletionMessage(
role="assistant",
content="Hosted web search: " + search.model_dump_json(exclude_none=True),
)
]
if LiteLLMCompletionResponsesConfig._is_input_item_tool_call_output(input_item):
if LiteLLMCompletionResponsesConfig._is_input_item_tool_call_output(normalized_item):
# handle executed tool call results
return (
LiteLLMCompletionResponsesConfig._transform_responses_api_tool_call_output_to_chat_completion_message(
tool_call_output=input_item
tool_call_output=normalized_item
)
)
elif LiteLLMCompletionResponsesConfig._is_input_item_function_call(input_item):
elif LiteLLMCompletionResponsesConfig._is_input_item_function_call(normalized_item):
# handle function call input items
return LiteLLMCompletionResponsesConfig._transform_responses_api_function_call_to_chat_completion_message(
function_call=cast( # cast-ok: callee coerces every field it reads with `or ""` / str()
Mapping[str, str], input_item
Mapping[str, str], normalized_item
)
)
elif input_item.get("type") == "compaction":
elif normalized_item.get("type") == "compaction":
return list(
LiteLLMCompletionResponsesConfig._transform_compaction_item_to_chat_completion_messages(
input_item, native_compaction
normalized_item, native_compaction
)
)
elif input_item.get("type") == "reasoning":
elif normalized_item.get("type") == "reasoning":
# A ResponseReasoningItemParam carries the prior-turn chain-of-thought.
# Chat-completions providers (DeepSeek V4, Kimi K2.6, ...) expect this
# to be replayed as `reasoning_content` on an assistant message, not as
@ -2103,26 +2119,26 @@ class LiteLLMCompletionResponsesConfig:
# would replay; an empty or block-only `content` falls back to
# the summary text, which is what that branch replays instead.
inspectable: Final[object] = (
input_item.get("content")
if LiteLLMCompletionResponsesConfig._reasoning_text_from_content(input_item) is not None
else LiteLLMCompletionResponsesConfig._reasoning_text_from_summary(input_item)
or input_item.get("content")
normalized_item.get("content")
if LiteLLMCompletionResponsesConfig._reasoning_text_from_content(normalized_item) is not None
else LiteLLMCompletionResponsesConfig._reasoning_text_from_summary(normalized_item)
or normalized_item.get("content")
)
if inspectable is None:
return []
return [
GenericChatCompletionMessage(
role=_input_item_role(input_item),
role=_input_item_role(normalized_item),
content=LiteLLMCompletionResponsesConfig._transform_responses_api_content_to_chat_completion_content(
inspectable
),
)
]
reasoning_text = LiteLLMCompletionResponsesConfig._extract_reasoning_text_from_input_item( # rebind-ok: extraction result
input_item
normalized_item
)
thinking_blocks = LiteLLMCompletionResponsesConfig._decode_thinking_blocks_from_input_item( # rebind-ok: extraction result
input_item
normalized_item
)
if not reasoning_text and not thinking_blocks:
return []
@ -2133,12 +2149,12 @@ class LiteLLMCompletionResponsesConfig:
)
]
else:
content: Final[object] = input_item.get("content")
content: Final[object] = normalized_item.get("content")
# Handle None content: Responses API allows None content, but GenericChatCompletionMessage requires content
# Since guardrails skip None content anyway, we return empty list to exclude it from structured messages
if content is None:
return []
return [LiteLLMCompletionResponsesConfig._transform_responses_api_message_item(input_item, content)]
return [LiteLLMCompletionResponsesConfig._transform_responses_api_message_item(normalized_item, content)]
@staticmethod
def _transform_compaction_item_to_chat_completion_messages(
@ -2535,9 +2551,10 @@ class LiteLLMCompletionResponsesConfig:
return ""
elif isinstance(content, str):
return content
elif isinstance(content, list):
elif isinstance(content, Iterable) and not isinstance(content, (bytes, bytearray, Mapping)):
content_list: Final[list[str | dict[str, object]]] = []
for item in content:
for raw_item in content:
item = raw_item.model_dump(exclude_none=True) if isinstance(raw_item, BaseModel) else raw_item
if isinstance(item, str):
content_list.append(item)
elif isinstance(item, dict):
@ -3523,7 +3540,11 @@ class LiteLLMCompletionResponsesConfig:
messages: Final[list[AllMessageValues | GenericChatCompletionMessage | ChatCompletionMessageToolCall]] = []
output_items: Final = responses_api_output.output
for _output_item in output_items:
output_item: dict = dict(_output_item)
output_item: dict = (
_output_item.model_dump(exclude_none=True)
if isinstance(_output_item, BaseModel)
else dict(_output_item)
)
if output_item.get("type") == "function_call":
# handle function call output
messages.append(

View file

@ -258,3 +258,65 @@ def test_translate_responses_chunk_passthrough_chat_completion_chunk():
assert result.choices[0].delta.content == "Hi! How can I help?"
assert result.choices[0].finish_reason is None
def test_multi_turn_responses_input_preserves_assistant_turns_for_models_and_validated_params():
from openai.types.responses import ResponseInputParam, ResponseOutputMessage, ResponseOutputText
from pydantic import TypeAdapter
from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
)
from litellm.types.responses.main import GenericResponseOutputItem, OutputText
validated_input = TypeAdapter(ResponseInputParam).validate_python(
[
{"role": "user", "content": [{"type": "input_text", "text": "Turn 1 user"}]},
{
"type": "message",
"id": "msg_1",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "Turn 1 assistant", "annotations": []}],
},
{"role": "user", "content": [{"type": "input_text", "text": "Turn 2 user"}]},
]
)
messages_from_validated = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
input=validated_input,
responses_api_request={},
)
assert messages_from_validated == [
{"role": "user", "content": [{"type": "text", "text": "Turn 1 user"}]},
{"role": "assistant", "content": [{"type": "text", "text": "Turn 1 assistant"}]},
{"role": "user", "content": [{"type": "text", "text": "Turn 2 user"}]},
]
messages_from_sdk_items = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
input=[
{"role": "user", "content": "Turn 1 user"},
GenericResponseOutputItem(
type="message",
id="msg_1",
status="completed",
role="assistant",
content=[OutputText(type="output_text", text="Turn 1 assistant", annotations=[])],
),
ResponseOutputMessage(
id="msg_2",
type="message",
role="assistant",
status="completed",
content=[ResponseOutputText(type="output_text", text="Turn 1b assistant", annotations=[])],
),
{"role": "user", "content": "Turn 2 user"},
],
responses_api_request={},
)
assert messages_from_sdk_items == [
{"role": "user", "content": "Turn 1 user"},
{"role": "assistant", "content": [{"type": "text", "text": "Turn 1 assistant"}]},
{"role": "assistant", "content": [{"type": "text", "text": "Turn 1b assistant"}]},
{"role": "user", "content": "Turn 2 user"},
]

View file

@ -2621,3 +2621,57 @@ class TestNumRetries:
assert sent.qsize() == num_retries + 1
assert result == {"texts": ["hi"]}
@pytest.mark.asyncio
async def test_echoed_multi_turn_block_content_assistant_row_is_recognized_as_shown(self) -> None:
from litellm.llms.openai.responses.guardrail_translation.handler import OpenAIResponsesHandler
from litellm.types.responses.main import GenericResponseOutputItem, OutputText
assistant_item: Final = GenericResponseOutputItem(
type="message",
id="msg_1",
status="completed",
role="assistant",
content=[OutputText(type="output_text", text="Turn 1 assistant", annotations=[])],
)
echoed_structured_messages: Final = [
{"role": "user", "content": "Turn 1 user"},
{"role": "assistant", "content": [{"type": "text", "text": "Turn 1 assistant"}]},
{"role": "user", "content": "Turn 2 user"},
]
transport, sent = _scripted(
partial(
httpx.Response,
200,
json={
"action": "GUARDRAIL_INTERVENED",
"texts": ["Turn 1 user", "Turn 1 assistant", "Turn 2 user redacted"],
"structured_messages": echoed_structured_messages,
},
)
)
guardrail: Final = _guardrail_on(transport, num_retries=0)
handler: Final = OpenAIResponsesHandler()
data: Final[dict[str, object]] = {
"model": "gpt-4o",
"input": [
{"role": "user", "content": "Turn 1 user"},
assistant_item,
{"role": "user", "content": "Turn 2 user"},
],
}
processed: Final = await handler.process_input_messages(data=data, guardrail_to_apply=guardrail)
assert sent.qsize() == 1
assert processed["input"] == [
{"role": "user", "content": "Turn 1 user"},
{
"type": "message",
"id": "msg_1",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "Turn 1 assistant", "annotations": []}],
},
{"role": "user", "content": "Turn 2 user redacted"},
]