mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge 1d01c54b95 into 6d93445350
This commit is contained in:
commit
c7102448b0
4 changed files with 186 additions and 23 deletions
|
|
@ -75,6 +75,7 @@ from litellm.responses.litellm_completion_transformation.custom_tools import wit
|
|||
from litellm.responses.litellm_completion_transformation.tool_search import is_server_tool_search_tool
|
||||
from litellm.responses.litellm_completion_transformation.transformation import (
|
||||
LiteLLMCompletionResponsesConfig,
|
||||
ResponsesInput,
|
||||
)
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.llms.openai import (
|
||||
|
|
@ -617,10 +618,12 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
`instructions` into chat completion messages.
|
||||
"""
|
||||
input_data: Final = data.get("input")
|
||||
if input_data is None:
|
||||
if not isinstance(input_data, (str, dict, list, tuple)):
|
||||
return None
|
||||
messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
|
||||
input=input_data,
|
||||
input=cast( # cast-ok: runtime container shape is validated immediately above before Responses-specific parsing enforces each nested item type.
|
||||
"ResponsesInput", input_data
|
||||
),
|
||||
responses_api_request=data,
|
||||
)
|
||||
return cast(list[AllMessageValues], messages) if messages else None
|
||||
|
|
@ -636,9 +639,12 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
|
||||
Handles both string input and list of message objects.
|
||||
"""
|
||||
input_data: Final[str | ResponseInputParam | None] = data.get("input")
|
||||
if not isinstance(input_data, (str, list)):
|
||||
input_data: Final = data.get("input")
|
||||
if not isinstance(input_data, (str, dict, list, tuple)):
|
||||
return data
|
||||
typed_input: Final = cast( # cast-ok: runtime container shape is validated immediately above before Responses-specific parsing enforces each nested item type.
|
||||
"ResponsesInput", input_data
|
||||
)
|
||||
skip_system: Final = effective_skip_system_message_for_guardrail(guardrail_to_apply)
|
||||
structured_messages: Final = self.get_structured_messages(data)
|
||||
scoped_indices: Final = scoped_structured_message_indices(
|
||||
|
|
@ -661,7 +667,7 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
)
|
||||
)
|
||||
extracted: Final = self._extract_guardrail_inputs(
|
||||
data, input_data, flattened_tool_groups, skip_system=skip_system
|
||||
data, typed_input, flattened_tool_groups, skip_system=skip_system
|
||||
)
|
||||
if not extracted.inputs.get("texts"):
|
||||
return data
|
||||
|
|
@ -691,14 +697,14 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
else:
|
||||
data["instructions"] = written_back.instructions # rebind-ok: data is an out-param
|
||||
else:
|
||||
await self._apply_guardrailed_texts(data, input_data, extracted, guardrail_to_apply, guardrailed_inputs)
|
||||
await self._apply_guardrailed_texts(data, typed_input, extracted, guardrail_to_apply, guardrailed_inputs)
|
||||
verbose_proxy_logger.debug("OpenAI Responses API: Processed input messages: %s", data.get("input"))
|
||||
return data
|
||||
|
||||
async def _apply_guardrailed_texts(
|
||||
self,
|
||||
data: dict[str, object],
|
||||
input_data: "str | ResponseInputParam",
|
||||
input_data: ResponsesInput,
|
||||
extracted: _ExtractedInputs,
|
||||
guardrail_to_apply: "CustomGuardrail",
|
||||
guardrailed_inputs: GenericGuardrailAPIInputs,
|
||||
|
|
@ -718,7 +724,7 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
data["input"] = input_texts[0] # rebind-ok: data is an out-param
|
||||
return
|
||||
await self._apply_guardrail_responses_to_input(
|
||||
messages=input_data,
|
||||
messages=(input_data,) if isinstance(input_data, dict) else input_data,
|
||||
responses=input_texts,
|
||||
task_mappings=extracted.task_mappings,
|
||||
)
|
||||
|
|
@ -726,7 +732,7 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
def _extract_guardrail_inputs(
|
||||
self,
|
||||
data: Mapping[str, object],
|
||||
input_data: "str | ResponseInputParam",
|
||||
input_data: ResponsesInput,
|
||||
flattened_tool_groups: Sequence[Sequence[Mapping[str, object]]],
|
||||
*,
|
||||
skip_system: bool = False,
|
||||
|
|
@ -747,7 +753,8 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
if isinstance(input_data, str):
|
||||
texts_to_check.append(input_data)
|
||||
else:
|
||||
for msg_idx, message in enumerate(input_data):
|
||||
input_messages: Final = (input_data,) if isinstance(input_data, dict) else input_data
|
||||
for msg_idx, message in enumerate(input_messages):
|
||||
if role_out_of_guardrail_scope(
|
||||
_input_item_role(message), skip_system_message=skip_system, skip_tool_message=False
|
||||
):
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from typing import (
|
|||
Any,
|
||||
Final,
|
||||
Literal,
|
||||
NoReturn,
|
||||
Protocol,
|
||||
TypeAlias,
|
||||
TypeVar,
|
||||
|
|
@ -28,14 +29,18 @@ from openai.types.chat.chat_completion_named_tool_choice_param import (
|
|||
from openai.types.chat.chat_completion_named_tool_choice_param import (
|
||||
Function as NamedToolChoiceFunction,
|
||||
)
|
||||
from openai.types.responses import ResponseFunctionToolCall, ResponseFunctionWebSearch
|
||||
from openai.types.responses import (
|
||||
ResponseFunctionToolCall,
|
||||
ResponseFunctionWebSearch,
|
||||
ResponseInputItemParam,
|
||||
)
|
||||
from openai.types.responses.response_compaction_item import ResponseCompactionItem
|
||||
from openai.types.responses.response_create_params import ResponseInputParam
|
||||
from openai.types.responses.tool_choice_custom_param import ToolChoiceCustomParam
|
||||
from openai.types.responses.tool_choice_function_param import ToolChoiceFunctionParam
|
||||
from openai.types.responses.tool_param import FunctionToolParam
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
from typing_extensions import ReadOnly, TypedDict, assert_never
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -166,6 +171,7 @@ NamespaceNameMap: TypeAlias = Mapping[str, tuple[str, str]]
|
|||
NamespaceTool: TypeAlias = Mapping[str, object]
|
||||
ResponseTools: TypeAlias = Sequence[Mapping[str, object]] | None
|
||||
ChatToolParam: TypeAlias = ChatCompletionToolParam | OpenAIMcpServerTool | AnthropicToolSearchToolRegex
|
||||
ResponsesInput: TypeAlias = str | ResponseInputItemParam | ResponseInputParam | tuple[ResponseInputItemParam, ...]
|
||||
NAMESPACE_DESCRIPTION_SEPARATOR: Final = "\n\n"
|
||||
NAMESPACE_MEMBER_TYPES_WITH_CHAT_TOOLS: Final = frozenset({"function", "custom"})
|
||||
_INCOMPLETE_REASON_BY_FINISH_REASON: Final[Mapping[str, Literal["max_output_tokens", "content_filter"]]] = (
|
||||
|
|
@ -225,6 +231,14 @@ class ResponsesReasoningChatForm:
|
|||
summary: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _InvalidResponseInputType:
|
||||
input_type: str
|
||||
|
||||
|
||||
_ResponseInputTransformError: TypeAlias = _InvalidResponseInputType
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from openai.types.responses.response_apply_patch_tool_call import (
|
||||
ResponseApplyPatchToolCall,
|
||||
|
|
@ -465,6 +479,20 @@ def _without_cache_control_key(item: object) -> object:
|
|||
return {key: value for key, value in _STR_KEY_DICT_ADAPTER.validate_python(item).items() if key != "cache_control"}
|
||||
|
||||
|
||||
def _as_input_item_list(input: ResponsesInput | None) -> str | ResponseInputParam:
|
||||
# A single input item, or a tuple of them, is read as the list that the compaction and
|
||||
# cache_control steps and the per-item transform expect. A request without input has no
|
||||
# input items (streamed usage estimates read such requests). Any other type is left for
|
||||
# the transform to reject.
|
||||
if input is None:
|
||||
return []
|
||||
if isinstance(input, dict):
|
||||
return [input]
|
||||
if isinstance(input, tuple):
|
||||
return list(input)
|
||||
return input
|
||||
|
||||
|
||||
def without_item_cache_control(input: str | ResponseInputParam) -> str | ResponseInputParam:
|
||||
if not isinstance(input, list):
|
||||
return input
|
||||
|
|
@ -980,7 +1008,7 @@ class LiteLLMCompletionResponsesConfig:
|
|||
@staticmethod
|
||||
def transform_responses_api_request_to_chat_completion_request(
|
||||
model: str,
|
||||
input: str | ResponseInputParam,
|
||||
input: ResponsesInput,
|
||||
responses_api_request: ResponsesAPIOptionalRequestParams,
|
||||
custom_llm_provider: str | None = None,
|
||||
stream: bool | None = None,
|
||||
|
|
@ -1030,7 +1058,7 @@ class LiteLLMCompletionResponsesConfig:
|
|||
keeps_item_marks: Final = _target_keeps_item_marks(
|
||||
model, custom_llm_provider, drops_unsupported_params(request_kwargs.get("drop_params"))
|
||||
)
|
||||
request_input: Final = with_compaction_items_as_params(input)
|
||||
request_input: Final = with_compaction_items_as_params(_as_input_item_list(input))
|
||||
native_compaction: Final = _replays_compaction_block(
|
||||
request_input, context_management
|
||||
) and provider_takes_compaction_blocks(model, custom_llm_provider)
|
||||
|
|
@ -1098,7 +1126,7 @@ class LiteLLMCompletionResponsesConfig:
|
|||
|
||||
@staticmethod
|
||||
def transform_responses_api_input_to_messages(
|
||||
input: str | ResponseInputParam,
|
||||
input: ResponsesInput | None,
|
||||
responses_api_request: ResponsesAPIOptionalRequestParams | dict,
|
||||
replay_reasoning: bool = False,
|
||||
item_cache_control: bool = False,
|
||||
|
|
@ -1153,15 +1181,18 @@ class LiteLLMCompletionResponsesConfig:
|
|||
)
|
||||
)
|
||||
|
||||
dumped_input: Final = with_compaction_items_as_params(input)
|
||||
dumped_input: Final = with_compaction_items_as_params(_as_input_item_list(input))
|
||||
read_input: Final = _input_to_read(dumped_input, only_since_compaction, native_compaction)
|
||||
messages.extend(
|
||||
transformed_input: Final = (
|
||||
LiteLLMCompletionResponsesConfig._transform_response_input_param_to_chat_completion_message(
|
||||
input=read_input if item_cache_control else without_item_cache_control(read_input),
|
||||
replay_reasoning=replay_reasoning,
|
||||
native_compaction=native_compaction,
|
||||
)
|
||||
)
|
||||
if isinstance(transformed_input, _InvalidResponseInputType):
|
||||
LiteLLMCompletionResponsesConfig._raise_response_input_transform_error(transformed_input)
|
||||
messages.extend(transformed_input)
|
||||
|
||||
return messages
|
||||
|
||||
|
|
@ -1319,9 +1350,15 @@ class LiteLLMCompletionResponsesConfig:
|
|||
input: str | ResponseInputParam,
|
||||
replay_reasoning: bool = False,
|
||||
native_compaction: bool = False,
|
||||
) -> list[
|
||||
AllMessageValues | GenericChatCompletionMessage | ChatCompletionMessageToolCall | ChatCompletionResponseMessage
|
||||
]:
|
||||
) -> (
|
||||
list[
|
||||
AllMessageValues
|
||||
| GenericChatCompletionMessage
|
||||
| ChatCompletionMessageToolCall
|
||||
| ChatCompletionResponseMessage
|
||||
]
|
||||
| _ResponseInputTransformError
|
||||
):
|
||||
"""
|
||||
Transform a ResponseInputParam into a Chat Completion message
|
||||
|
||||
|
|
@ -1441,10 +1478,26 @@ class LiteLLMCompletionResponsesConfig:
|
|||
continue
|
||||
|
||||
messages.extend(chat_completion_messages)
|
||||
else:
|
||||
return _InvalidResponseInputType(input_type=type(input).__name__)
|
||||
if not replay_reasoning:
|
||||
return messages
|
||||
return LiteLLMCompletionResponsesConfig._merge_reasoning_only_assistant_messages(messages)
|
||||
|
||||
@staticmethod
|
||||
def _raise_response_input_transform_error(error: _ResponseInputTransformError) -> NoReturn:
|
||||
import litellm
|
||||
|
||||
match error:
|
||||
case _InvalidResponseInputType(input_type=input_type):
|
||||
raise litellm.BadRequestError(
|
||||
message=f"Invalid input type: {input_type}. Expected str, dict, list, or tuple of message items.",
|
||||
model="",
|
||||
llm_provider="",
|
||||
)
|
||||
case _:
|
||||
assert_never(error)
|
||||
|
||||
@staticmethod
|
||||
def _reasoning_only_assistant_message(
|
||||
reasoning_text: str | None,
|
||||
|
|
|
|||
|
|
@ -300,6 +300,21 @@ class TestOpenAIResponsesHandlerInputProcessing:
|
|||
assert result["input"][1]["content"] == "World [GUARDRAILED]"
|
||||
assert result["model"] == "gpt-4"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_process_input_dict_with_string_content(self):
|
||||
handler = OpenAIResponsesHandler()
|
||||
guardrail = MockGuardrail(guardrail_name="test")
|
||||
data = {
|
||||
"input": {"role": "user", "content": "Hello", "type": "message"},
|
||||
"model": "gpt-4",
|
||||
}
|
||||
|
||||
result = await handler.process_input_messages(data, guardrail)
|
||||
|
||||
result_input = result["input"]
|
||||
assert isinstance(result_input, dict)
|
||||
assert result_input["content"] == "Hello [GUARDRAILED]"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compaction_summary_is_scanned_and_rewritten_inside_its_item(self) -> None:
|
||||
affinity_prefix: Final = "litellm_enc:bW9kZWxfaWQ6ZGVwLWE=;"
|
||||
|
|
|
|||
|
|
@ -6505,6 +6505,86 @@ def test_transform_request_reads_service_tier_from_responses_api_request() -> No
|
|||
assert result["service_tier"] == "priority"
|
||||
|
||||
|
||||
def test_transform_response_input_param_dict_input():
|
||||
"""Verify that a single dictionary input item is correctly converted to a chat completion message."""
|
||||
dict_input = {"type": "message", "role": "user", "content": "hello from dict"}
|
||||
messages = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
|
||||
input=dict_input,
|
||||
responses_api_request={},
|
||||
)
|
||||
assert len(messages) == 1
|
||||
assert messages[0]["role"] == "user"
|
||||
assert messages[0]["content"] == "hello from dict"
|
||||
|
||||
|
||||
def test_transform_response_input_param_tuple_input():
|
||||
"""Verify that tuple input items are accepted like list input items."""
|
||||
tuple_input = (
|
||||
{"type": "message", "role": "user", "content": "first"},
|
||||
{"type": "message", "role": "user", "content": "second"},
|
||||
)
|
||||
messages = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
|
||||
input=tuple_input,
|
||||
responses_api_request={},
|
||||
)
|
||||
assert [message["content"] for message in messages] == ["first", "second"]
|
||||
|
||||
|
||||
def test_transform_response_input_param_invalid_type_raises_bad_request():
|
||||
"""Verify that passing invalid input types like integer raises BadRequestError."""
|
||||
import litellm
|
||||
|
||||
with pytest.raises(litellm.BadRequestError) as exc_info:
|
||||
LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
|
||||
input=123,
|
||||
responses_api_request={},
|
||||
)
|
||||
assert "Invalid input type" in str(exc_info.value)
|
||||
assert "tuple" in str(exc_info.value)
|
||||
|
||||
|
||||
def test_transform_response_input_param_single_item_drops_cache_control_like_list_input() -> None:
|
||||
item: Final = {"type": "message", "role": "user", "content": "hi", "cache_control": {"type": "ephemeral"}}
|
||||
|
||||
as_item: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
|
||||
input=item, responses_api_request={}
|
||||
)
|
||||
|
||||
assert as_item == LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
|
||||
input=[item], responses_api_request={}
|
||||
)
|
||||
assert as_item == [{"role": "user", "content": "hi"}]
|
||||
|
||||
|
||||
def test_transform_response_input_param_tuple_drops_items_before_compaction_like_list_input() -> None:
|
||||
summary: Final = {"type": "compaction", "content": "Earlier turns covered the Tokyo trip."}
|
||||
items: Final = (
|
||||
{"type": "message", "role": "user", "content": "Plan the Tokyo trip."},
|
||||
{"type": "compaction", "id": "cmp_litellm", "encrypted_content": json.dumps(summary)},
|
||||
{"type": "message", "role": "user", "content": "And Osaka?"},
|
||||
)
|
||||
|
||||
as_tuple: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
|
||||
input=items, responses_api_request={}, only_since_compaction=True
|
||||
)
|
||||
|
||||
assert as_tuple == LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
|
||||
input=list(items), responses_api_request={}, only_since_compaction=True
|
||||
)
|
||||
assert as_tuple == [
|
||||
{"role": "user", "content": "Earlier turns covered the Tokyo trip."},
|
||||
{"role": "user", "content": "And Osaka?"},
|
||||
]
|
||||
|
||||
|
||||
def test_transform_response_input_param_without_input_returns_only_instructions() -> None:
|
||||
messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
|
||||
input=None, responses_api_request={"instructions": "Be brief."}
|
||||
)
|
||||
|
||||
assert messages == [{"role": "system", "content": "Be brief."}]
|
||||
|
||||
|
||||
_COMPACTION_BLOCK: Final = {"type": "compaction", "content": "Earlier turns covered the Tokyo trip."}
|
||||
|
||||
|
||||
|
|
@ -6668,7 +6748,6 @@ def test_summary_replay_drops_the_history_its_compaction_item_replaces() -> None
|
|||
]
|
||||
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("call", "output"),
|
||||
[
|
||||
|
|
@ -6819,7 +6898,14 @@ def test_compaction_replay_invents_no_call_for_an_output_whose_call_was_never_st
|
|||
[
|
||||
pytest.param(
|
||||
_COMPACTION_REQUESTED,
|
||||
[("user", ()), ("assistant", ("call_0",)), ("tool", "Sunny."), ("assistant", ()), ("assistant", ("call_0",)), ("tool", "Rain.")],
|
||||
[
|
||||
("user", ()),
|
||||
("assistant", ("call_0",)),
|
||||
("tool", "Sunny."),
|
||||
("assistant", ()),
|
||||
("assistant", ("call_0",)),
|
||||
("tool", "Rain."),
|
||||
],
|
||||
id="native_compaction_block",
|
||||
),
|
||||
pytest.param(
|
||||
|
|
@ -6961,7 +7047,9 @@ async def test_bridge_partial_response_on_disconnect_bills_the_relayed_chat_chun
|
|||
assert incomplete_event.type == "response.incomplete"
|
||||
assert incomplete_event.response.status == "incomplete"
|
||||
assert incomplete_event.response.usage is not None
|
||||
assert incomplete_event.response.usage.output_tokens == litellm.token_counter(model="gpt-4o-mini", text=relayed_text)
|
||||
assert incomplete_event.response.usage.output_tokens == litellm.token_counter(
|
||||
model="gpt-4o-mini", text=relayed_text
|
||||
)
|
||||
assert incomplete_event.response.usage.input_tokens > 0
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue