mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-22 00:31:44 +00:00
Merge pull request #25618 from eric-liu-nvidia/nvidia/eliu/fix_litellm_translation_layer
fix: codex and /v1/responses support issue
This commit is contained in:
ishaan-berri
GitHub
commit
0df1b9dd26
2 changed files with 439 additions and 7 deletions
|
|
@ -375,6 +375,11 @@ class LiteLLMCompletionResponsesConfig:
|
|||
messages.append(ChatCompletionUserMessage(role="user", content=input))
|
||||
elif isinstance(input, list):
|
||||
existing_tool_call_ids: Set[str] = set()
|
||||
# Track whether tool result messages have been added since
|
||||
# the last assistant message. When True, a new function_call
|
||||
# means we've crossed a turn boundary and must NOT merge into
|
||||
# the previous assistant message.
|
||||
tool_results_since_last_assistant: bool = False
|
||||
for _input in input:
|
||||
chat_completion_messages = LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message(
|
||||
input_item=_input
|
||||
|
|
@ -395,8 +400,13 @@ class LiteLLMCompletionResponsesConfig:
|
|||
# assistant message, producing back-to-back assistant messages that
|
||||
# Anthropic rejects with "tool_use ids were found without
|
||||
# tool_result blocks immediately after".
|
||||
#
|
||||
# However, do NOT merge across turn boundaries. If tool
|
||||
# results have been appended since the last assistant
|
||||
# message, this function_call starts a new turn and must
|
||||
# produce a separate assistant message.
|
||||
#########################################################
|
||||
if messages:
|
||||
if messages and not tool_results_since_last_assistant:
|
||||
last_msg = messages[-1]
|
||||
last_role = (
|
||||
last_msg.get("role")
|
||||
|
|
@ -425,6 +435,12 @@ class LiteLLMCompletionResponsesConfig:
|
|||
)
|
||||
continue
|
||||
|
||||
# A new assistant message is being created (either first
|
||||
# in the conversation or after a turn boundary). Reset
|
||||
# the flag so subsequent consecutive function_calls in
|
||||
# the same turn get merged.
|
||||
tool_results_since_last_assistant = False
|
||||
|
||||
#########################################################
|
||||
# If Input Item is a Tool Call Output, add it to the tool_call_output_messages list
|
||||
# preserving the ordering of tool call outputs. Some models require the tool
|
||||
|
|
@ -433,6 +449,10 @@ class LiteLLMCompletionResponsesConfig:
|
|||
if LiteLLMCompletionResponsesConfig._is_input_item_tool_call_output(
|
||||
input_item=_input
|
||||
):
|
||||
# Mark that tool results have been seen — the next
|
||||
# function_call (if any) starts a new turn.
|
||||
tool_results_since_last_assistant = True
|
||||
|
||||
if not chat_completion_messages:
|
||||
continue
|
||||
|
||||
|
|
@ -1401,9 +1421,60 @@ class LiteLLMCompletionResponsesConfig:
|
|||
cast(ChatCompletionToolParam, chat_completion_tool)
|
||||
)
|
||||
else:
|
||||
chat_completion_tools.append(
|
||||
cast(Union[ChatCompletionToolParam, OpenAIMcpServerTool], tool)
|
||||
)
|
||||
#########################################################
|
||||
# Handle Codex CLI tool definition format.
|
||||
#
|
||||
# Codex sends tools as:
|
||||
# {id: "tool_name", inputSchema: {jsonSchema: {...}}}
|
||||
# without a "type" field. Detect this by checking for
|
||||
# "inputSchema" — the distinguishing Codex marker.
|
||||
# All other unrecognized tool types (code_execution,
|
||||
# tool_search, etc.) are passed through as-is.
|
||||
#########################################################
|
||||
if "inputSchema" in tool or (
|
||||
"id" in tool and "type" not in tool and "name" not in tool
|
||||
):
|
||||
# Tool name: prefer "name", fall back to "id" (Codex format)
|
||||
tool_name = tool.get("name") or tool.get("id", "")
|
||||
|
||||
# Skip tools with empty names (e.g. ghost entries from Codex)
|
||||
if not tool_name:
|
||||
chat_completion_tools.append(
|
||||
cast(
|
||||
Union[ChatCompletionToolParam, OpenAIMcpServerTool],
|
||||
tool,
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
# Parameters: prefer "parameters", fall back to
|
||||
# "inputSchema.jsonSchema" (Codex format)
|
||||
codex_parameters = tool.get("parameters")
|
||||
if codex_parameters is None:
|
||||
input_schema = tool.get("inputSchema")
|
||||
if isinstance(input_schema, dict):
|
||||
codex_parameters = input_schema.get("jsonSchema", {})
|
||||
else:
|
||||
codex_parameters = {}
|
||||
codex_parameters = dict(codex_parameters or {})
|
||||
if not codex_parameters or "type" not in codex_parameters:
|
||||
codex_parameters["type"] = "object"
|
||||
|
||||
codex_tool: Dict[str, Any] = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": tool_name,
|
||||
"description": tool.get("description", ""),
|
||||
"parameters": codex_parameters,
|
||||
},
|
||||
}
|
||||
chat_completion_tools.append(
|
||||
cast(ChatCompletionToolParam, codex_tool)
|
||||
)
|
||||
else:
|
||||
chat_completion_tools.append(
|
||||
cast(Union[ChatCompletionToolParam, OpenAIMcpServerTool], tool)
|
||||
)
|
||||
return chat_completion_tools, web_search_options
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -2123,9 +2194,9 @@ class LiteLLMCompletionResponsesConfig:
|
|||
hasattr(completion_details, "reasoning_tokens")
|
||||
and completion_details.reasoning_tokens is not None
|
||||
):
|
||||
output_details_dict[
|
||||
"reasoning_tokens"
|
||||
] = completion_details.reasoning_tokens
|
||||
output_details_dict["reasoning_tokens"] = (
|
||||
completion_details.reasoning_tokens
|
||||
)
|
||||
else:
|
||||
output_details_dict["reasoning_tokens"] = 0
|
||||
|
||||
|
|
|
|||
|
|
@ -2169,3 +2169,364 @@ class TestEnsureOutputItemContentPartAdded:
|
|||
|
||||
events = iterator._pending_response_events
|
||||
assert len(events) == 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests for multi-turn tool call boundary detection (Fix #1)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMultiTurnToolCallBoundary:
|
||||
"""Verify that function_call items from different turns produce separate
|
||||
assistant messages instead of being incorrectly merged into one."""
|
||||
|
||||
def test_single_turn_tool_calls_are_merged(self):
|
||||
"""Within a single turn, consecutive function_calls should merge
|
||||
into one assistant message (existing behaviour, must not regress)."""
|
||||
input_items = [
|
||||
{"role": "user", "content": "Do two things"},
|
||||
{
|
||||
"type": "function_call",
|
||||
"name": "tool_a",
|
||||
"call_id": "call_a",
|
||||
"arguments": "{}",
|
||||
},
|
||||
{
|
||||
"type": "function_call",
|
||||
"name": "tool_b",
|
||||
"call_id": "call_b",
|
||||
"arguments": "{}",
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_a",
|
||||
"output": "result_a",
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_b",
|
||||
"output": "result_b",
|
||||
},
|
||||
]
|
||||
messages = LiteLLMCompletionResponsesConfig._transform_response_input_param_to_chat_completion_message(
|
||||
input=input_items
|
||||
)
|
||||
asst_msgs = [m for m in messages if _get_role(m) == "assistant"]
|
||||
assert len(asst_msgs) == 1, "Single turn: should have 1 assistant message"
|
||||
assert len(_get_tool_calls(asst_msgs[0])) == 2
|
||||
|
||||
def test_two_turns_produce_separate_assistant_messages(self):
|
||||
"""Two turns of tool calls must produce two separate assistant messages.
|
||||
|
||||
Sequence: user → call → call → output → output → call → call → output → output
|
||||
Expected: user, assistant(2 calls), tool, tool, assistant(2 calls), tool, tool
|
||||
"""
|
||||
input_items = [
|
||||
{"role": "user", "content": "Multi-turn task"},
|
||||
# Turn 1
|
||||
{
|
||||
"type": "function_call",
|
||||
"name": "update_plan",
|
||||
"call_id": "t1_a",
|
||||
"arguments": "{}",
|
||||
},
|
||||
{
|
||||
"type": "function_call",
|
||||
"name": "apply_patch",
|
||||
"call_id": "t1_b",
|
||||
"arguments": "{}",
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "t1_a",
|
||||
"output": "Plan updated",
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "t1_b",
|
||||
"output": "Patch applied",
|
||||
},
|
||||
# Turn 2
|
||||
{
|
||||
"type": "function_call",
|
||||
"name": "exec_cmd",
|
||||
"call_id": "t2_a",
|
||||
"arguments": '{"cmd":"ls"}',
|
||||
},
|
||||
{
|
||||
"type": "function_call",
|
||||
"name": "exec_cmd",
|
||||
"call_id": "t2_b",
|
||||
"arguments": '{"cmd":"pwd"}',
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "t2_a",
|
||||
"output": "file1 file2",
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "t2_b",
|
||||
"output": "/workspace",
|
||||
},
|
||||
]
|
||||
messages = LiteLLMCompletionResponsesConfig._transform_response_input_param_to_chat_completion_message(
|
||||
input=input_items
|
||||
)
|
||||
asst_msgs = [m for m in messages if _get_role(m) == "assistant"]
|
||||
tool_msgs = [m for m in messages if _get_role(m) == "tool"]
|
||||
|
||||
assert len(asst_msgs) == 2, "Should have 2 assistant messages (one per turn)"
|
||||
assert len(_get_tool_calls(asst_msgs[0])) == 2, "Turn 1: 2 tool calls"
|
||||
assert len(_get_tool_calls(asst_msgs[1])) == 2, "Turn 2: 2 tool calls"
|
||||
assert len(tool_msgs) == 4, "Total 4 tool results"
|
||||
|
||||
# Verify ordering: user, assistant, tool, tool, assistant, tool, tool
|
||||
expected_roles = [
|
||||
"user",
|
||||
"assistant",
|
||||
"tool",
|
||||
"tool",
|
||||
"assistant",
|
||||
"tool",
|
||||
"tool",
|
||||
]
|
||||
actual_roles = [_get_role(m) for m in messages]
|
||||
assert actual_roles == expected_roles
|
||||
|
||||
def test_three_turns_produce_three_assistant_messages(self):
|
||||
"""Three turns of tool calls produce 3 separate assistant messages."""
|
||||
input_items = [
|
||||
{"role": "user", "content": "Create a script"},
|
||||
# Turn 1: 2 calls
|
||||
{
|
||||
"type": "function_call",
|
||||
"name": "update_plan",
|
||||
"call_id": "t1_a",
|
||||
"arguments": "{}",
|
||||
},
|
||||
{
|
||||
"type": "function_call",
|
||||
"name": "apply_patch",
|
||||
"call_id": "t1_b",
|
||||
"arguments": "{}",
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "t1_a",
|
||||
"output": "Plan updated",
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "t1_b",
|
||||
"output": "unsupported",
|
||||
},
|
||||
# Turn 2: 3 calls
|
||||
{
|
||||
"type": "function_call",
|
||||
"name": "exec_command",
|
||||
"call_id": "t2_a",
|
||||
"arguments": '{"cmd":"ls"}',
|
||||
},
|
||||
{
|
||||
"type": "function_call",
|
||||
"name": "exec_command",
|
||||
"call_id": "t2_b",
|
||||
"arguments": '{"cmd":"echo hi"}',
|
||||
},
|
||||
{
|
||||
"type": "function_call",
|
||||
"name": "exec_command",
|
||||
"call_id": "t2_c",
|
||||
"arguments": '{"cmd":"pwd"}',
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "t2_a",
|
||||
"output": "file1 file2",
|
||||
},
|
||||
{"type": "function_call_output", "call_id": "t2_b", "output": "hi"},
|
||||
{"type": "function_call_output", "call_id": "t2_c", "output": "/workspace"},
|
||||
# Turn 3: 2 calls
|
||||
{
|
||||
"type": "function_call",
|
||||
"name": "exec_command",
|
||||
"call_id": "t3_a",
|
||||
"arguments": '{"cmd":"cat file1"}',
|
||||
},
|
||||
{
|
||||
"type": "function_call",
|
||||
"name": "exec_command",
|
||||
"call_id": "t3_b",
|
||||
"arguments": '{"cmd":"cat file2"}',
|
||||
},
|
||||
{"type": "function_call_output", "call_id": "t3_a", "output": "content1"},
|
||||
{"type": "function_call_output", "call_id": "t3_b", "output": "content2"},
|
||||
]
|
||||
messages = LiteLLMCompletionResponsesConfig._transform_response_input_param_to_chat_completion_message(
|
||||
input=input_items
|
||||
)
|
||||
asst_msgs = [m for m in messages if _get_role(m) == "assistant"]
|
||||
tool_msgs = [m for m in messages if _get_role(m) == "tool"]
|
||||
|
||||
assert len(asst_msgs) == 3, "Should have 3 assistant messages"
|
||||
assert len(_get_tool_calls(asst_msgs[0])) == 2, "Turn 1: 2 calls"
|
||||
assert len(_get_tool_calls(asst_msgs[1])) == 3, "Turn 2: 3 calls"
|
||||
assert len(_get_tool_calls(asst_msgs[2])) == 2, "Turn 3: 2 calls"
|
||||
assert len(tool_msgs) == 7, "Total 7 tool results"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests for Codex tool definition format (Fix #2)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCodexToolDefinitionFormat:
|
||||
"""Verify that Codex CLI tool definitions using 'id' and
|
||||
'inputSchema.jsonSchema' are correctly handled."""
|
||||
|
||||
def test_standard_function_tool_unchanged(self):
|
||||
"""Standard Responses API function tools should work as before."""
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"name": "get_weather",
|
||||
"description": "Get the weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"location": {"type": "string"}},
|
||||
},
|
||||
}
|
||||
]
|
||||
result, _ = (
|
||||
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
|
||||
tools=tools
|
||||
)
|
||||
)
|
||||
assert len(result) == 1
|
||||
func = result[0]["function"]
|
||||
assert func["name"] == "get_weather"
|
||||
assert func["parameters"]["properties"]["location"]["type"] == "string"
|
||||
|
||||
def test_codex_tool_with_id_and_input_schema(self):
|
||||
"""Codex CLI sends tools with 'id' instead of 'name' and
|
||||
'inputSchema.jsonSchema' instead of 'parameters'."""
|
||||
tools = [
|
||||
{
|
||||
"id": "exec_command",
|
||||
"inputSchema": {
|
||||
"jsonSchema": {
|
||||
"type": "object",
|
||||
"properties": {"cmd": {"type": "string"}},
|
||||
}
|
||||
},
|
||||
}
|
||||
]
|
||||
result, _ = (
|
||||
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
|
||||
tools=tools
|
||||
)
|
||||
)
|
||||
assert len(result) == 1
|
||||
func = result[0]["function"]
|
||||
assert func["name"] == "exec_command"
|
||||
assert func["parameters"]["type"] == "object"
|
||||
assert func["parameters"]["properties"]["cmd"]["type"] == "string"
|
||||
|
||||
def test_codex_tool_with_name_takes_precedence_over_id(self):
|
||||
"""If both 'name' and 'id' are present, 'name' takes precedence."""
|
||||
tools = [
|
||||
{
|
||||
"id": "old_name",
|
||||
"name": "new_name",
|
||||
"inputSchema": {
|
||||
"jsonSchema": {
|
||||
"type": "object",
|
||||
"properties": {},
|
||||
}
|
||||
},
|
||||
}
|
||||
]
|
||||
result, _ = (
|
||||
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
|
||||
tools=tools
|
||||
)
|
||||
)
|
||||
assert len(result) == 1
|
||||
func = result[0]["function"]
|
||||
assert func["name"] == "new_name"
|
||||
|
||||
def test_codex_tool_parameters_takes_precedence_over_input_schema(self):
|
||||
"""If both 'parameters' and 'inputSchema' are present,
|
||||
'parameters' takes precedence."""
|
||||
tools = [
|
||||
{
|
||||
"id": "my_tool",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"from_params": {"type": "string"}},
|
||||
},
|
||||
"inputSchema": {
|
||||
"jsonSchema": {
|
||||
"type": "object",
|
||||
"properties": {"from_schema": {"type": "string"}},
|
||||
}
|
||||
},
|
||||
}
|
||||
]
|
||||
result, _ = (
|
||||
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
|
||||
tools=tools
|
||||
)
|
||||
)
|
||||
func = result[0]["function"]
|
||||
assert "from_params" in func["parameters"]["properties"]
|
||||
assert "from_schema" not in func["parameters"]["properties"]
|
||||
|
||||
def test_empty_name_tool_passed_through(self):
|
||||
"""Tools with empty names (ghost entries) should be passed through
|
||||
as-is to avoid silently dropping provider-specific tool types."""
|
||||
tools = [
|
||||
{"id": "", "inputSchema": {"jsonSchema": {}}},
|
||||
{"id": "valid_tool", "inputSchema": {"jsonSchema": {"type": "object"}}},
|
||||
]
|
||||
result, _ = (
|
||||
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
|
||||
tools=tools
|
||||
)
|
||||
)
|
||||
# First tool passed through as-is, second converted
|
||||
assert len(result) == 2
|
||||
assert result[1]["function"]["name"] == "valid_tool"
|
||||
|
||||
def test_codex_tool_missing_input_schema(self):
|
||||
"""Codex tool with 'id' but no 'inputSchema' gets empty parameters."""
|
||||
tools = [{"id": "simple_tool"}]
|
||||
result, _ = (
|
||||
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
|
||||
tools=tools
|
||||
)
|
||||
)
|
||||
assert len(result) == 1
|
||||
func = result[0]["function"]
|
||||
assert func["name"] == "simple_tool"
|
||||
assert func["parameters"]["type"] == "object"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers for multi-turn tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _get_role(msg):
|
||||
"""Extract role from a message (dict or object)."""
|
||||
if isinstance(msg, dict):
|
||||
return msg.get("role", "")
|
||||
return getattr(msg, "role", "")
|
||||
|
||||
|
||||
def _get_tool_calls(msg):
|
||||
"""Extract tool_calls list from a message (dict or object)."""
|
||||
if isinstance(msg, dict):
|
||||
return msg.get("tool_calls") or []
|
||||
return getattr(msg, "tool_calls", None) or []
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue