scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-16 23:41:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 102

fix(responses): hoist Codex additional_tools input items into the chat bridge tools

Browse source
This commit is contained in:
mateo-berri 2026-09-13 02:11:59 -07:00
parent 30f33a949b
commit e732a484f6
9 changed files with 387 additions and 75 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

66
litellm/llms/bedrock_mantle/responses/transformation.py
View file

@ -17,7 +17,7 @@ BaseAWSLLM._sign_request after the request body is finalized.
import json
from collections.abc import Mapping
from typing import Any, Final
from typing import Any, Final, cast # noqa: TID251 # map_openai_params returns the filtered params as a bare dict
import httpx
from typing_extensions import ReadOnly, TypedDict
@ -32,6 +32,7 @@ from litellm.llms.bedrock_mantle.common_utils import (
BedrockMantleAuthMixin,
)
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.responses.additional_tools import HoistedAdditionalTools, hoist_additional_tools
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import (
ResponseInputParam,
@ -58,8 +59,6 @@ _BEDROCK_MANTLE_SUPPORTED_RESPONSE_TOOL_TYPES: Final = frozenset(
_BEDROCK_MANTLE_SUPPORTED_SERVICE_TIERS: Final = frozenset({"auto", "default"})
_BEDROCK_MANTLE_OPENAI_PATH_SUPPORTED_REASONING_SUMMARIES: Final = frozenset({"auto"})
_CODEX_ADDITIONAL_TOOLS_INPUT_ITEM_TYPE: Final = "additional_tools"
_CODEX_AGENT_MESSAGE_INPUT_ITEM_TYPE: Final = "agent_message"
_CODEX_CONTEXT_COMPACTION_INPUT_ITEM_TYPE: Final = "context_compaction"
_CODEX_LOCAL_SHELL_CALL_INPUT_ITEM_TYPE: Final = "local_shell_call"
@ -233,62 +232,29 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> dict:
remaining_input, hoisted_tools = self._hoist_codex_additional_tools(input)
normalized_input: Final = self._normalize_codex_input_items(remaining_input)
request_params: Final = (
{
**response_api_optional_request_params,
"tools": [
*(response_api_optional_request_params.get("tools") or []),
*hoisted_tools,
],
}
if hoisted_tools
else response_api_optional_request_params
params: Final = cast( # cast-ok: the base signature leaves the params dict untyped
"ResponsesAPIOptionalRequestParams", response_api_optional_request_params
)
hoisted: Final = hoist_additional_tools(input, params.get("tools"))
normalized_input: Final = self._normalize_codex_input_items(hoisted.input)
return super().transform_responses_api_request(
model=model,
input=normalized_input,
response_api_optional_request_params=request_params,
response_api_optional_request_params=self._params_with_hoisted_tools(params, hoisted),
litellm_params=litellm_params,
headers=headers,
)
@staticmethod
def _is_codex_additional_tools_item(item: Any) -> bool:
return isinstance(item, dict) and item.get("type") == _CODEX_ADDITIONAL_TOOLS_INPUT_ITEM_TYPE
@staticmethod
def _tools_of_additional_tools_item(item: "dict[str, Any]") -> "list[Any]":
tools: Final = item.get("tools")
return tools if isinstance(tools, list) else []
@classmethod
def _hoist_codex_additional_tools(
cls,
input: "str | ResponseInputParam",
) -> "tuple[str | ResponseInputParam, list[Any]]":
"""Codex's "responses lite" wire mode ships tool definitions inside
`input` as {"type": "additional_tools", "role": "developer",
"tools": [...]} items. api.openai.com accepts that item type; Mantle
rejects the whole request with 400 "Invalid 'input': value did not
match any expected variant" but accepts the same tools at the top
level, so move them there and strip the items from `input`.
"""
if not isinstance(input, list):
return input, []
additional_tools_items: Final = [item for item in input if cls._is_codex_additional_tools_item(item)]
if not additional_tools_items:
return input, []
remaining_input: Final = [item for item in input if not cls._is_codex_additional_tools_item(item)]
hoisted_tools = [tool for item in additional_tools_items for tool in cls._tools_of_additional_tools_item(item)]
verbose_logger.debug(
"Bedrock Mantle Responses API: hoisting %d tool(s) out of %d 'additional_tools' input item(s) "
"into the top-level tools param (Mantle rejects that input item type).",
len(hoisted_tools),
len(additional_tools_items),
)
return remaining_input, cls._filter_unsupported_tools(hoisted_tools)
def _params_with_hoisted_tools(
cls, params: Mapping[str, object], hoisted: HoistedAdditionalTools
) -> dict[str, object]:
if not hoisted.hoisted:
return dict(params)
supported_tools: Final = cls._filter_unsupported_tools(list(hoisted.tools))
if supported_tools:
return {**params, "tools": supported_tools}
return {key: value for key, value in params.items() if key != "tools"}
@staticmethod
def _agent_message_text(item: "Mapping[str, object]") -> str:

65
litellm/responses/additional_tools.py Normal file
View file

@ -0,0 +1,65 @@
from collections.abc import Sequence
from dataclasses import dataclass
from typing import Final, cast # noqa: TID251 # validating the openai tool union strips vendor keys from raw tools
from pydantic import BaseModel, ValidationError
from litellm._logging import verbose_logger
from litellm.types.llms.openai import ALL_RESPONSES_API_TOOL_PARAMS, ResponseInputParam
ADDITIONAL_TOOLS_INPUT_ITEM_TYPE: Final = "additional_tools"
class _InputItemType(BaseModel):
type: str = ""
class _AdditionalToolsItem(BaseModel):
tools: tuple[dict[str, object], ...] = ()
@dataclass(frozen=True, slots=True)
class HoistedAdditionalTools:
input: str | ResponseInputParam
tools: tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]
hoisted: tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]
def _is_additional_tools_item(item: object) -> bool:
try:
return _InputItemType.model_validate(item).type == ADDITIONAL_TOOLS_INPUT_ITEM_TYPE
except ValidationError:
return False
def _tools_of_item(item: object) -> tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]:
try:
parsed: Final = _AdditionalToolsItem.model_validate(item)
except ValidationError:
return ()
return tuple(
cast(
"ALL_RESPONSES_API_TOOL_PARAMS", tool
) # cast-ok: nested tools carry the same raw tool JSON as top-level tools
for tool in parsed.tools
)
def hoist_additional_tools(
input: str | ResponseInputParam,
tools: Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None,
) -> HoistedAdditionalTools:
existing: Final = tuple(tools or ())
if isinstance(input, str):
return HoistedAdditionalTools(input=input, tools=existing, hoisted=())
items: Final = tuple(item for item in input if _is_additional_tools_item(item))
if not items:
return HoistedAdditionalTools(input=input, tools=existing, hoisted=())
hoisted: Final = tuple(tool for item in items for tool in _tools_of_item(item))
verbose_logger.debug(
"Responses API: hoisting %d tool(s) out of %d 'additional_tools' input item(s) into the top-level tools param.",
len(hoisted),
len(items),
)
remaining_input: Final = [item for item in input if not _is_additional_tools_item(item)]
return HoistedAdditionalTools(input=remaining_input, tools=(*existing, *hoisted), hoisted=hoisted)

28
litellm/responses/litellm_completion_transformation/custom_tools.py
View file

@ -39,15 +39,27 @@ def openai_shaped_tool_call_item_id(item_type: str, tool_id: str) -> str:
return f"{prefix}_{tool_id}"
class _ToolNameFields(BaseModel):
type: str = ""
name: str = ""
tools: tuple[object, ...] = ()
def _custom_tool_names_of(tool: object) -> tuple[str, ...]:
try:
parsed: Final = _ToolNameFields.model_validate(tool)
except ValidationError:
return ()
if parsed.type == "custom":
return (parsed.name,) if parsed.name else ()
if parsed.type != "namespace":
return ()
return tuple(name for nested_tool in parsed.tools for name in _custom_tool_names_of(nested_tool))
def extract_custom_tool_names(tools: Sequence[object] | None) -> set[str]:
"""Extract names of tools originally defined as ``type: "custom"``."""
if not tools:
return set()
names: Final[set[str]] = set()
for tool in tools:
if isinstance(tool, dict) and tool.get("type") == "custom" and "name" in tool:
names.add(tool["name"])
return names
"""Extract names of tools defined as ``type: "custom"``, at the top level or inside a ``namespace`` tool."""
return {name for tool in tools or () for name in _custom_tool_names_of(tool)}
def is_custom_tool_call(tool_name: str, custom_tool_names: set[str]) -> bool:

22
litellm/responses/litellm_completion_transformation/handler.py
View file

@ -6,6 +6,7 @@ from collections.abc import Coroutine, Mapping
from typing import Final
import litellm
from litellm.responses.additional_tools import hoist_additional_tools
from litellm.responses.litellm_completion_transformation.streaming_iterator import (
LiteLLMCompletionStreamingIterator,
)
@ -37,11 +38,16 @@ class LiteLLMCompletionTransformationHandler:
| BaseResponsesAPIStreamingIterator
| Coroutine[object, object, ResponsesAPIResponse | BaseResponsesAPIStreamingIterator]
):
hoisted: Final = hoist_additional_tools(input, responses_api_request.get("tools"))
bridged_input: Final = hoisted.input
bridged_request: Final[ResponsesAPIOptionalRequestParams] = (
{**responses_api_request, "tools": list(hoisted.tools)} if hoisted.hoisted else responses_api_request
)
litellm_completion_request: Final[dict] = (
LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
model=model,
input=input,
responses_api_request=responses_api_request,
input=bridged_input,
responses_api_request=bridged_request,
custom_llm_provider=custom_llm_provider,
stream=stream,
extra_headers=extra_headers,
@ -52,8 +58,8 @@ class LiteLLMCompletionTransformationHandler:
if _is_async:
return self.async_response_api_handler(
litellm_completion_request=litellm_completion_request,
request_input=input,
responses_api_request=responses_api_request,
request_input=bridged_input,
responses_api_request=bridged_request,
**kwargs,
)
@ -70,8 +76,8 @@ class LiteLLMCompletionTransformationHandler:
responses_api_response: Final[ResponsesAPIResponse] = (
LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response(
chat_completion_response=litellm_completion_response,
request_input=input,
responses_api_request=responses_api_request,
request_input=bridged_input,
responses_api_request=bridged_request,
)
)
@ -81,8 +87,8 @@ class LiteLLMCompletionTransformationHandler:
return LiteLLMCompletionStreamingIterator(
model=model,
litellm_custom_stream_wrapper=litellm_completion_response,
request_input=input,
responses_api_request=responses_api_request,
request_input=bridged_input,
responses_api_request=bridged_request,
custom_llm_provider=custom_llm_provider,
litellm_metadata=kwargs.get("litellm_metadata", {}),
)

22
litellm/responses/litellm_completion_transformation/transformation.py
View file

@ -1890,9 +1890,21 @@ class LiteLLMCompletionResponsesConfig:
namespace_tool: NamespaceTool,
nested: bool,
) -> ChatCompletionToolParam | None:
if nested and namespace_tool.get("type") != "function":
tool_type: Final = namespace_tool.get("type")
if nested and tool_type not in ("function", "custom"):
return None
raw_description: Final = str(namespace_tool.get("description") or "")
description: Final = (
f"{namespace_description}{NAMESPACE_DESCRIPTION_SEPARATOR}{raw_description}"
if nested and namespace_description and raw_description
else namespace_description
if nested and namespace_description
else raw_description
)
if nested and tool_type == "custom":
return convert_custom_tool_to_function_tool({**namespace_tool, "description": description})
raw_parameters: Final = namespace_tool.get("parameters")
parameters: Final = (
MappingProxyType(raw_parameters) if isinstance(raw_parameters, Mapping) else MappingProxyType({})
@ -1901,14 +1913,6 @@ class LiteLLMCompletionResponsesConfig:
parameters if parameters and "type" in parameters else MappingProxyType({**parameters, "type": "object"})
)
tool_name: Final = str(namespace_tool.get("name") or "")
raw_description: Final = str(namespace_tool.get("description") or "")
description: Final = (
f"{namespace_description}{NAMESPACE_DESCRIPTION_SEPARATOR}{raw_description}"
if nested and namespace_description and raw_description
else namespace_description
if nested and namespace_description
else raw_description
)
chat_tool_name: Final = f"{namespace}__{tool_name}" if nested else tool_name
function: Final = ChatCompletionToolParamFunctionChunk(
name=chat_tool_name,

102
tests/test_litellm/responses/litellm_completion_transformation/test_handler.py
View file

@ -68,3 +68,105 @@ async def test_async_fallback_tags_skip_responses_api_bridge():
await coro
assert captured.get("_skip_responses_api_bridge") is True
_CODEX_ADDITIONAL_TOOLS_ITEM = {
"type": "additional_tools",
"id": "at_codex",
"role": "developer",
"tools": [
{
"type": "namespace",
"name": "functions",
"description": "",
"tools": [
{
"type": "custom",
"name": "exec",
"description": "Runs a shell command.",
"format": {"type": "grammar", "syntax": "lark", "definition": "start: /.+/"},
},
{
"type": "function",
"name": "wait",
"description": "Waits for a background command.",
"parameters": {"type": "object", "properties": {"id": {"type": "string"}}},
},
],
}
],
}
_CODEX_INPUT = [_CODEX_ADDITIONAL_TOOLS_ITEM, {"type": "message", "role": "user", "content": "Run ls"}]
def test_sync_fallback_hoists_additional_tools_input_items_into_chat_tools():
handler = LiteLLMCompletionTransformationHandler()
captured: dict = {}
def fake_completion(**kwargs):
captured.update(kwargs)
raise _StopForwarding()
with patch("litellm.completion", fake_completion): # test-quality-ok: no DI seam; the file stubs this same boundary
with pytest.raises(_StopForwarding):
handler.response_api_handler(
model="bedrock/us.openai.gpt-5.6",
input=_CODEX_INPUT,
responses_api_request={},
custom_llm_provider="bedrock",
_is_async=False,
)
assert [message["role"] for message in captured["messages"]] == ["user"]
functions_by_name = {tool["function"]["name"]: tool["function"] for tool in captured["tools"]}
assert set(functions_by_name) == {"exec", "functions__wait"}
assert set(functions_by_name["exec"]["parameters"]["properties"]) == {"content"}
@pytest.mark.asyncio
async def test_async_fallback_returns_hoisted_nested_custom_tool_call_as_custom_tool_call():
from litellm.responses.litellm_completion_transformation.transformation import TOOL_CALLS_CACHE
from litellm.types.utils import ChatCompletionMessageToolCall, Choices, Function, Message, ModelResponse
handler = LiteLLMCompletionTransformationHandler()
tool_call_id = "call_exec_hoisted"
async def fake_acompletion(**kwargs):
return ModelResponse(
id="chatcmpl-exec",
created=1,
model="us.openai.gpt-5.6",
object="chat.completion",
choices=[
Choices(
finish_reason="tool_calls",
index=0,
message=Message(
content=None,
role="assistant",
tool_calls=[
ChatCompletionMessageToolCall(
id=tool_call_id,
type="function",
function=Function(name="exec", arguments='{"content": "ls"}'),
)
],
),
)
],
)
try:
with patch("litellm.acompletion", fake_acompletion): # test-quality-ok: no DI seam; file stubs this boundary
response = await handler.response_api_handler(
model="bedrock/us.openai.gpt-5.6",
input=_CODEX_INPUT,
responses_api_request={},
custom_llm_provider="bedrock",
_is_async=True,
)
finally:
TOOL_CALLS_CACHE.delete_cache(key=tool_call_id)
tool_calls = [(item.type, item.name, item.input) for item in response.output if item.type == "custom_tool_call"]
assert tool_calls == [("custom_tool_call", "exec", "ls")]

91
tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py
View file

@ -2506,6 +2506,7 @@ class TestToolTransformation:
"tools": [
"ignored",
{"type": "namespace", "name": "ignored"},
{"type": "web_search", "name": "ignored"},
{
"type": "function",
"name": "spawn_agent",
@ -2527,6 +2528,36 @@ class TestToolTransformation:
"type": "object",
}
def test_transform_nested_namespace_custom_tool_becomes_a_content_function_under_its_short_name(self):
namespace_tool = {
"type": "namespace",
"name": "functions",
"description": "Codex shell tools.",
"tools": [
{
"type": "custom",
"name": "exec",
"description": "Runs a shell command.",
"format": {"type": "grammar", "syntax": "lark", "definition": "start: /.+/"},
},
],
}
result_tools, _ = (
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=[namespace_tool]
)
)
assert len(result_tools) == 1
function = result_tools[0]["function"]
assert function["name"] == "exec"
assert function["description"].startswith("Codex shell tools.")
assert "Runs a shell command." in function["description"]
assert "start: /.+/" in function["description"]
assert function["parameters"]["required"] == ["content"]
assert function["parameters"]["properties"]["content"]["type"] == "string"
@pytest.mark.parametrize(
"model, custom_llm_provider",
[
@ -3786,6 +3817,66 @@ class TestEnsureOutputItemContentPartAdded:
assert added.item.name == "spawn_agent"
assert added.item.namespace == "collaboration"
def test_streaming_nested_custom_tool_call_comes_back_as_custom_tool_call(self):
from litellm.responses.litellm_completion_transformation.custom_tools import extract_custom_tool_names
iterator = self._make_iterator()
iterator.responses_api_request = {
"tools": [
{
"type": "namespace",
"name": "functions",
"tools": [
{
"type": "custom",
"name": "exec",
"format": {"type": "grammar", "syntax": "lark", "definition": "start: /.+/"},
}
],
}
]
}
iterator._custom_tool_names = extract_custom_tool_names(iterator.responses_api_request.get("tools"))
iterator._namespace_tool_names = LiteLLMCompletionResponsesConfig.namespace_tool_name_map(
iterator.responses_api_request.get("tools")
)
iterator._queue_tool_call_delta_events(
[{"index": 0, "id": "call_exec", "function": {"name": "exec", "arguments": '{"content":"ls"}'}}]
)
iterator._queue_final_tool_call_done_events(
ModelResponse(
id="chatcmpl-exec",
created=1,
model="us.openai.gpt-5.6",
object="chat.completion",
choices=[
Choices(
finish_reason="tool_calls",
index=0,
message=Message(
content=None,
role="assistant",
tool_calls=[
ChatCompletionMessageToolCall(
id="call_exec",
type="function",
function=Function(name="exec", arguments='{"content":"ls"}'),
)
],
),
)
],
)
)
added = iterator._pending_tool_events[0]
assert added.item.type == "custom_tool_call"
assert added.item.name == "exec"
done = iterator._pending_tool_events[-1]
assert done.item.type == "custom_tool_call"
assert done.item.input == "ls"
def test_streaming_unqualified_namespace_tool_calls_restore_namespace(self):
"""A unique nested tool name without the namespace still maps back."""
iterator = self._make_iterator()

48
tests/test_litellm/responses/test_additional_tools.py Normal file
View file

@ -0,0 +1,48 @@
from litellm.responses.additional_tools import hoist_additional_tools
_EXEC_TOOL = {"type": "custom", "name": "exec", "format": {"type": "grammar", "syntax": "lark", "definition": "start: /.+/"}}
_WAIT_TOOL = {"type": "function", "name": "wait", "parameters": {"type": "object", "properties": {}}}
_TOP_LEVEL_TOOL = {"type": "function", "name": "top_level", "parameters": {"type": "object", "properties": {}}}
_USER_MESSAGE = {"type": "message", "role": "user", "content": "Run ls"}
def test_string_input_passes_through_with_existing_tools():
hoisted = hoist_additional_tools("hello", [_TOP_LEVEL_TOOL])
assert hoisted.input == "hello"
assert hoisted.tools == (_TOP_LEVEL_TOOL,)
assert hoisted.hoisted == ()
def test_input_without_additional_tools_items_is_returned_untouched():
request_input = [_USER_MESSAGE]
hoisted = hoist_additional_tools(request_input, None)
assert hoisted.input is request_input
assert hoisted.tools == ()
assert hoisted.hoisted == ()
def test_additional_tools_items_are_stripped_and_appended_after_top_level_tools_in_item_order():
request_input = [
{"type": "additional_tools", "id": "at_1", "role": "developer", "tools": [_EXEC_TOOL]},
_USER_MESSAGE,
{"type": "additional_tools", "id": "at_2", "role": "developer", "tools": [_WAIT_TOOL]},
]
hoisted = hoist_additional_tools(request_input, [_TOP_LEVEL_TOOL])
assert hoisted.input == [_USER_MESSAGE]
assert hoisted.tools == (_TOP_LEVEL_TOOL, _EXEC_TOOL, _WAIT_TOOL)
assert hoisted.hoisted == (_EXEC_TOOL, _WAIT_TOOL)
def test_additional_tools_item_without_a_tools_list_is_stripped_and_contributes_nothing():
request_input = [{"type": "additional_tools", "id": "at_1", "role": "developer", "tools": "exec"}, _USER_MESSAGE]
hoisted = hoist_additional_tools(request_input, None)
assert hoisted.input == [_USER_MESSAGE]
assert hoisted.tools == ()
assert hoisted.hoisted == ()

18
tests/test_litellm/responses/test_custom_tool_call.py
View file

@ -55,6 +55,24 @@ class TestCustomToolUtilities:
names = extract_custom_tool_names(tools)
assert names == set()
def test_extract_custom_tool_names_walks_namespace_tools(self):
tools = [
{"type": "function", "name": "regular_tool"},
{
"type": "namespace",
"name": "functions",
"tools": [
{"type": "custom", "name": "exec"},
{"type": "function", "name": "wait"},
"ignored",
],
},
{"type": "namespace", "name": "empty", "tools": "not-a-list"},
]
names = extract_custom_tool_names(tools)
assert names == {"exec"}
def test_extract_custom_tool_names_none(self):
"""Test extraction with None input."""
names = extract_custom_tool_names(None)