fix(proxy): keep Codex namespace tool calls intact across LiteLLM providers

Fireworks and Bedrock Mantle accept Codex namespace tools natively, so the
native route now hoists them without flattening, like OpenAI and Azure and
the chat completions bridge. Other native Responses providers still get
flat names, now built with the bridge's namespace_tool_name_map, and each
function_call they return gets its tool name and namespace back in the
non-streaming body and in streamed output_item.added, output_item.done,
and completed/incomplete/failed events. Every other event passes through
unchanged
This commit is contained in:
mateo-berri 2026-10-29 11:44:37 -07:00
parent 6ac4cfe9cf
commit c7d95a338e
3 changed files with 352 additions and 37 deletions

View file

@ -2,14 +2,15 @@ import hashlib
import hmac
import json
import re
from collections.abc import Iterator, Mapping
from collections.abc import AsyncIterable, AsyncIterator, Iterator, Mapping
from dataclasses import dataclass
from types import MappingProxyType
from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol
import httpx
from fastapi import APIRouter, Depends, HTTPException, Request, Response
from fastapi.responses import StreamingResponse
from pydantic import TypeAdapter, ValidationError
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from starlette.types import Message
import litellm
@ -39,10 +40,13 @@ from litellm.proxy.common_utils.http_parsing_utils import (
)
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import pass_through_request
from litellm.proxy.response_api_endpoints.endpoints import responses_api
from litellm.responses.additional_tools import with_additional_tools_hoisted, with_namespace_members_as_tools
from litellm.responses.additional_tools import with_additional_tools_hoisted, with_namespaces_flattened
from litellm.responses.litellm_completion_transformation.reasoning_items import is_other_provider_reasoning_item
from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig
from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import EncryptedContentAffinityCheck
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.custom_http import httpxSpecialProvider
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.router import Deployment
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager, supports_none_reasoning_effort
@ -76,6 +80,12 @@ _ACCOUNT_SLUGS_CACHE_PREFIX: Final = "codex_account_slugs:"
_MODELS_ETAG_CACHE_PREFIX: Final = "codex_models_etag:"
_MODELS_ETAG_TTL_SECONDS: Final = 300
_OPENAI_API_PROVIDERS: Final = frozenset({LlmProviders.OPENAI.value, LlmProviders.AZURE.value})
_NAMESPACE_TOOL_PROVIDERS: Final = frozenset(
{*_OPENAI_API_PROVIDERS, LlmProviders.FIREWORKS_AI.value, LlmProviders.BEDROCK_MANTLE.value}
)
_TOOL_DEFINITIONS: Final = TypeAdapter(tuple[dict[str, object], ...])
_NOTHING_FLATTENED: Final[Mapping[str, tuple[str, str]]] = MappingProxyType({})
_SSE_DATA_FIELD: Final = "data:"
_HEADERS: Final = frozenset(
{
"authorization",
@ -95,6 +105,23 @@ _HEADERS: Final = frozenset(
)
class _FunctionCallName(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True, from_attributes=True)
type: Literal["function_call"]
name: str
class _ResponseOutput(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True, from_attributes=True)
output: tuple[object, ...] = ()
@dataclass(frozen=True, slots=True)
class _LiteLLMRequest:
body: Mapping[str, object]
flattened_names: Mapping[str, tuple[str, str]]
def _is_chatgpt(deployment: Deployment) -> bool:
return (
deployment.litellm_params.custom_llm_provider == _CHATGPT_PROVIDER
@ -373,39 +400,132 @@ def _is_openai_api(deployment: Deployment) -> bool:
return resolved is not None and resolved[1] in _OPENAI_API_PROVIDERS
def _bridges_to_chat_completions(deployment: Deployment) -> bool:
def _takes_namespace_tools(deployment: Deployment) -> bool:
resolved: Final = _model_and_provider(deployment)
if resolved is None:
return False
model, provider = resolved
return (
resolved is not None
and ProviderConfigManager.get_provider_responses_api_config(
provider=resolved[1], model=resolved[0], api_base=deployment.litellm_params.api_base
provider in _NAMESPACE_TOOL_PROVIDERS
or ProviderConfigManager.get_provider_responses_api_config(
provider=provider, model=model, api_base=deployment.litellm_params.api_base
)
is None
)
def _with_tools_the_targets_accept(
body: Mapping[str, object], deployments: tuple[Deployment, ...]
) -> Mapping[str, object]:
def _flattened_names(body: Mapping[str, object]) -> Mapping[str, tuple[str, str]]:
try:
tools: Final = _TOOL_DEFINITIONS.validate_python(body.get("tools") or ())
except ValidationError:
return _NOTHING_FLATTENED
names: Final = LiteLLMCompletionResponsesConfig.namespace_tool_name_map(tools)
return MappingProxyType(
{flat_name: (namespace, tool) for flat_name, (namespace, tool) in names.items() if flat_name != tool}
)
def _with_tools_the_targets_accept(body: Mapping[str, object], deployments: tuple[Deployment, ...]) -> _LiteLLMRequest:
if all(_is_openai_api(deployment) for deployment in deployments):
return body
return _LiteLLMRequest(body=body, flattened_names=_NOTHING_FLATTENED)
hoisted: Final = with_additional_tools_hoisted(body)
if all(_bridges_to_chat_completions(deployment) for deployment in deployments):
return hoisted
return with_namespace_members_as_tools(hoisted)
if all(_takes_namespace_tools(deployment) for deployment in deployments):
return _LiteLLMRequest(body=hoisted, flattened_names=_NOTHING_FLATTENED)
flattened_names: Final = _flattened_names(hoisted)
return _LiteLLMRequest(body=with_namespaces_flattened(hoisted, flattened_names), flattened_names=flattened_names)
def _litellm_body(body: Mapping[str, object], llm_router: "Router | None", team_id: str | None) -> Mapping[str, object]:
def _litellm_request(body: Mapping[str, object], llm_router: "Router | None", team_id: str | None) -> _LiteLLMRequest:
wrapped: Final = _with_wrapped_reasoning(_with_provider_default_reasoning(body, llm_router, team_id))
model: Final = body.get("model")
if not isinstance(model, str) or llm_router is None:
return wrapped
return _LiteLLMRequest(body=wrapped, flattened_names=_NOTHING_FLATTENED)
deployments: Final = _candidate_deployments(model, llm_router, team_id)
return _with_tools_the_targets_accept(
_with_reasoning_the_targets_can_decrypt(wrapped, model, llm_router, deployments), deployments
)
def _namespaced(item: object, flattened_names: Mapping[str, tuple[str, str]]) -> tuple[str, str] | None:
try:
call: Final = _FunctionCallName.model_validate(item)
except ValidationError:
return None
return flattened_names.get(call.name)
def _with_namespace(item: object, flattened_names: Mapping[str, tuple[str, str]]) -> object:
namespaced: Final = _namespaced(item, flattened_names)
if namespaced is None:
return item
namespace, name = namespaced
restored: Final = {"name": name, "namespace": namespace}
match item:
case BaseModel():
return item.model_copy(update=restored)
case _:
return {**_json_object(item), **restored}
def _response_output(response: object) -> tuple[object, ...]:
try:
return _ResponseOutput.model_validate(response).output
except ValidationError:
return ()
def _output_with_namespaces(
response: object, flattened_names: Mapping[str, tuple[str, str]]
) -> tuple[object, ...] | None:
output: Final = _response_output(response)
if all(_namespaced(item, flattened_names) is None for item in output):
return None
return tuple(_with_namespace(item, flattened_names) for item in output)
def _event_with_namespaces(
event: Mapping[str, object], flattened_names: Mapping[str, tuple[str, str]]
) -> Mapping[str, object] | None:
match event.get("type"):
case "response.output_item.added" | "response.output_item.done":
item: Final = event.get("item")
if _namespaced(item, flattened_names) is None:
return None
return {**event, "item": _with_namespace(item, flattened_names)}
case "response.completed" | "response.incomplete" | "response.failed":
response: Final = _json_object(event.get("response"))
output: Final = _output_with_namespaces(response, flattened_names)
return None if output is None else {**event, "response": {**response, "output": list(output)}}
case _:
return None
def _sse_line_with_namespaces(line: str, flattened_names: Mapping[str, tuple[str, str]]) -> str:
if not line.startswith(_SSE_DATA_FIELD):
return line
try:
event: Final = _JSON_OBJECT.validate_json(line.removeprefix(_SSE_DATA_FIELD))
except ValidationError:
return line
restored: Final = _event_with_namespaces(event, flattened_names)
return line if restored is None else f"{_SSE_DATA_FIELD} {json.dumps(restored)}"
def _frame_with_namespaces(
frame: str | bytes | memoryview, flattened_names: Mapping[str, tuple[str, str]]
) -> str | bytes | memoryview:
if not isinstance(frame, str) or not any(flat_name in frame for flat_name in flattened_names):
return frame
return "\n".join(_sse_line_with_namespaces(line, flattened_names) for line in frame.split("\n"))
async def _frames_with_namespaces(
frames: AsyncIterable[str | bytes | memoryview], flattened_names: Mapping[str, tuple[str, str]]
) -> AsyncIterator[str | bytes | memoryview]:
async for frame in frames:
yield _frame_with_namespaces(frame, flattened_names)
def _is_unpersisted_reasoning_reference(item: object) -> bool:
try:
fields: Final = _JSON_OBJECT.validate_python(item)
@ -553,6 +673,33 @@ async def _catalog(
return _with_etag(upstream, merged if merged is not None else upstream_body, etag)
async def _litellm_responses(
request: Request,
fastapi_response: Response,
user_api_key_dict: UserAPIKeyAuth,
body: Mapping[str, object],
llm_router: "Router | None",
) -> object:
litellm_request: Final = _litellm_request(body, llm_router, user_api_key_dict.team_id)
served: Final = await responses_api(
_as_litellm_responses_request(request, json.dumps(litellm_request.body).encode(), litellm_request.body),
fastapi_response,
user_api_key_dict.model_copy(update={"request_route": _LITELLM_RESPONSES_PATH}),
)
flattened_names: Final = litellm_request.flattened_names
if not flattened_names:
return served
match served:
case StreamingResponse():
served.body_iterator = _frames_with_namespaces(served.body_iterator, flattened_names)
return served
case ResponsesAPIResponse():
output: Final = _output_with_namespaces(served, flattened_names)
return served if output is None else served.model_copy(update={"output": list(output)})
case _:
return served
async def _responses(
request: Request,
fastapi_response: Response,
@ -565,12 +712,7 @@ async def _responses(
body: Final = _JSON_OBJECT.validate_python(await read_request_body(request))
if await _serves_through_litellm(body.get("model"), llm_router, user_api_key_dict, headers, cache):
litellm_body: Final = _litellm_body(body, llm_router, user_api_key_dict.team_id)
return await responses_api(
_as_litellm_responses_request(request, json.dumps(litellm_body).encode(), litellm_body),
fastapi_response,
user_api_key_dict.model_copy(update={"request_route": _LITELLM_RESPONSES_PATH}),
)
return await _litellm_responses(request, fastapi_response, user_api_key_dict, body, llm_router)
chatgpt_body: Final = _chatgpt_body(body, get_client_requested_model(request))
upstream: Final = await pass_through_request(
request=request,

View file

@ -3,9 +3,14 @@ from collections.abc import Iterator, Mapping, Sequence
from dataclasses import dataclass
from itertools import chain, groupby
from types import MappingProxyType
from typing import Final, cast # noqa: TID251 # validating the openai tool union strips vendor keys from raw tools
from typing import (
Annotated,
Final,
Literal,
cast, # noqa: TID251 # validating the openai tool union strips vendor keys from raw tools
)
from pydantic import TypeAdapter, ValidationError
from pydantic import ConfigDict, Field, TypeAdapter, ValidationError
from litellm._logging import verbose_logger
from litellm.types.llms.base import LiteLLMBaseModel
@ -13,8 +18,8 @@ from litellm.types.llms.openai import ALL_RESPONSES_API_TOOL_PARAMS, ResponseInp
ADDITIONAL_TOOLS_INPUT_ITEM_TYPE: Final = "additional_tools"
_NAMESPACE_TOOL_TYPE: Final = "namespace"
_QUALIFIED_NAME_SEPARATOR: Final = "__"
_TOOLS: Final = TypeAdapter(tuple[object, ...])
_ITEMS: Final = TypeAdapter(tuple[object, ...])
_JSON_OBJECT: Final = TypeAdapter(dict[str, object])
class _InputItemType(LiteLLMBaseModel):
@ -25,6 +30,13 @@ class _ToolList(LiteLLMBaseModel):
tools: tuple[Mapping[str, object], ...] = ()
class _NamespacedFunctionCall(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
type: Literal["function_call"]
name: str
namespace: Annotated[str, Field(min_length=1)]
@dataclass(frozen=True, slots=True)
class HoistedAdditionalTools:
input: str | ResponseInputParam
@ -119,7 +131,7 @@ def with_additional_tools_hoisted(request: Mapping[str, object]) -> Mapping[str,
return request
input_items: Final = cast(ResponseInputParam, request_input) # cast-ok: forwarded as sent, as inference does
top_level_tools: Final = cast( # cast-ok: tools are forwarded as sent, as inference does
"tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]", _TOOLS.validate_python(request.get("tools") or ())
"tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]", _ITEMS.validate_python(request.get("tools") or ())
)
hoisted: Final = hoist_additional_tools(input_items, top_level_tools)
if not hoisted.hoisted:
@ -127,20 +139,45 @@ def with_additional_tools_hoisted(request: Mapping[str, object]) -> Mapping[str,
return {**request, "input": hoisted.input, "tools": list(hoisted.tools)}
def _qualified_member(namespace: str, member: Mapping[str, object]) -> Mapping[str, object]:
prefix: Final = f"{namespace.removesuffix(_QUALIFIED_NAME_SEPARATOR)}{_QUALIFIED_NAME_SEPARATOR}"
return {**member, "name": f"{prefix}{member.get('name')}"}
def _flattened_member(
namespace: str, member: Mapping[str, object], flat_names: Mapping[tuple[str, str], str]
) -> Mapping[str, object]:
flat_name: Final = flat_names.get((namespace, str(member.get("name") or "")))
return member if flat_name is None else {**member, "name": flat_name}
def _members_as_tools(tool: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
def _members_as_tools(
tool: Mapping[str, object], flat_names: Mapping[tuple[str, str], str]
) -> tuple[Mapping[str, object], ...]:
if tool.get("type") != _NAMESPACE_TOOL_TYPE:
return (tool,)
namespace: Final = str(tool.get("name") or "")
return tuple(_qualified_member(namespace, member) for member in _tool_list_of(tool))
return tuple(_flattened_member(namespace, member, flat_names) for member in _tool_list_of(tool))
def with_namespace_members_as_tools(request: Mapping[str, object]) -> Mapping[str, object]:
tools: Final = _tool_list_of(request)
if not any(tool.get("type") == _NAMESPACE_TOOL_TYPE for tool in tools):
def _flattened_call(item: object, flat_names: Mapping[tuple[str, str], str]) -> object:
try:
call: Final = _NamespacedFunctionCall.model_validate(item)
except ValidationError:
return item
flat_name: Final = flat_names.get((call.namespace, call.name))
if flat_name is None:
return item
fields: Final = _JSON_OBJECT.validate_python(item)
return {**{key: value for key, value in fields.items() if key != "namespace"}, "name": flat_name}
def with_namespaces_flattened(
request: Mapping[str, object], namespaced_names: Mapping[str, tuple[str, str]]
) -> Mapping[str, object]:
if not namespaced_names:
return request
return {**request, "tools": list(chain.from_iterable(map(_members_as_tools, tools)))}
flat_names: Final = MappingProxyType({namespaced: flat for flat, namespaced in namespaced_names.items()})
tools: Final = chain.from_iterable(_members_as_tools(tool, flat_names) for tool in _tool_list_of(request))
request_input: Final = request.get("input")
replayed: Final = (
{"input": [_flattened_call(item, flat_names) for item in _ITEMS.validate_python(request_input)]}
if isinstance(request_input, list)
else {}
)
return {**request, "tools": list(tools), **replayed}

View file

@ -1181,7 +1181,7 @@ _ADDITIONAL_TOOLS: Final = {
pytest.param(
"grok-4.7",
[_USER_TURN],
["exec_command", "collaboration__followup_task", "mcp__docs__search"],
["exec_command", "collaboration__followup_task", "mcp__docs____search"],
id="xai gets the item's tools as top-level functions",
),
pytest.param("oai-luna", [_ADDITIONAL_TOOLS, _USER_TURN], ["exec_command"], id="the openai api keeps the item"),
@ -1268,6 +1268,142 @@ def test_codex_namespace_tool_called_through_the_chat_bridge_comes_back_with_its
assert (call["namespace"], call["name"]) == ("mcp__docs__", "search")
_CLOCK_NAMESPACE: Final = _namespace("clock", "sleep")
_CLOCK_TOOLS: Final = {"type": "additional_tools", "role": "developer", "tools": [_CLOCK_NAMESPACE]}
_FLAT_SLEEP_TOOL: Final = {"type": "function", "name": "clock__sleep", "parameters": {"type": "object"}}
_FLAT_SLEEP_CALL: Final = {
"type": "function_call",
"id": "fc_sleep",
"call_id": "call_sleep",
"name": "clock__sleep",
"arguments": '{"seconds":1}',
"status": "completed",
}
def test_codex_namespace_tool_reaches_fireworks_as_a_namespace(
client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router
) -> None:
_account_catalog(respx_mock)
provider: Final = _open_weights_provider(respx_mock)
response: Final = client.post(
"/backend-api/codex/responses",
json={"model": "open-weights", "input": [_CLOCK_TOOLS, _USER_TURN], "stream": False, "store": False},
headers=_ACCOUNT_HEADERS,
)
assert response.status_code == 200, response.text
assert json.loads(provider.calls.last.request.content)["tools"] == [_CLOCK_NAMESPACE]
def test_xai_namespace_call_round_trips_to_codex_with_its_namespace(
client: TestClient, respx_mock: respx.MockRouter, xai_and_openai_router: Router
) -> None:
_account_catalog(respx_mock)
xai: Final = respx_mock.post(f"{_XAI_BASE}/responses").respond(
json={**_provider_completed(""), "output": [_FLAT_SLEEP_CALL]}
)
replayed_call: Final = {**_FLAT_SLEEP_CALL, "name": "sleep", "namespace": "clock"}
replayed_output: Final = {"type": "function_call_output", "call_id": "call_sleep", "output": "slept"}
response: Final = client.post(
"/backend-api/codex/responses",
json={
"model": "grok-4.7",
"input": [_CLOCK_TOOLS, _USER_TURN, replayed_call, replayed_output],
"stream": False,
"store": False,
},
headers=_ACCOUNT_HEADERS,
)
assert response.status_code == 200, response.text
sent: Final = json.loads(xai.calls.last.request.content)
assert sent["tools"] == [_FLAT_SLEEP_TOOL]
assert sent["input"][1:] == [_FLAT_SLEEP_CALL, replayed_output]
(call,) = (item for item in response.json()["output"] if item["type"] == "function_call")
assert (call["name"], call["namespace"], call["call_id"]) == ("sleep", "clock", "call_sleep")
def _sse_events(*events: dict[str, object]) -> bytes:
return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events).encode()
def _xai_sleep_call_stream() -> bytes:
completed: Final = {**_provider_completed(""), "output": [_FLAT_SLEEP_CALL]}
return _sse_events(
{
"type": "response.created",
"sequence_number": 0,
"response": {**completed, "status": "in_progress", "output": []},
},
{
"type": "response.output_item.added",
"sequence_number": 1,
"output_index": 0,
"item": {**_FLAT_SLEEP_CALL, "arguments": "", "status": "in_progress"},
},
{
"type": "response.function_call_arguments.delta",
"sequence_number": 2,
"item_id": "fc_sleep",
"output_index": 0,
"delta": '{"seconds":1}',
},
{"type": "response.output_item.done", "sequence_number": 3, "output_index": 0, "item": _FLAT_SLEEP_CALL},
{"type": "response.completed", "sequence_number": 4, "response": completed},
)
def _data_lines(stream: bytes) -> list[bytes]:
return [line.removeprefix(b"data: ") for line in stream.splitlines() if line.startswith(b"data: {")]
def test_xai_streamed_namespace_call_reaches_codex_with_its_namespace_and_other_events_unchanged(
client: TestClient, respx_mock: respx.MockRouter, xai_and_openai_router: Router
) -> None:
_account_catalog(respx_mock)
respx_mock.post(f"{_XAI_BASE}/responses").respond(
content=_xai_sleep_call_stream(), headers={"content-type": "text/event-stream"}
)
native: Final = client.post(
"/backend-api/codex/responses",
json={"model": "grok-4.7", "input": [_CLOCK_TOOLS, _USER_TURN], "stream": True, "store": False},
headers=_ACCOUNT_HEADERS,
)
v1: Final = client.post(
"/v1/responses",
json={"model": "grok-4.7", "input": [_USER_TURN], "tools": [_FLAT_SLEEP_TOOL], "stream": True, "store": False},
headers={"authorization": f"Bearer {MASTER_KEY}"},
)
assert native.status_code == v1.status_code == 200
native_lines: Final = _data_lines(native.content)
v1_lines: Final = _data_lines(v1.content)
native_events: Final = [json.loads(line) for line in native_lines]
v1_events: Final = [json.loads(line) for line in v1_lines]
assert (
[event["type"] for event in native_events]
== [event["type"] for event in v1_events]
== [
"response.created",
"response.output_item.added",
"response.function_call_arguments.delta",
"response.output_item.done",
"response.completed",
]
)
restored: Final = {"name": "sleep", "namespace": "clock"}
for native_line, v1_line, native_event, v1_event in zip(native_lines, v1_lines, native_events, v1_events):
match v1_event["type"]:
case "response.output_item.added" | "response.output_item.done":
assert native_event == {**v1_event, "item": {**v1_event["item"], **restored}}
case "response.completed":
(v1_call,) = v1_event["response"]["output"]
assert native_event == {
**v1_event,
"response": {**v1_event["response"], "output": [{**v1_call, **restored}]},
}
case _:
assert native_line == v1_line
def test_encrypted_reasoning_stays_only_when_every_deployment_the_router_may_pick_can_decrypt_it(
client: TestClient, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None: