diff --git a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py index 8ef4c64dd92..ac73af18293 100644 --- a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py @@ -2,14 +2,15 @@ import hashlib import hmac import json import re -from collections.abc import Iterator, Mapping +from collections.abc import AsyncIterable, AsyncIterator, Iterator, Mapping +from dataclasses import dataclass from types import MappingProxyType from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol import httpx from fastapi import APIRouter, Depends, HTTPException, Request, Response from fastapi.responses import StreamingResponse -from pydantic import TypeAdapter, ValidationError +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from starlette.types import Message import litellm @@ -39,10 +40,13 @@ from litellm.proxy.common_utils.http_parsing_utils import ( ) from litellm.proxy.pass_through_endpoints.pass_through_endpoints import pass_through_request from litellm.proxy.response_api_endpoints.endpoints import responses_api -from litellm.responses.additional_tools import with_additional_tools_hoisted, with_namespace_members_as_tools +from litellm.responses.additional_tools import with_additional_tools_hoisted, with_namespaces_flattened from litellm.responses.litellm_completion_transformation.reasoning_items import is_other_provider_reasoning_item +from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import EncryptedContentAffinityCheck +from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.custom_http import httpxSpecialProvider +from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.router import Deployment from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager, supports_none_reasoning_effort @@ -76,6 +80,12 @@ _ACCOUNT_SLUGS_CACHE_PREFIX: Final = "codex_account_slugs:" _MODELS_ETAG_CACHE_PREFIX: Final = "codex_models_etag:" _MODELS_ETAG_TTL_SECONDS: Final = 300 _OPENAI_API_PROVIDERS: Final = frozenset({LlmProviders.OPENAI.value, LlmProviders.AZURE.value}) +_NAMESPACE_TOOL_PROVIDERS: Final = frozenset( + {*_OPENAI_API_PROVIDERS, LlmProviders.FIREWORKS_AI.value, LlmProviders.BEDROCK_MANTLE.value} +) +_TOOL_DEFINITIONS: Final = TypeAdapter(tuple[dict[str, object], ...]) +_NOTHING_FLATTENED: Final[Mapping[str, tuple[str, str]]] = MappingProxyType({}) +_SSE_DATA_FIELD: Final = "data:" _HEADERS: Final = frozenset( { "authorization", @@ -95,6 +105,23 @@ _HEADERS: Final = frozenset( ) +class _FunctionCallName(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + type: Literal["function_call"] + name: str + + +class _ResponseOutput(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + output: tuple[object, ...] = () + + +@dataclass(frozen=True, slots=True) +class _LiteLLMRequest: + body: Mapping[str, object] + flattened_names: Mapping[str, tuple[str, str]] + + def _is_chatgpt(deployment: Deployment) -> bool: return ( deployment.litellm_params.custom_llm_provider == _CHATGPT_PROVIDER @@ -373,39 +400,132 @@ def _is_openai_api(deployment: Deployment) -> bool: return resolved is not None and resolved[1] in _OPENAI_API_PROVIDERS -def _bridges_to_chat_completions(deployment: Deployment) -> bool: +def _takes_namespace_tools(deployment: Deployment) -> bool: resolved: Final = _model_and_provider(deployment) + if resolved is None: + return False + model, provider = resolved return ( - resolved is not None - and ProviderConfigManager.get_provider_responses_api_config( - provider=resolved[1], model=resolved[0], api_base=deployment.litellm_params.api_base + provider in _NAMESPACE_TOOL_PROVIDERS + or ProviderConfigManager.get_provider_responses_api_config( + provider=provider, model=model, api_base=deployment.litellm_params.api_base ) is None ) -def _with_tools_the_targets_accept( - body: Mapping[str, object], deployments: tuple[Deployment, ...] -) -> Mapping[str, object]: +def _flattened_names(body: Mapping[str, object]) -> Mapping[str, tuple[str, str]]: + try: + tools: Final = _TOOL_DEFINITIONS.validate_python(body.get("tools") or ()) + except ValidationError: + return _NOTHING_FLATTENED + names: Final = LiteLLMCompletionResponsesConfig.namespace_tool_name_map(tools) + return MappingProxyType( + {flat_name: (namespace, tool) for flat_name, (namespace, tool) in names.items() if flat_name != tool} + ) + + +def _with_tools_the_targets_accept(body: Mapping[str, object], deployments: tuple[Deployment, ...]) -> _LiteLLMRequest: if all(_is_openai_api(deployment) for deployment in deployments): - return body + return _LiteLLMRequest(body=body, flattened_names=_NOTHING_FLATTENED) hoisted: Final = with_additional_tools_hoisted(body) - if all(_bridges_to_chat_completions(deployment) for deployment in deployments): - return hoisted - return with_namespace_members_as_tools(hoisted) + if all(_takes_namespace_tools(deployment) for deployment in deployments): + return _LiteLLMRequest(body=hoisted, flattened_names=_NOTHING_FLATTENED) + flattened_names: Final = _flattened_names(hoisted) + return _LiteLLMRequest(body=with_namespaces_flattened(hoisted, flattened_names), flattened_names=flattened_names) -def _litellm_body(body: Mapping[str, object], llm_router: "Router | None", team_id: str | None) -> Mapping[str, object]: +def _litellm_request(body: Mapping[str, object], llm_router: "Router | None", team_id: str | None) -> _LiteLLMRequest: wrapped: Final = _with_wrapped_reasoning(_with_provider_default_reasoning(body, llm_router, team_id)) model: Final = body.get("model") if not isinstance(model, str) or llm_router is None: - return wrapped + return _LiteLLMRequest(body=wrapped, flattened_names=_NOTHING_FLATTENED) deployments: Final = _candidate_deployments(model, llm_router, team_id) return _with_tools_the_targets_accept( _with_reasoning_the_targets_can_decrypt(wrapped, model, llm_router, deployments), deployments ) +def _namespaced(item: object, flattened_names: Mapping[str, tuple[str, str]]) -> tuple[str, str] | None: + try: + call: Final = _FunctionCallName.model_validate(item) + except ValidationError: + return None + return flattened_names.get(call.name) + + +def _with_namespace(item: object, flattened_names: Mapping[str, tuple[str, str]]) -> object: + namespaced: Final = _namespaced(item, flattened_names) + if namespaced is None: + return item + namespace, name = namespaced + restored: Final = {"name": name, "namespace": namespace} + match item: + case BaseModel(): + return item.model_copy(update=restored) + case _: + return {**_json_object(item), **restored} + + +def _response_output(response: object) -> tuple[object, ...]: + try: + return _ResponseOutput.model_validate(response).output + except ValidationError: + return () + + +def _output_with_namespaces( + response: object, flattened_names: Mapping[str, tuple[str, str]] +) -> tuple[object, ...] | None: + output: Final = _response_output(response) + if all(_namespaced(item, flattened_names) is None for item in output): + return None + return tuple(_with_namespace(item, flattened_names) for item in output) + + +def _event_with_namespaces( + event: Mapping[str, object], flattened_names: Mapping[str, tuple[str, str]] +) -> Mapping[str, object] | None: + match event.get("type"): + case "response.output_item.added" | "response.output_item.done": + item: Final = event.get("item") + if _namespaced(item, flattened_names) is None: + return None + return {**event, "item": _with_namespace(item, flattened_names)} + case "response.completed" | "response.incomplete" | "response.failed": + response: Final = _json_object(event.get("response")) + output: Final = _output_with_namespaces(response, flattened_names) + return None if output is None else {**event, "response": {**response, "output": list(output)}} + case _: + return None + + +def _sse_line_with_namespaces(line: str, flattened_names: Mapping[str, tuple[str, str]]) -> str: + if not line.startswith(_SSE_DATA_FIELD): + return line + try: + event: Final = _JSON_OBJECT.validate_json(line.removeprefix(_SSE_DATA_FIELD)) + except ValidationError: + return line + restored: Final = _event_with_namespaces(event, flattened_names) + return line if restored is None else f"{_SSE_DATA_FIELD} {json.dumps(restored)}" + + +def _frame_with_namespaces( + frame: str | bytes | memoryview, flattened_names: Mapping[str, tuple[str, str]] +) -> str | bytes | memoryview: + if not isinstance(frame, str) or not any(flat_name in frame for flat_name in flattened_names): + return frame + return "\n".join(_sse_line_with_namespaces(line, flattened_names) for line in frame.split("\n")) + + +async def _frames_with_namespaces( + frames: AsyncIterable[str | bytes | memoryview], flattened_names: Mapping[str, tuple[str, str]] +) -> AsyncIterator[str | bytes | memoryview]: + async for frame in frames: + yield _frame_with_namespaces(frame, flattened_names) + + def _is_unpersisted_reasoning_reference(item: object) -> bool: try: fields: Final = _JSON_OBJECT.validate_python(item) @@ -553,6 +673,33 @@ async def _catalog( return _with_etag(upstream, merged if merged is not None else upstream_body, etag) +async def _litellm_responses( + request: Request, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth, + body: Mapping[str, object], + llm_router: "Router | None", +) -> object: + litellm_request: Final = _litellm_request(body, llm_router, user_api_key_dict.team_id) + served: Final = await responses_api( + _as_litellm_responses_request(request, json.dumps(litellm_request.body).encode(), litellm_request.body), + fastapi_response, + user_api_key_dict.model_copy(update={"request_route": _LITELLM_RESPONSES_PATH}), + ) + flattened_names: Final = litellm_request.flattened_names + if not flattened_names: + return served + match served: + case StreamingResponse(): + served.body_iterator = _frames_with_namespaces(served.body_iterator, flattened_names) + return served + case ResponsesAPIResponse(): + output: Final = _output_with_namespaces(served, flattened_names) + return served if output is None else served.model_copy(update={"output": list(output)}) + case _: + return served + + async def _responses( request: Request, fastapi_response: Response, @@ -565,12 +712,7 @@ async def _responses( body: Final = _JSON_OBJECT.validate_python(await read_request_body(request)) if await _serves_through_litellm(body.get("model"), llm_router, user_api_key_dict, headers, cache): - litellm_body: Final = _litellm_body(body, llm_router, user_api_key_dict.team_id) - return await responses_api( - _as_litellm_responses_request(request, json.dumps(litellm_body).encode(), litellm_body), - fastapi_response, - user_api_key_dict.model_copy(update={"request_route": _LITELLM_RESPONSES_PATH}), - ) + return await _litellm_responses(request, fastapi_response, user_api_key_dict, body, llm_router) chatgpt_body: Final = _chatgpt_body(body, get_client_requested_model(request)) upstream: Final = await pass_through_request( request=request, diff --git a/litellm/responses/additional_tools.py b/litellm/responses/additional_tools.py index 03972fbd441..8fa222f0bcc 100644 --- a/litellm/responses/additional_tools.py +++ b/litellm/responses/additional_tools.py @@ -3,9 +3,14 @@ from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass from itertools import chain, groupby from types import MappingProxyType -from typing import Final, cast # noqa: TID251 # validating the openai tool union strips vendor keys from raw tools +from typing import ( + Annotated, + Final, + Literal, + cast, # noqa: TID251 # validating the openai tool union strips vendor keys from raw tools +) -from pydantic import TypeAdapter, ValidationError +from pydantic import ConfigDict, Field, TypeAdapter, ValidationError from litellm._logging import verbose_logger from litellm.types.llms.base import LiteLLMBaseModel @@ -13,8 +18,8 @@ from litellm.types.llms.openai import ALL_RESPONSES_API_TOOL_PARAMS, ResponseInp ADDITIONAL_TOOLS_INPUT_ITEM_TYPE: Final = "additional_tools" _NAMESPACE_TOOL_TYPE: Final = "namespace" -_QUALIFIED_NAME_SEPARATOR: Final = "__" -_TOOLS: Final = TypeAdapter(tuple[object, ...]) +_ITEMS: Final = TypeAdapter(tuple[object, ...]) +_JSON_OBJECT: Final = TypeAdapter(dict[str, object]) class _InputItemType(LiteLLMBaseModel): @@ -25,6 +30,13 @@ class _ToolList(LiteLLMBaseModel): tools: tuple[Mapping[str, object], ...] = () +class _NamespacedFunctionCall(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + type: Literal["function_call"] + name: str + namespace: Annotated[str, Field(min_length=1)] + + @dataclass(frozen=True, slots=True) class HoistedAdditionalTools: input: str | ResponseInputParam @@ -119,7 +131,7 @@ def with_additional_tools_hoisted(request: Mapping[str, object]) -> Mapping[str, return request input_items: Final = cast(ResponseInputParam, request_input) # cast-ok: forwarded as sent, as inference does top_level_tools: Final = cast( # cast-ok: tools are forwarded as sent, as inference does - "tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]", _TOOLS.validate_python(request.get("tools") or ()) + "tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]", _ITEMS.validate_python(request.get("tools") or ()) ) hoisted: Final = hoist_additional_tools(input_items, top_level_tools) if not hoisted.hoisted: @@ -127,20 +139,45 @@ def with_additional_tools_hoisted(request: Mapping[str, object]) -> Mapping[str, return {**request, "input": hoisted.input, "tools": list(hoisted.tools)} -def _qualified_member(namespace: str, member: Mapping[str, object]) -> Mapping[str, object]: - prefix: Final = f"{namespace.removesuffix(_QUALIFIED_NAME_SEPARATOR)}{_QUALIFIED_NAME_SEPARATOR}" - return {**member, "name": f"{prefix}{member.get('name')}"} +def _flattened_member( + namespace: str, member: Mapping[str, object], flat_names: Mapping[tuple[str, str], str] +) -> Mapping[str, object]: + flat_name: Final = flat_names.get((namespace, str(member.get("name") or ""))) + return member if flat_name is None else {**member, "name": flat_name} -def _members_as_tools(tool: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: +def _members_as_tools( + tool: Mapping[str, object], flat_names: Mapping[tuple[str, str], str] +) -> tuple[Mapping[str, object], ...]: if tool.get("type") != _NAMESPACE_TOOL_TYPE: return (tool,) namespace: Final = str(tool.get("name") or "") - return tuple(_qualified_member(namespace, member) for member in _tool_list_of(tool)) + return tuple(_flattened_member(namespace, member, flat_names) for member in _tool_list_of(tool)) -def with_namespace_members_as_tools(request: Mapping[str, object]) -> Mapping[str, object]: - tools: Final = _tool_list_of(request) - if not any(tool.get("type") == _NAMESPACE_TOOL_TYPE for tool in tools): +def _flattened_call(item: object, flat_names: Mapping[tuple[str, str], str]) -> object: + try: + call: Final = _NamespacedFunctionCall.model_validate(item) + except ValidationError: + return item + flat_name: Final = flat_names.get((call.namespace, call.name)) + if flat_name is None: + return item + fields: Final = _JSON_OBJECT.validate_python(item) + return {**{key: value for key, value in fields.items() if key != "namespace"}, "name": flat_name} + + +def with_namespaces_flattened( + request: Mapping[str, object], namespaced_names: Mapping[str, tuple[str, str]] +) -> Mapping[str, object]: + if not namespaced_names: return request - return {**request, "tools": list(chain.from_iterable(map(_members_as_tools, tools)))} + flat_names: Final = MappingProxyType({namespaced: flat for flat, namespaced in namespaced_names.items()}) + tools: Final = chain.from_iterable(_members_as_tools(tool, flat_names) for tool in _tool_list_of(request)) + request_input: Final = request.get("input") + replayed: Final = ( + {"input": [_flattened_call(item, flat_names) for item in _ITEMS.validate_python(request_input)]} + if isinstance(request_input, list) + else {} + ) + return {**request, "tools": list(tools), **replayed} diff --git a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py index 07dcc2859d6..f05d6d62fcf 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py @@ -1181,7 +1181,7 @@ _ADDITIONAL_TOOLS: Final = { pytest.param( "grok-4.7", [_USER_TURN], - ["exec_command", "collaboration__followup_task", "mcp__docs__search"], + ["exec_command", "collaboration__followup_task", "mcp__docs____search"], id="xai gets the item's tools as top-level functions", ), pytest.param("oai-luna", [_ADDITIONAL_TOOLS, _USER_TURN], ["exec_command"], id="the openai api keeps the item"), @@ -1268,6 +1268,142 @@ def test_codex_namespace_tool_called_through_the_chat_bridge_comes_back_with_its assert (call["namespace"], call["name"]) == ("mcp__docs__", "search") +_CLOCK_NAMESPACE: Final = _namespace("clock", "sleep") +_CLOCK_TOOLS: Final = {"type": "additional_tools", "role": "developer", "tools": [_CLOCK_NAMESPACE]} +_FLAT_SLEEP_TOOL: Final = {"type": "function", "name": "clock__sleep", "parameters": {"type": "object"}} +_FLAT_SLEEP_CALL: Final = { + "type": "function_call", + "id": "fc_sleep", + "call_id": "call_sleep", + "name": "clock__sleep", + "arguments": '{"seconds":1}', + "status": "completed", +} + + +def test_codex_namespace_tool_reaches_fireworks_as_a_namespace( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router +) -> None: + _account_catalog(respx_mock) + provider: Final = _open_weights_provider(respx_mock) + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": "open-weights", "input": [_CLOCK_TOOLS, _USER_TURN], "stream": False, "store": False}, + headers=_ACCOUNT_HEADERS, + ) + assert response.status_code == 200, response.text + assert json.loads(provider.calls.last.request.content)["tools"] == [_CLOCK_NAMESPACE] + + +def test_xai_namespace_call_round_trips_to_codex_with_its_namespace( + client: TestClient, respx_mock: respx.MockRouter, xai_and_openai_router: Router +) -> None: + _account_catalog(respx_mock) + xai: Final = respx_mock.post(f"{_XAI_BASE}/responses").respond( + json={**_provider_completed(""), "output": [_FLAT_SLEEP_CALL]} + ) + replayed_call: Final = {**_FLAT_SLEEP_CALL, "name": "sleep", "namespace": "clock"} + replayed_output: Final = {"type": "function_call_output", "call_id": "call_sleep", "output": "slept"} + response: Final = client.post( + "/backend-api/codex/responses", + json={ + "model": "grok-4.7", + "input": [_CLOCK_TOOLS, _USER_TURN, replayed_call, replayed_output], + "stream": False, + "store": False, + }, + headers=_ACCOUNT_HEADERS, + ) + assert response.status_code == 200, response.text + sent: Final = json.loads(xai.calls.last.request.content) + assert sent["tools"] == [_FLAT_SLEEP_TOOL] + assert sent["input"][1:] == [_FLAT_SLEEP_CALL, replayed_output] + (call,) = (item for item in response.json()["output"] if item["type"] == "function_call") + assert (call["name"], call["namespace"], call["call_id"]) == ("sleep", "clock", "call_sleep") + + +def _sse_events(*events: dict[str, object]) -> bytes: + return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events).encode() + + +def _xai_sleep_call_stream() -> bytes: + completed: Final = {**_provider_completed(""), "output": [_FLAT_SLEEP_CALL]} + return _sse_events( + { + "type": "response.created", + "sequence_number": 0, + "response": {**completed, "status": "in_progress", "output": []}, + }, + { + "type": "response.output_item.added", + "sequence_number": 1, + "output_index": 0, + "item": {**_FLAT_SLEEP_CALL, "arguments": "", "status": "in_progress"}, + }, + { + "type": "response.function_call_arguments.delta", + "sequence_number": 2, + "item_id": "fc_sleep", + "output_index": 0, + "delta": '{"seconds":1}', + }, + {"type": "response.output_item.done", "sequence_number": 3, "output_index": 0, "item": _FLAT_SLEEP_CALL}, + {"type": "response.completed", "sequence_number": 4, "response": completed}, + ) + + +def _data_lines(stream: bytes) -> list[bytes]: + return [line.removeprefix(b"data: ") for line in stream.splitlines() if line.startswith(b"data: {")] + + +def test_xai_streamed_namespace_call_reaches_codex_with_its_namespace_and_other_events_unchanged( + client: TestClient, respx_mock: respx.MockRouter, xai_and_openai_router: Router +) -> None: + _account_catalog(respx_mock) + respx_mock.post(f"{_XAI_BASE}/responses").respond( + content=_xai_sleep_call_stream(), headers={"content-type": "text/event-stream"} + ) + native: Final = client.post( + "/backend-api/codex/responses", + json={"model": "grok-4.7", "input": [_CLOCK_TOOLS, _USER_TURN], "stream": True, "store": False}, + headers=_ACCOUNT_HEADERS, + ) + v1: Final = client.post( + "/v1/responses", + json={"model": "grok-4.7", "input": [_USER_TURN], "tools": [_FLAT_SLEEP_TOOL], "stream": True, "store": False}, + headers={"authorization": f"Bearer {MASTER_KEY}"}, + ) + assert native.status_code == v1.status_code == 200 + native_lines: Final = _data_lines(native.content) + v1_lines: Final = _data_lines(v1.content) + native_events: Final = [json.loads(line) for line in native_lines] + v1_events: Final = [json.loads(line) for line in v1_lines] + assert ( + [event["type"] for event in native_events] + == [event["type"] for event in v1_events] + == [ + "response.created", + "response.output_item.added", + "response.function_call_arguments.delta", + "response.output_item.done", + "response.completed", + ] + ) + restored: Final = {"name": "sleep", "namespace": "clock"} + for native_line, v1_line, native_event, v1_event in zip(native_lines, v1_lines, native_events, v1_events): + match v1_event["type"]: + case "response.output_item.added" | "response.output_item.done": + assert native_event == {**v1_event, "item": {**v1_event["item"], **restored}} + case "response.completed": + (v1_call,) = v1_event["response"]["output"] + assert native_event == { + **v1_event, + "response": {**v1_event["response"], "output": [{**v1_call, **restored}]}, + } + case _: + assert native_line == v1_line + + def test_encrypted_reasoning_stays_only_when_every_deployment_the_router_may_pick_can_decrypt_it( client: TestClient, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ) -> None: