From c4f6d69af4770da3c6c043583a797470acdca23f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 17:09:07 -0700 Subject: [PATCH 01/20] feat(proxy): mixed Codex model picker and LiteLLM routing on native codex routes The native Codex catalog now lists the key's other chat and responses models after ChatGPT's, within Codex's catalog size limit, and drops the ETag so the merged list refreshes on Codex's timer. A request for a model configured by exact name on another provider goes through /v1/responses without the ChatGPT bearer or account header, while ChatGPT names and wildcard-only matches keep the subscription passthrough. Reasoning items another provider produced are dropped before forwarding to ChatGPT. --- .../chatgpt_subscription_passthrough.md | 32 +- litellm/proxy/_lazy_openapi_snapshot.json | 6 +- .../proxy/common_utils/codex_model_catalog.py | 77 ++++- .../codex_passthrough_endpoints.py | 159 ++++++++- litellm/proxy/proxy_server.py | 194 +++++++---- .../reasoning_items.py | 28 +- .../common_utils/test_codex_model_catalog.py | 21 ++ .../test_codex_passthrough_endpoints.py | 314 +++++++++++++++++- .../test_reasoning_items.py | 59 ++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 8 +- 10 files changed, 788 insertions(+), 110 deletions(-) diff --git a/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md b/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md index e1f606a2eb7..f0dc6481bd5 100644 --- a/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md +++ b/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md @@ -46,7 +46,35 @@ For a managed workspace rollout, use OpenAI's [gateway configuration guide](http ## Native compatibility and Pi -For a native Codex model catalog and request format, change Codex's `base_url` to `https://litellm.example.com/backend-api/codex`. This route uses the same credentials but does not need a model deployment or resolve model aliases. Restricted keys must allow the upstream model name. +For a native Codex model catalog and request format, change Codex's `base_url` to `https://litellm.example.com/backend-api/codex`. This route uses the same credentials. A ChatGPT model keeps its upstream name and needs no deployment of its own. + +### Mix ChatGPT and LiteLLM models + +On the native route, Codex's model picker shows OpenAI's catalog first and then the other chat and responses models the virtual key can use. Pick one of those and Codex's requests go through LiteLLM's `/v1/responses` with the key's usual routing, budgets, guardrails, and logging. The proxy removes the ChatGPT bearer and account header from that request, so the other provider and your callbacks never receive them. Only a model listed under its exact `model_name` on another provider is routed this way. A ChatGPT model, or a name that only a wildcard deployment matches, keeps the passthrough, so a model OpenAI adds later still uses the subscription. When the key can use many models, the picker lists the ones that fit in Codex's catalog size limit. + +Codex sends earlier reasoning items back with each turn. When a session switches from another provider's model to a ChatGPT model, the proxy drops the reasoning items that other provider produced before forwarding the request to OpenAI, which would reject them. Messages, tool calls, and ChatGPT's own reasoning are kept. + +The merged catalog carries no ETag, so Codex picks up a newly allowed model on its next regular catalog refresh. + +### Restrict native models with an access group + +A key with a model allowlist needs a rule for ChatGPT names that OpenAI adds over time. Put the subscription wildcards in an access group: + +```yaml +model_list: + - model_name: gpt-* + litellm_params: + model: chatgpt/gpt-* + model_info: + access_groups: ["chatgpt-native"] + - model_name: codex-* + litellm_params: + model: chatgpt/codex-* + model_info: + access_groups: ["chatgpt-native"] +``` + +Then add `chatgpt-native` to the key's models, next to any other model names it should reach. The key can call every ChatGPT model that matches the wildcards, including ones the proxy configuration does not name. These wildcards are also served on `/v1`, where a request without a ChatGPT login falls back to the proxy's own saved login if it has one, so keep that login unset as described above. ### Configure Pi @@ -71,7 +99,7 @@ Keep Pi's subscription login as the provider credential. The header value names Send a prompt and check the completed request in LiteLLM's Logs page. Its gateway user/key and token counts should match the caller. Give each person their own LiteLLM virtual key for separate attribution. The gateway credential and ChatGPT login are independently authenticated; LiteLLM does not establish that both identities belong to the same person. -The native routes are `GET /backend-api/codex/models` and `POST /backend-api/codex/responses`. Model requests use the upstream model name, so a restricted LiteLLM key must allow that name. OpenAI's model catalog is returned in its native format; seeing a model there does not override gateway permissions. +The native routes are `GET /backend-api/codex/models` and `POST /backend-api/codex/responses`. Model requests use the upstream model name, so a restricted LiteLLM key needs the `chatgpt-native` access group above or that exact name. OpenAI's model catalog is returned in its native format; seeing a model there does not override gateway permissions. The proxy forwards the ChatGPT bearer and account header to the fixed OpenAI Codex endpoint. It removes gateway credentials and cookies from the upstream request and redacts credential headers in logging callbacks. The client remains responsible for refresh and reauthentication. Native passthrough returns authentication and quota errors without switching subscriptions. Standard model routes retain the caller login across ChatGPT retries; explicitly configured cross-provider fallbacks use their own credentials. An explicit workspace header is preserved for OpenAI to validate. diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index b10b5ae98b5..b9a4b359641 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -8734,7 +8734,7 @@ "paths": { "/backend-api/codex/{endpoint}": { "get": { - "description": "Forward a client's ChatGPT subscription with separate LiteLLM authentication.", + "description": "Forward a client's ChatGPT subscription with separate LiteLLM authentication, listing and serving the\nkey's other chat models through LiteLLM.", "operationId": "codex_passthrough_route_backend_api_codex__endpoint__get", "parameters": [ { @@ -8782,7 +8782,7 @@ ] }, "post": { - "description": "Forward a client's ChatGPT subscription with separate LiteLLM authentication.", + "description": "Forward a client's ChatGPT subscription with separate LiteLLM authentication, listing and serving the\nkey's other chat models through LiteLLM.", "operationId": "codex_passthrough_route_backend_api_codex__endpoint__post", "parameters": [ { @@ -8825,7 +8825,7 @@ } } }, - "description": "Unmodified Codex response or server-sent events" + "description": "Codex response, the model catalog with the key's LiteLLM models, or server-sent events" }, "422": { "content": { diff --git a/litellm/proxy/common_utils/codex_model_catalog.py b/litellm/proxy/common_utils/codex_model_catalog.py index 9b42f99305e..f8d578fb529 100644 --- a/litellm/proxy/common_utils/codex_model_catalog.py +++ b/litellm/proxy/common_utils/codex_model_catalog.py @@ -18,6 +18,7 @@ a newer tag. from __future__ import annotations +import json from collections.abc import Mapping, Sequence from dataclasses import dataclass from functools import cache @@ -43,6 +44,8 @@ CODEX_CHAT_MODES: Final = frozenset({"chat", "responses"}) _TierId = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)] _BODY_PREFIX: Final = '{"models":[' _BODY_SUFFIX: Final = "]}" +_JSON_OBJECT: Final = TypeAdapter(dict[str, object]) +_JSON_OBJECTS: Final = TypeAdapter(tuple[dict[str, object], ...]) class CodexServiceTier(LiteLLMBaseModel): @@ -311,13 +314,12 @@ def _bytes_left_after(left: int, size: int) -> int: return left - size if size <= left else left -def _fits(serialized: Sequence[str], byte_limit: int | None) -> tuple[bool, ...]: - """Whether each entry, taken in order, fits in what is left of a body of at most `byte_limit` - bytes: an entry larger than the bytes left is passed over and a smaller one after it is still - taken, each entry counted with its separating comma and the last one's comma given back.""" - if byte_limit is None: +def _fits(serialized: Sequence[str], budget: int | None) -> tuple[bool, ...]: + """Whether each entry, taken in order, fits in what is left of `budget` bytes: an entry larger + than the bytes left is passed over and a smaller one after it is still taken, each entry + counted with its separating comma.""" + if budget is None: return (True,) * len(serialized) - budget: Final = byte_limit - len(_BODY_PREFIX) - len(_BODY_SUFFIX) + 1 left: Final = tuple( accumulate((len(entry.encode()) + 1 for entry in serialized), _bytes_left_after, initial=budget) ) @@ -325,17 +327,29 @@ def _fits(serialized: Sequence[str], byte_limit: int | None) -> tuple[bool, ...] def _kept_positions( - entries: Sequence[CodexCatalogEntry], serialized: Sequence[str], byte_limit: int | None + entries: Sequence[CodexCatalogEntry], serialized: Sequence[str], budget: int | None ) -> frozenset[int]: """The listing positions that survive the byte limit: an entry offering a service tier is taken ahead of one offering none, each group in listing order, since a model without a tier is driven by name just as well off Codex's bundled catalog; an entry too large for the bytes left never costs the smaller ones after it their place.""" survival: Final = sorted(range(len(entries)), key=lambda position: (not entries[position].service_tiers, position)) - fits: Final = _fits(tuple(serialized[position] for position in survival), byte_limit) + fits: Final = _fits(tuple(serialized[position] for position in survival), budget) return frozenset(position for position, fit in zip(survival, fits, strict=True) if fit) +def _entries( + drivable: Sequence[CodexCatalogRow], + stock: Mapping[str, CodexStockModel] | None, + instructions: str | None, + first_priority: int, +) -> tuple[CodexCatalogEntry, ...]: + known: Final = bundled_codex_models() if stock is None else stock + prompt: Final = codex_base_instructions() if instructions is None else instructions + served: Final = frozenset(row.id for row in drivable) + return tuple(_entry(first_priority + index, row, known, served, prompt) for index, row in enumerate(drivable)) + + @dataclass(frozen=True, slots=True) class CodexCatalogBody: """A `ModelsResponse` body, the ids it lists, and the chat-capable ids the byte limit left out.""" @@ -361,13 +375,12 @@ def codex_models_response_json( for the bytes left passed over, every kept entry at its listing position; `byte_limit=None` keeps every entry. """ - known: Final = bundled_codex_models() if stock is None else stock - prompt: Final = codex_base_instructions() if instructions is None else instructions drivable: Final = tuple(row for row in rows if _codex_can_drive(row)) - served: Final = frozenset(row.id for row in drivable) - entries: Final = tuple(_entry(index, row, known, served, prompt) for index, row in enumerate(drivable)) + entries: Final = _entries(drivable, stock, instructions, 0) serialized: Final = tuple(_serialized(entry) for entry in entries) - kept: Final = _kept_positions(entries, serialized, byte_limit) + kept: Final = _kept_positions( + entries, serialized, None if byte_limit is None else byte_limit - len(_BODY_PREFIX) - len(_BODY_SUFFIX) + 1 + ) return CodexCatalogBody( json=f"{_BODY_PREFIX}{','.join(serialized[position] for position in sorted(kept))}{_BODY_SUFFIX}", listed=tuple(row.id for position, row in enumerate(drivable) if position in kept), @@ -375,6 +388,44 @@ def codex_models_response_json( ) +def _priority(model: Mapping[str, object]) -> int: + priority: Final = model.get("priority") + return priority if isinstance(priority, int) else -1 + + +def _compact_json(value: object) -> str: + return json.dumps(value, separators=(",", ":")) + + +def merged_codex_catalog( + upstream_body: bytes, + rows: Sequence[CodexCatalogRow], + *, + stock: Mapping[str, CodexStockModel] | None = None, + instructions: str | None = None, + byte_limit: int = CODEX_CATALOG_BYTE_LIMIT, +) -> str | None: + try: + top: Final = _JSON_OBJECT.validate_json(upstream_body) + upstream: Final = _JSON_OBJECTS.validate_python(top.get("models")) + except ValidationError: + return None + slugs: Final = frozenset(model.get("slug") for model in upstream) + added: Final = tuple(row for row in rows if _codex_can_drive(row) and row.id not in slugs) + first_priority: Final = max((_priority(model) for model in upstream), default=-1) + 1 + entries: Final = _entries(added, stock, instructions, first_priority) + added_models: Final = tuple( + _JSON_OBJECT.validate_python(entry.model_dump(mode="json", exclude_unset=isinstance(entry, CodexStockModel))) + for entry in entries + ) + serialized: Final = tuple(_compact_json(model) for model in added_models) + budget: Final = byte_limit - len(_compact_json({**top, "models": upstream})) + kept: Final = tuple(sorted(_kept_positions(entries, serialized, budget))) + if not kept: + return None + return _compact_json({**top, "models": [*upstream, *(added_models[position] for position in kept)]}) + + def _catalog_row( row: ModelInfoResponse, lookup_id: str, label_id: str, llm_router: Router | None, team_id: str | None ) -> CodexCatalogRow: diff --git a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py index 3087faa9f67..1a38271b986 100644 --- a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py @@ -1,15 +1,37 @@ import hmac +from collections.abc import Iterator, Mapping from types import MappingProxyType -from typing import Annotated, Final, Literal +from typing import TYPE_CHECKING, Annotated, Final, Literal from fastapi import APIRouter, Depends, HTTPException, Request, Response +from fastapi.responses import StreamingResponse +from pydantic import TypeAdapter, ValidationError +from starlette.types import Message +import litellm from litellm.llms.chatgpt.common_utils import CHATGPT_API_BASE from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.codex_model_catalog import bundled_codex_models, merged_codex_catalog +from litellm.proxy.common_utils.http_parsing_utils import ( + read_request_body, # pyright: ignore[reportUnknownVariableType] # returns a bare dict, validated at the call +) from litellm.proxy.pass_through_endpoints.pass_through_endpoints import pass_through_request +from litellm.proxy.response_api_endpoints.endpoints import responses_api +from litellm.responses.litellm_completion_transformation.reasoning_items import is_other_provider_reasoning_item +from litellm.types.router import Deployment + +if TYPE_CHECKING: + from litellm.router import Router router: Final = APIRouter() +_CHATGPT_PROVIDER: Final = "chatgpt" +_CHATGPT_CREDENTIAL_HEADERS: Final = frozenset({b"authorization", b"chatgpt-account-id"}) +_MERGED_CATALOG_DROPPED_HEADERS: Final = frozenset({"etag", "content-length", "content-type"}) +_MODELS_ETAG_HEADER: Final = "x-models-etag" +_LITELLM_RESPONSES_PATH: Final = "/v1/responses" +_JSON_OBJECT: Final = TypeAdapter(dict[str, object]) +_INPUT_ITEMS: Final = TypeAdapter(tuple[object, ...]) _HEADERS: Final = frozenset( { "authorization", @@ -29,10 +51,124 @@ _HEADERS: Final = frozenset( ) -@router.get("/backend-api/codex/{endpoint}", tags=["ChatGPT Pass-through", "pass-through"]) +def _is_chatgpt(deployment: Deployment) -> bool: + return ( + deployment.litellm_params.custom_llm_provider == _CHATGPT_PROVIDER + or deployment.litellm_params.model.partition("/")[0] == _CHATGPT_PROVIDER + ) + + +def _serves_by_name(deployment: Deployment, names: frozenset[str]) -> bool: + return not _is_chatgpt(deployment) and ( + deployment.model_name in names or deployment.model_info.team_public_model_name in names + ) + + +def _routes_to_litellm(model: object, llm_router: "Router | None", team_id: str | None) -> bool: + if ( + not isinstance(model, str) + or llm_router is None + or model in bundled_codex_models() + or f"{_CHATGPT_PROVIDER}/{model}" in litellm.model_cost # pyright: ignore[reportUnknownMemberType] # registry is dynamically typed + ): + return False + candidates: Final = tuple( + llm_router.get_deployment(model_id) for model_id in llm_router.get_candidate_model_ids_for_route(model, team_id) + ) + deployments: Final = tuple(deployment for deployment in candidates if deployment is not None) + names: Final = frozenset({model, llm_router.get_model_from_alias(model=model) or model}) + return bool(deployments) and all(_serves_by_name(deployment, names) for deployment in deployments) + + +def _as_litellm_responses_request(request: Request, body: bytes) -> Request: + replayed: Final[Iterator[Message]] = iter(({"type": "http.request", "body": body, "more_body": False},)) + + async def receive() -> Message: + return next(replayed, None) or await request.receive() + + return Request( + { + **request.scope, + "path": _LITELLM_RESPONSES_PATH, + "raw_path": _LITELLM_RESPONSES_PATH.encode(), + "headers": [ + (name, value) for name, value in request.headers.raw if name not in _CHATGPT_CREDENTIAL_HEADERS + ], + }, + receive, + ) + + +def _input_without_other_provider_reasoning(body: Mapping[str, object]) -> tuple[object, ...] | None: + try: + items: Final = _INPUT_ITEMS.validate_python(body.get("input")) + except ValidationError: + return None + kept: Final = tuple(item for item in items if not is_other_provider_reasoning_item(item)) + return kept if len(kept) < len(items) else None + + +async def _catalog(request: Request, user_api_key_dict: UserAPIKeyAuth, headers: Mapping[str, str]) -> Response: + from litellm.proxy.proxy_server import codex_catalog_rows_for_key, llm_router + + upstream: Final = await pass_through_request( + request=request, + target=f"{CHATGPT_API_BASE}/models", + custom_headers=dict(headers), + user_api_key_dict=user_api_key_dict, + custom_llm_provider=_CHATGPT_PROVIDER, + upstream_headers_filter=lambda _: headers, + ) + if llm_router is None or upstream.status_code != 200 or isinstance(upstream, StreamingResponse): + return upstream + rows: Final = await codex_catalog_rows_for_key(user_api_key_dict) + merged: Final = merged_codex_catalog( + bytes(upstream.body), + tuple(row for row in rows if _routes_to_litellm(row.id, llm_router, user_api_key_dict.team_id)), + ) + if merged is None: + return upstream + return Response( + content=merged, + status_code=upstream.status_code, + headers={ + name: value for name, value in upstream.headers.items() if name not in _MERGED_CATALOG_DROPPED_HEADERS + }, + media_type="application/json", + ) + + +async def _responses( + request: Request, fastapi_response: Response, user_api_key_dict: UserAPIKeyAuth, headers: Mapping[str, str] +) -> object: + from litellm.proxy.proxy_server import llm_router + + body: Final = _JSON_OBJECT.validate_python(await read_request_body(request)) + if _routes_to_litellm(body.get("model"), llm_router, user_api_key_dict.team_id): + return await responses_api( + _as_litellm_responses_request(request, await request.body()), + fastapi_response, + user_api_key_dict.model_copy(update={"request_route": _LITELLM_RESPONSES_PATH}), + ) + kept_input: Final = _input_without_other_provider_reasoning(body) + upstream: Final = await pass_through_request( + request=request, + target=f"{CHATGPT_API_BASE}/responses", + custom_headers=dict(headers), + user_api_key_dict=user_api_key_dict, + custom_body={**body, "input": list(kept_input)} if kept_input is not None else None, + custom_llm_provider=_CHATGPT_PROVIDER, + upstream_headers_filter=lambda _: headers, + ) + del upstream.headers[_MODELS_ETAG_HEADER] + return upstream + + +@router.get("/backend-api/codex/{endpoint}", tags=["ChatGPT Pass-through", "pass-through"], response_model=None) @router.post( "/backend-api/codex/{endpoint}", tags=["ChatGPT Pass-through", "pass-through"], + response_model=None, openapi_extra={ "requestBody": { "required": True, @@ -41,7 +177,7 @@ _HEADERS: Final = frozenset( }, responses={ 200: { - "description": "Unmodified Codex response or server-sent events", + "description": "Codex response, the model catalog with the key's LiteLLM models, or server-sent events", "content": { "application/json": {"schema": {"type": "object", "additionalProperties": True}}, "text/event-stream": {"schema": {"type": "string"}}, @@ -52,9 +188,11 @@ _HEADERS: Final = frozenset( async def codex_passthrough_route( endpoint: Literal["models", "responses"], request: Request, + fastapi_response: Response, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], -) -> Response: - """Forward a client's ChatGPT subscription with separate LiteLLM authentication.""" +) -> object: + """Forward a client's ChatGPT subscription with separate LiteLLM authentication, listing and serving the + key's other chat models through LiteLLM.""" expected_method: Final = "GET" if endpoint == "models" else "POST" if request.method != expected_method: raise HTTPException(status_code=405, detail="Method not allowed", headers={"Allow": expected_method}) @@ -77,11 +215,6 @@ async def codex_passthrough_route( raise HTTPException(status_code=401, detail="ChatGPT and LiteLLM credentials must be separate") headers: Final = MappingProxyType({name: value for name, value in request.headers.items() if name in _HEADERS}) - return await pass_through_request( - request=request, - target=f"{CHATGPT_API_BASE}/{endpoint}", - custom_headers=dict(headers), - user_api_key_dict=user_api_key_dict, - custom_llm_provider="chatgpt", - upstream_headers_filter=lambda _: headers, - ) + if endpoint == "models": + return await _catalog(request, user_api_key_dict, headers) + return await _responses(request, fastapi_response, user_api_key_dict, headers) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 64fb2791913..37b2969846d 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -213,6 +213,7 @@ if TYPE_CHECKING: from litellm.integrations.opentelemetry import OpenTelemetry from litellm.proxy.health_check_utils.shared_health_check_manager import SharedHealthCheckManager + from litellm.types.proxy.model_listing import ModelInfoResponse Span = _Span | Any else: @@ -472,7 +473,7 @@ from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import ( AuthCacheInvalidationSubscriber, ) from litellm.proxy.common_utils.callback_utils import initialize_callbacks_on_proxy -from litellm.proxy.common_utils.codex_model_catalog import codex_model_list_body +from litellm.proxy.common_utils.codex_model_catalog import CodexCatalogRow, codex_catalog_rows, codex_model_list_body from litellm.proxy.common_utils.config_includes import resolve_include_file_path, resolve_includes from litellm.proxy.common_utils.config_sync_pubsub import ConfigSyncSubscriber from litellm.proxy.common_utils.credential_hydration import decrypted_or_stored @@ -507,6 +508,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401, RUF10 from litellm.proxy.common_utils.load_config_utils import get_config_from_bucket from litellm.proxy.common_utils.model_deprecation import collect_model_deprecations from litellm.proxy.common_utils.model_listing_utils import ( + CallerAliases, ClaudeCodeRoutingNames, DeployedModelGroups, TeamModelNameTranslator, @@ -12263,6 +12265,112 @@ async def _deployment_hidden_by_listing_callbacks(deployment: Deployment, user_a return listed_name in await _names_hidden_by_listing_callbacks(user_api_key_dict, (listed_name,)) +async def _caller_aliases(user_api_key_dict: UserAPIKeyAuth, team_id: str | None) -> CallerAliases: + caller_router_settings: Final = await proxy_config.get_hierarchical_router_settings( + user_api_key_dict, prisma_client, proxy_logging_obj + ) + return caller_alias_maps( + user_api_key_dict.aliases, + user_api_key_dict.team_model_aliases, + user_api_key_dict.team_id, + team_id, + caller_router_settings.get("model_group_alias") if isinstance(caller_router_settings, Mapping) else None, + ) + + +async def _listing_hidden_names(healthy_only: bool | None) -> frozenset[str]: + blocked_names: Final = llm_router.get_fully_blocked_model_names() if llm_router is not None else set[str]() + unhealthy_names: Final = await get_hidden_unhealthy_model_names( + healthy_only=healthy_only, + general_settings=general_settings_view(), + llm_router=llm_router, + ) + return frozenset(blocked_names | unhealthy_names) + + +@dataclass(frozen=True, slots=True) +class _KeyModelListing: + rows: "tuple[ModelInfoResponse, ...]" + entries: tuple[tuple[str, str], ...] + labels: tuple[tuple[str, str], ...] + + +def _listed_model_info( + response_id: str, lookup_id: str, include_metadata: bool, fallback_type: str | None +) -> "ModelInfoResponse": + from litellm.proxy.utils import create_model_info_response + + model_info: Final = create_model_info_response( + model_id=lookup_id, + provider="openai", + include_metadata=include_metadata, + fallback_type=fallback_type, + llm_router=llm_router, + ) + model_info["id"] = response_id + return model_info + + +async def _key_model_listing( + user_api_key_dict: UserAPIKeyAuth, + caller_aliases: CallerAliases, + hidden_names: frozenset[str], + *, + team_id: str | None = None, + include_model_access_groups: bool = False, + only_model_access_groups: bool = False, + include_wildcard_routes: bool = False, + include_metadata: bool = False, + fallback_type: str | None = None, +) -> _KeyModelListing: + from litellm.proxy.utils import get_available_models_for_user + + available: Final = await get_available_models_for_user( + user_api_key_dict=user_api_key_dict, + llm_router=llm_router, + general_settings=general_settings, + user_model=user_model, + prisma_client=prisma_client, + proxy_logging_obj=proxy_logging_obj, + team_id=team_id, + include_model_access_groups=include_model_access_groups, + only_model_access_groups=only_model_access_groups, + return_wildcard_routes=include_wildcard_routes, + user_api_key_cache=user_api_key_cache, + ) + excluded: Final = hidden_names | undiscoverable_model_names( + available, llm_router, user_api_key_dict, team_id or user_api_key_dict.team_id + ) + listed_entries: Final = await _entries_kept_by_listing_callbacks( + TeamModelNameTranslator.listing_entries( + [name for name in available if name not in excluded], llm_router, general_settings_view() + ), + user_api_key_dict, + ) + entries: Final = alias_listing_entries( + listed_entries, + caller_aliases, + _describable_model_groups(listed_entries, hidden_names, user_api_key_dict, team_id), + ) + return _KeyModelListing( + rows=tuple( + _listed_model_info(response_id, lookup_id, include_metadata, fallback_type) + for response_id, lookup_id in entries + ), + entries=entries, + labels=label_entries(listed_entries, entries), + ) + + +async def codex_catalog_rows_for_key(user_api_key_dict: UserAPIKeyAuth) -> tuple[CodexCatalogRow, ...]: + listing: Final = await _key_model_listing( + user_api_key_dict, + await _caller_aliases(user_api_key_dict, None), + await _listing_hidden_names(None), + ) + return codex_catalog_rows(listing.rows, listing.entries, llm_router, user_api_key_dict.team_id, listing.labels) + + @router.get("/v1/models", dependencies=[Depends(user_api_key_auth)], tags=["model management"]) @router.get( "/models", dependencies=[Depends(user_api_key_auth)], tags=["model management"] @@ -12327,10 +12435,7 @@ async def model_list( from litellm.proxy.management_endpoints.common_utils import ( user_has_admin_privileges, ) - from litellm.proxy.utils import ( - create_model_info_response, - get_available_models_for_user, - ) + from litellm.proxy.utils import create_model_info_response from litellm.types.proxy.model_listing import ModelInfoResponse http_request: Final = cast(Request | None, request) # cast-ok: in-process callers pass no request @@ -12338,16 +12443,7 @@ async def model_list( http_request is not None and http_request.headers.get("anthropic-version") is not None ) client_headers: Final[Mapping[str, str]] = http_request.headers if http_request is not None else _EMPTY_HEADERS - caller_router_settings: Final = await proxy_config.get_hierarchical_router_settings( - user_api_key_dict, prisma_client, proxy_logging_obj - ) - caller_aliases: Final = caller_alias_maps( - user_api_key_dict.aliases, - user_api_key_dict.team_model_aliases, - user_api_key_dict.team_id, - team_id, - caller_router_settings.get("model_group_alias") if isinstance(caller_router_settings, Mapping) else None, - ) + caller_aliases: Final = await _caller_aliases(user_api_key_dict, team_id) routing_names: Final = ClaudeCodeRoutingNames( llm_router, team_id or user_api_key_dict.team_id, @@ -12373,18 +12469,7 @@ async def model_list( proxy_logging_obj=proxy_logging_obj, ) - # Compute once — used in both branches below to hide paused models from the listing. - blocked_names: Final = llm_router.get_fully_blocked_model_names() if llm_router is not None else set[str]() - - # Opt-in: also hide models whose deployments are all unhealthy per background - # health checks. Empty when health state is unavailable or stale (fail open). - unhealthy_names: Final = await get_hidden_unhealthy_model_names( - healthy_only=healthy_only, - general_settings=settings, - llm_router=llm_router, - ) - - hidden_names: Final = blocked_names | unhealthy_names + hidden_names: Final = await _listing_hidden_names(healthy_only) # If scope=expand and user has admin privileges, return all proxy models if should_expand_scope: @@ -12465,53 +12550,20 @@ async def model_list( object="list", ) - # Otherwise, use the normal behavior (current implementation) - # Get available models for the user - all_models = await get_available_models_for_user( - user_api_key_dict=user_api_key_dict, - llm_router=llm_router, - general_settings=general_settings, - user_model=user_model, - prisma_client=prisma_client, - proxy_logging_obj=proxy_logging_obj, + key_listing: Final = await _key_model_listing( + user_api_key_dict, + caller_aliases, + hidden_names, team_id=team_id, include_model_access_groups=include_model_access_groups or False, only_model_access_groups=only_model_access_groups or False, - return_wildcard_routes=include_wildcard_routes, - user_api_key_cache=user_api_key_cache, + include_wildcard_routes=include_wildcard_routes, + include_metadata=include_metadata or False, + fallback_type=fallback_type, ) - - undiscoverable_names: Final = undiscoverable_model_names( - all_models, llm_router, user_api_key_dict, team_id or user_api_key_dict.team_id - ) - if hidden_names or undiscoverable_names: - all_models = [m for m in all_models if m not in hidden_names and m not in undiscoverable_names] - - # Surface the public team name by default; legacy internal keys via flag. - # The internal routing key drives the metadata/fallback lookup, while the - # public name is what the client sees as the model id. - model_data = [] - listed_entries: Final = await _entries_kept_by_listing_callbacks( - TeamModelNameTranslator.listing_entries(all_models, llm_router, settings), user_api_key_dict - ) - entries: Final = alias_listing_entries( - listed_entries, - caller_aliases, - _describable_model_groups(listed_entries, hidden_names, user_api_key_dict, team_id), - ) - labels: Final = label_entries(listed_entries, entries) - for response_id, lookup_id in entries: - model_info = create_model_info_response( - model_id=lookup_id, - provider="openai", - include_metadata=include_metadata or False, - fallback_type=fallback_type, - llm_router=llm_router, - ) - model_info["id"] = response_id - model_data.append(model_info) - - listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above + listing: Final = key_listing.rows + entries: Final = key_listing.entries + labels: Final = key_listing.labels if client_version is not None: return Response( content=codex_model_list_body(listing, entries, llm_router, team_id or user_api_key_dict.team_id, labels), @@ -12530,7 +12582,7 @@ async def model_list( ) return dict( - data=model_data, + data=list(listing), object="list", ) diff --git a/litellm/responses/litellm_completion_transformation/reasoning_items.py b/litellm/responses/litellm_completion_transformation/reasoning_items.py index ab982222902..dda313bfc1b 100644 --- a/litellm/responses/litellm_completion_transformation/reasoning_items.py +++ b/litellm/responses/litellm_completion_transformation/reasoning_items.py @@ -6,6 +6,7 @@ from typing import Final from pydantic import BaseModel, TypeAdapter, ValidationError REASONING_ITEM_ID_PREFIX: Final = "rs_" +WRAPPED_ENCRYPTED_CONTENT_PREFIX: Final = "litellm_enc:" _JSON_LIST: Final = TypeAdapter(list[object]) _JSON_OBJECT: Final = TypeAdapter(dict[str, object]) @@ -58,16 +59,35 @@ def is_minted_reasoning_item_id(item_id: object) -> bool: return parsed.version == 4 and str(parsed) == suffix -def is_litellm_minted_reasoning_item(item: object) -> bool: +def _reasoning_fields(item: object) -> Mapping[str, object] | None: try: fields: Final = _JSON_OBJECT.validate_python( item.model_dump(exclude_none=True) if isinstance(item, BaseModel) else item ) except ValidationError: - return False - if fields.get("type") != "reasoning": - return False + return None + return fields if fields.get("type") == "reasoning" else None + + +def _is_minted(fields: Mapping[str, object]) -> bool: return ( is_minted_reasoning_item_id(fields.get("id")) or decode_thinking_blocks(fields.get("encrypted_content")) is not None ) + + +def is_litellm_minted_reasoning_item(item: object) -> bool: + fields: Final = _reasoning_fields(item) + return fields is not None and _is_minted(fields) + + +def is_other_provider_reasoning_item(item: object) -> bool: + fields: Final = _reasoning_fields(item) + if fields is None: + return False + encrypted_content: Final = fields.get("encrypted_content") + return ( + _is_minted(fields) + or bool(fields.get("content")) + or (isinstance(encrypted_content, str) and encrypted_content.startswith(WRAPPED_ENCRYPTED_CONTENT_PREFIX)) + ) diff --git a/tests/unit/proxy/common_utils/test_codex_model_catalog.py b/tests/unit/proxy/common_utils/test_codex_model_catalog.py index c9d4127c6d4..318a5595f30 100644 --- a/tests/unit/proxy/common_utils/test_codex_model_catalog.py +++ b/tests/unit/proxy/common_utils/test_codex_model_catalog.py @@ -15,6 +15,7 @@ from litellm.proxy.common_utils.codex_model_catalog import ( codex_model_list_body, codex_models_response_json, configured_service_tiers, + merged_codex_catalog, ) _PROMPT = "base prompt" @@ -360,6 +361,26 @@ def test_an_entry_too_large_for_the_bytes_left_is_passed_over_and_the_smaller_on assert len(body.json.encode()) <= limit +def test_merged_catalog_keeps_upstream_whole_and_adds_only_the_entries_that_fit_the_byte_limit(): + upstream = {"models": [{"slug": "upstream-model", "priority": 4, "base_instructions": "u" * 500}]} + upstream_only = len(json.dumps(upstream, separators=(",", ":"))) + rows = tuple(_row(f"model-{index}") for index in range(4)) + + def merged(*merged_rows, byte_limit): + return merged_codex_catalog( + json.dumps(upstream).encode(), merged_rows, stock=_STOCK, instructions=_PROMPT, byte_limit=byte_limit + ) + + unlimited = json.loads(merged(*rows, byte_limit=upstream_only * 100))["models"] + limit = len(merged(*rows[:2], byte_limit=upstream_only * 100).encode()) + + body = merged(*rows, byte_limit=limit) + + assert len(body.encode()) <= limit + assert json.loads(body)["models"] == unlimited[:3] + assert merged(*rows, byte_limit=upstream_only) is None + + def test_unlimited_body_keeps_every_entry(): rows = tuple(_row(f"model-{index}") for index in range(3)) diff --git a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py index 354c65776dc..f32f694bb8b 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py @@ -1,6 +1,7 @@ import asyncio +import base64 import json -from collections.abc import Iterator +from collections.abc import Callable, Iterator from datetime import datetime from typing import Final @@ -10,8 +11,18 @@ import respx from fastapi.testclient import TestClient import litellm +from litellm import Router from litellm.integrations.custom_logger import CustomLogger +from litellm.llms.chatgpt.common_utils import OPENAI_AUTH_CLAIM_KEY from litellm.proxy import proxy_server +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.common_utils.codex_model_catalog import bundled_codex_models +from litellm.proxy.utils import hash_token +from litellm.responses.litellm_completion_transformation.reasoning_items import ( + encode_thinking_blocks, + mint_reasoning_item_id, +) +from litellm.responses.utils import ResponsesAPIRequestUtils from tests._master_key import MASTER_KEY _BASE: Final = "https://chatgpt.com/backend-api/codex" @@ -183,3 +194,304 @@ async def test_usage_logged_without_credentials( serialized: Final = json.dumps(logged, default=str) for secret in (_TOKEN, MASTER_KEY, "test-hidden-token", "test-hidden-cookie"): assert secret not in serialized + + +_PROVIDER_BASE: Final = "https://open-weights.test/v1" +_CHATGPT_JWT: Final = ".".join( + base64.urlsafe_b64encode(json.dumps(part).encode()).decode().rstrip("=") + for part in ({"alg": "none"}, {OPENAI_AUTH_CLAIM_KEY: {"chatgpt_account_id": "account-a"}}) +) + "." +_PROVIDER_KEY: Final = "test-provider-key" +_BUNDLED_SLUG: Final = next(iter(bundled_codex_models())) +_PRICED_CHATGPT_SLUG: Final = "priced-subscription-model" +_UPSTREAM_CATALOG: Final = { + "models": [ + {"slug": "subscription-model", "priority": 0, "supported_reasoning_levels": ["high"]}, + {"slug": "shared-name", "priority": 7, "visibility": "list"}, + {"slug": "subscription-hidden", "priority": 3, "visibility": "hide"}, + ] +} + + +def _open_weights(model_name: str, mode: str | None = None) -> dict[str, object]: + return { + "model_name": model_name, + "litellm_params": {"model": f"fireworks_ai/{model_name}", "api_base": _PROVIDER_BASE, "api_key": _PROVIDER_KEY}, + "model_info": {"mode": mode} if mode else {}, + } + + +@pytest.fixture +def mixed_router(monkeypatch: pytest.MonkeyPatch) -> Router: + router: Final = Router( + model_list=[ + { + "model_name": "gpt-*", + "litellm_params": {"model": "chatgpt/gpt-*"}, + "model_info": {"access_groups": ["chatgpt-native"]}, + }, + { + "model_name": "codex-*", + "litellm_params": {"model": "chatgpt/codex-*"}, + "model_info": {"access_groups": ["chatgpt-native"]}, + }, + {"model_name": "chatgpt-subscription", "litellm_params": {"model": "chatgpt/subscription-model"}}, + _open_weights("open-weights"), + _open_weights("shared-name"), + _open_weights(_BUNDLED_SLUG), + _open_weights(_PRICED_CHATGPT_SLUG), + _open_weights("open-embedder", mode="embedding"), + _open_weights("*"), + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + unpriced_bundled: Final = { + name: price for name, price in litellm.model_cost.items() if name != f"chatgpt/{_BUNDLED_SLUG}" + } + monkeypatch.setattr( + litellm, + "model_cost", + {**unpriced_bundled, f"chatgpt/{_PRICED_CHATGPT_SLUG}": {"litellm_provider": "chatgpt", "mode": "responses"}}, + ) + return router + + +def test_native_catalog_lists_the_keys_other_chat_models_after_chatgpts( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router +) -> None: + respx_mock.get(f"{_BASE}/models").respond(json=_UPSTREAM_CATALOG, headers={"etag": '"upstream-v1"'}) + response: Final = client.get("/backend-api/codex/models?client_version=1.0.0", headers=_HEADERS) + assert response.status_code == 200 + models: Final = response.json()["models"] + assert models[: len(_UPSTREAM_CATALOG["models"])] == _UPSTREAM_CATALOG["models"] + added: Final = models[len(_UPSTREAM_CATALOG["models"]) :] + assert [model["slug"] for model in added] == ["open-weights"] + assert added[0]["visibility"] == "list" + assert added[0]["priority"] > max(model["priority"] for model in _UPSTREAM_CATALOG["models"]) + assert "etag" not in response.headers + + +@pytest.mark.parametrize( + "upstream", + [ + pytest.param(httpx.Response(503, json={"detail": "busy"}, headers={"etag": '"e"'}), id="non-200"), + pytest.param(httpx.Response(200, content=b"login", headers={"etag": '"e"'}), id="not json"), + pytest.param(httpx.Response(200, json={"models": "none"}, headers={"etag": '"e"'}), id="no model list"), + ], +) +def test_native_catalog_relays_an_unmergeable_upstream_answer_unchanged( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router, upstream: httpx.Response +) -> None: + respx_mock.get(f"{_BASE}/models").mock(return_value=upstream) + response: Final = client.get("/backend-api/codex/models?client_version=1.0.0", headers=_HEADERS) + assert response.status_code == upstream.status_code + assert response.content == upstream.content + assert response.headers["etag"] == '"e"' + + +def _provider_stream(text: str) -> bytes: + completed: Final = { + "id": "resp_open_weights", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "open-weights", + "output": [ + { + "type": "message", + "id": "msg_open_weights", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": text, "annotations": []}], + } + ], + "usage": {"input_tokens": 3, "output_tokens": 1, "total_tokens": 4}, + } + events: Final = ( + { + "type": "response.created", + "sequence_number": 0, + "response": {**completed, "status": "in_progress", "output": []}, + }, + { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": "msg_open_weights", + "output_index": 0, + "content_index": 0, + "delta": text, + }, + {"type": "response.completed", "sequence_number": 2, "response": completed}, + ) + return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events).encode() + + +def _event_types(stream: bytes) -> list[str]: + payloads: Final = [line.removeprefix(b"data: ") for line in stream.splitlines() if line.startswith(b"data: {")] + return [json.loads(payload)["type"] for payload in payloads] + + +class PreCallRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.requests: Final[list[tuple[str | None, str]]] = [] + + async def async_pre_call_hook( + self, user_api_key_dict: UserAPIKeyAuth, cache: object, data: dict[str, object], call_type: str + ) -> dict[str, object]: + self.requests.append((user_api_key_dict.request_route, repr(data))) + return data + + +def test_litellm_model_on_native_route_streams_like_v1_without_chatgpt_credentials( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + recorder: Final = PreCallRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + chatgpt: Final = respx_mock.post(f"{_BASE}/responses").respond(json={}) + provider: Final = respx_mock.post(f"{_PROVIDER_BASE}/responses").mock( + side_effect=lambda _: httpx.Response( + 200, content=_provider_stream("Hi"), headers={"content-type": "text/event-stream"} + ) + ) + body: Final = {"model": "open-weights", "input": "Say hi", "stream": True, "store": False} + native: Final = client.post( + "/backend-api/codex/responses", + json=body, + headers={**_HEADERS, "authorization": f"Bearer {_CHATGPT_JWT}", "chatgpt-account-id": "account-a"}, + ) + v1: Final = client.post("/v1/responses", json=body, headers={"authorization": f"Bearer {MASTER_KEY}"}) + assert native.status_code == v1.status_code == 200 + assert native.headers["content-type"].startswith("text/event-stream") + assert _event_types(native.content) == _event_types(v1.content) + assert "response.completed" in _event_types(native.content) + assert not chatgpt.called + provider_headers: Final = provider.calls[0].request.headers + assert provider_headers["authorization"] == f"Bearer {_PROVIDER_KEY}" + assert "chatgpt-account-id" not in provider_headers + assert all(_CHATGPT_JWT not in value for value in provider_headers.values()) + (native_route, native_call), (v1_route, _) = recorder.requests + assert native_route == v1_route == "/v1/responses" + assert _CHATGPT_JWT not in native_call + assert "account-a" not in native_call + + +@pytest.mark.parametrize( + "model", ["gpt-unlisted", "chatgpt-subscription", _BUNDLED_SLUG, _PRICED_CHATGPT_SLUG, "newly-released"] +) +def test_chatgpt_model_keeps_passthrough_without_other_providers_reasoning( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router, model: str +) -> None: + provider: Final = respx_mock.post(f"{_PROVIDER_BASE}/responses").respond(json={}) + chatgpt: Final = respx_mock.post(f"{_BASE}/responses").respond( + content=b"event: response.completed\ndata: {}\n\n", + headers={"content-type": "text/event-stream", "x-models-etag": '"upstream-v2"'}, + ) + user_turn: Final = {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Hello"}]} + chatgpt_reasoning: Final = { + "type": "reasoning", + "id": "rs_upstream", + "summary": [], + "encrypted_content": "opaque-upstream-blob", + } + assistant_turn: Final = {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "Hi"}]} + other_providers_reasoning: Final = [ + {"type": "reasoning", "id": mint_reasoning_item_id(), "summary": []}, + { + "type": "reasoning", + "summary": [], + "encrypted_content": encode_thinking_blocks([{"type": "thinking", "thinking": "t", "signature": "s"}]), + }, + {"type": "reasoning", "summary": [], "content": [{"type": "reasoning_text", "text": "t"}]}, + { + "type": "reasoning", + "summary": [], + "encrypted_content": ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id("blob", "deployment"), + }, + ] + body: Final = { + "model": model, + "input": [user_turn, *other_providers_reasoning, chatgpt_reasoning, assistant_turn], + "stream": True, + } + response: Final = client.post("/backend-api/codex/responses", json=body, headers=_HEADERS) + assert response.status_code == 200 + assert not provider.called + assert json.loads(chatgpt.calls.last.request.content) == { + **body, + "input": [user_turn, chatgpt_reasoning, assistant_turn], + } + assert "x-models-etag" not in response.headers + + +@pytest.fixture +def restricted_key(monkeypatch: pytest.MonkeyPatch) -> Callable[[list[str]], str]: + monkeypatch.setattr(proxy_server, "prisma_client", "test-database") + + def mint(models: list[str]) -> str: + key: Final = f"sk-test-{'-'.join(models)}" + proxy_server.user_api_key_cache.set_cache( + key=hash_token(key), value=UserAPIKeyAuth(token=hash_token(key), models=models) + ) + return key + + return mint + + +@pytest.mark.parametrize( + ("models", "added"), + [ + pytest.param(["chatgpt-native", "open-weights"], ["open-weights"], id="key with a LiteLLM model"), + pytest.param(["chatgpt-native"], [], id="key with ChatGPT models only"), + ], +) +def test_native_catalog_adds_only_the_litellm_models_the_key_can_call( + client: TestClient, + respx_mock: respx.MockRouter, + mixed_router: Router, + restricted_key: Callable[[list[str]], str], + models: list[str], + added: list[str], +) -> None: + respx_mock.get(f"{_BASE}/models").respond(json=_UPSTREAM_CATALOG, headers={"etag": '"upstream-v1"'}) + response: Final = client.get( + "/backend-api/codex/models?client_version=1.0.0", + headers={**_HEADERS, "x-litellm-api-key": restricted_key(models)}, + ) + assert response.status_code == 200 + assert [model["slug"] for model in response.json()["models"]] == [ + *(model["slug"] for model in _UPSTREAM_CATALOG["models"]), + *added, + ] + assert ("etag" in response.headers) is not added + + +@pytest.mark.parametrize( + ("models", "model", "allowed"), + [ + pytest.param(["chatgpt-native", "open-weights"], "gpt-unlisted", True, id="chatgpt access group"), + pytest.param(["chatgpt-subscription", "open-weights"], "gpt-unlisted", False, id="single subscription model"), + pytest.param(["chatgpt-native"], "open-weights", False, id="litellm model outside the key"), + ], +) +def test_native_route_serves_only_the_models_the_key_allows( + client: TestClient, + respx_mock: respx.MockRouter, + mixed_router: Router, + restricted_key: Callable[[list[str]], str], + models: list[str], + model: str, + allowed: bool, +) -> None: + provider: Final = respx_mock.post(f"{_PROVIDER_BASE}/responses").respond(json={}) + chatgpt: Final = respx_mock.post(f"{_BASE}/responses").respond( + content=b"event: response.completed\ndata: {}\n\n", headers={"content-type": "text/event-stream"} + ) + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": model, "input": "Hello", "stream": True}, + headers={**_HEADERS, "x-litellm-api-key": restricted_key(models)}, + ) + assert (response.status_code == 200) is allowed + assert chatgpt.called is allowed + assert not provider.called diff --git a/tests/unit/responses/litellm_completion_transformation/test_reasoning_items.py b/tests/unit/responses/litellm_completion_transformation/test_reasoning_items.py index 093d1744418..85ec4be2636 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_reasoning_items.py +++ b/tests/unit/responses/litellm_completion_transformation/test_reasoning_items.py @@ -1,12 +1,16 @@ import json +import pytest + from litellm.responses.litellm_completion_transformation.reasoning_items import ( decode_thinking_blocks, encode_thinking_blocks, is_litellm_minted_reasoning_item, is_minted_reasoning_item_id, + is_other_provider_reasoning_item, mint_reasoning_item_id, ) +from litellm.responses.utils import ResponsesAPIRequestUtils A_PROVIDER_OWNED_REASONING_ITEM_ID = "rs_08d3a89dbb92277a006abf04f4266087d0b4eedacd7848f306" A_PROVIDER_OWNED_ENCRYPTED_BLOB = "gAAAAABo-opaque-provider-blob" @@ -60,3 +64,58 @@ def test_a_reasoning_item_is_litellm_minted_by_its_id_or_by_its_encoded_thinking ) assert not is_litellm_minted_reasoning_item({"type": "message", "id": mint_reasoning_item_id(), "role": "assistant"}) assert not is_litellm_minted_reasoning_item("a bare string input") + + +SUMMARY = [{"type": "summary_text", "text": "Paris first."}] +PROVIDER_OWNED_ITEM = { + "type": "reasoning", + "id": A_PROVIDER_OWNED_REASONING_ITEM_ID, + "summary": SUMMARY, + "encrypted_content": A_PROVIDER_OWNED_ENCRYPTED_BLOB, +} + + +@pytest.mark.parametrize( + "item", + [ + pytest.param({"type": "reasoning", "id": mint_reasoning_item_id(), "summary": SUMMARY}, id="minted id"), + pytest.param( + {"type": "reasoning", "summary": SUMMARY, "encrypted_content": encode_thinking_blocks([SIGNED_BLOCK])}, + id="encoded thinking blocks", + ), + pytest.param( + {"type": "reasoning", "summary": [], "content": [{"type": "reasoning_text", "text": "Paris first."}]}, + id="plain-text reasoning content", + ), + pytest.param( + { + **PROVIDER_OWNED_ITEM, + "encrypted_content": ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id( + A_PROVIDER_OWNED_ENCRYPTED_BLOB, "another-deployment" + ), + }, + id="blob wrapped with a litellm deployment id", + ), + ], +) +def test_reasoning_items_another_provider_produced_are_recognized(item): + assert is_other_provider_reasoning_item(item) + + +@pytest.mark.parametrize( + "item", + [ + pytest.param(PROVIDER_OWNED_ITEM, id="provider-owned blob"), + pytest.param({"type": "reasoning", "summary": SUMMARY}, id="summary only, no id"), + pytest.param( + {"type": "reasoning", "summary": SUMMARY, "content": None, "encrypted_content": None}, id="nulled replay" + ), + pytest.param({"type": "reasoning", "summary": SUMMARY, "content": []}, id="empty content"), + pytest.param( + {"type": "message", "id": mint_reasoning_item_id(), "role": "user", "content": "hi"}, id="not reasoning" + ), + pytest.param("a stray string", id="not an object"), + ], +) +def test_reasoning_items_the_provider_can_replay_are_kept(item): + assert not is_other_provider_reasoning_item(item) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 45f9630983f..a5263761157 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -1801,13 +1801,15 @@ export interface paths { }; /** * Codex Passthrough Route - * @description Forward a client's ChatGPT subscription with separate LiteLLM authentication. + * @description Forward a client's ChatGPT subscription with separate LiteLLM authentication, listing and serving the + * key's other chat models through LiteLLM. */ get: operations["codex_passthrough_route_backend_api_codex__endpoint__get"]; put?: never; /** * Codex Passthrough Route - * @description Forward a client's ChatGPT subscription with separate LiteLLM authentication. + * @description Forward a client's ChatGPT subscription with separate LiteLLM authentication, listing and serving the + * key's other chat models through LiteLLM. */ post: operations["codex_passthrough_route_backend_api_codex__endpoint__post"]; delete?: never; @@ -53860,7 +53862,7 @@ export interface operations { }; }; responses: { - /** @description Unmodified Codex response or server-sent events */ + /** @description Codex response, the model catalog with the key's LiteLLM models, or server-sent events */ 200: { headers: { [name: string]: unknown; From c0bff17e92537b94243a8e7312f5b200fba9898b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 18:43:46 -0700 Subject: [PATCH 02/20] fix(proxy): keep Codex's picker from turning reasoning off for LiteLLM models Codex fills a missing default_reasoning_level with effort "none" when a model is picked in /model, while `codex -m ` sends no effort at all. A thinking-only provider model then answers 400. On the native /backend-api/codex/responses route, drop effort "none" for a LiteLLM model whose catalog entry names no default, so the provider applies its own default exactly as `codex -m` does --- .../proxy/common_utils/codex_model_catalog.py | 18 ++++++-- .../codex_passthrough_endpoints.py | 33 ++++++++++++-- .../common_utils/test_codex_model_catalog.py | 18 ++++++++ .../test_codex_passthrough_endpoints.py | 45 ++++++++++++++++++- 4 files changed, 105 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/common_utils/codex_model_catalog.py b/litellm/proxy/common_utils/codex_model_catalog.py index f8d578fb529..473c0ae9d1d 100644 --- a/litellm/proxy/common_utils/codex_model_catalog.py +++ b/litellm/proxy/common_utils/codex_model_catalog.py @@ -397,6 +397,10 @@ def _compact_json(value: object) -> str: return json.dumps(value, separators=(",", ":")) +def _entry_json(entry: CodexCatalogEntry) -> Mapping[str, object]: + return _JSON_OBJECT.validate_python(entry.model_dump(mode="json", exclude_unset=isinstance(entry, CodexStockModel))) + + def merged_codex_catalog( upstream_body: bytes, rows: Sequence[CodexCatalogRow], @@ -414,10 +418,7 @@ def merged_codex_catalog( added: Final = tuple(row for row in rows if _codex_can_drive(row) and row.id not in slugs) first_priority: Final = max((_priority(model) for model in upstream), default=-1) + 1 entries: Final = _entries(added, stock, instructions, first_priority) - added_models: Final = tuple( - _JSON_OBJECT.validate_python(entry.model_dump(mode="json", exclude_unset=isinstance(entry, CodexStockModel))) - for entry in entries - ) + added_models: Final = tuple(_entry_json(entry) for entry in entries) serialized: Final = tuple(_compact_json(model) for model in added_models) budget: Final = byte_limit - len(_compact_json({**top, "models": upstream})) kept: Final = tuple(sorted(_kept_positions(entries, serialized, budget))) @@ -478,3 +479,12 @@ def codex_model_list_body( ", ".join(body.left_out), ) return body.json + + +def codex_entry_names_default_reasoning_level(model_id: str, llm_router: Router, team_id: str | None) -> bool: + """Whether the catalog entry for `model_id` names a `default_reasoning_level`. Codex's model picker + sends reasoning effort `none` for a model whose entry names none, while showing that choice as + "default".""" + listed: Final[ModelInfoResponse] = {"id": model_id, "object": "model", "created": 0, "owned_by": ""} + (entry,) = _entries((_catalog_row(listed, model_id, model_id, llm_router, team_id),), None, "", 0) + return _entry_json(entry).get("default_reasoning_level") is not None diff --git a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py index 1a38271b986..e8c69ee10ab 100644 --- a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py @@ -1,4 +1,5 @@ import hmac +import json from collections.abc import Iterator, Mapping from types import MappingProxyType from typing import TYPE_CHECKING, Annotated, Final, Literal @@ -12,7 +13,11 @@ import litellm from litellm.llms.chatgpt.common_utils import CHATGPT_API_BASE from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.proxy.common_utils.codex_model_catalog import bundled_codex_models, merged_codex_catalog +from litellm.proxy.common_utils.codex_model_catalog import ( + bundled_codex_models, + codex_entry_names_default_reasoning_level, + merged_codex_catalog, +) from litellm.proxy.common_utils.http_parsing_utils import ( read_request_body, # pyright: ignore[reportUnknownVariableType] # returns a bare dict, validated at the call ) @@ -30,6 +35,7 @@ _CHATGPT_CREDENTIAL_HEADERS: Final = frozenset({b"authorization", b"chatgpt-acco _MERGED_CATALOG_DROPPED_HEADERS: Final = frozenset({"etag", "content-length", "content-type"}) _MODELS_ETAG_HEADER: Final = "x-models-etag" _LITELLM_RESPONSES_PATH: Final = "/v1/responses" +_CODEX_NO_DEFAULT_EFFORT: Final = "none" _JSON_OBJECT: Final = TypeAdapter(dict[str, object]) _INPUT_ITEMS: Final = TypeAdapter(tuple[object, ...]) _HEADERS: Final = frozenset( @@ -80,7 +86,7 @@ def _routes_to_litellm(model: object, llm_router: "Router | None", team_id: str return bool(deployments) and all(_serves_by_name(deployment, names) for deployment in deployments) -def _as_litellm_responses_request(request: Request, body: bytes) -> Request: +def _as_litellm_responses_request(request: Request, body: bytes, parsed_body: Mapping[str, object]) -> Request: replayed: Final[Iterator[Message]] = iter(({"type": "http.request", "body": body, "more_body": False},)) async def receive() -> Message: @@ -94,11 +100,30 @@ def _as_litellm_responses_request(request: Request, body: bytes) -> Request: "headers": [ (name, value) for name, value in request.headers.raw if name not in _CHATGPT_CREDENTIAL_HEADERS ], + "parsed_body": (tuple(parsed_body), parsed_body), }, receive, ) +def _with_provider_default_reasoning( + body: Mapping[str, object], llm_router: "Router | None", team_id: str | None +) -> Mapping[str, object]: + model: Final = body.get("model") + try: + reasoning: Final = _JSON_OBJECT.validate_python(body.get("reasoning")) + except ValidationError: + return body + if ( + reasoning.get("effort") != _CODEX_NO_DEFAULT_EFFORT + or not isinstance(model, str) + or llm_router is None + or codex_entry_names_default_reasoning_level(model, llm_router, team_id) + ): + return body + return {**body, "reasoning": {key: value for key, value in reasoning.items() if key != "effort"}} + + def _input_without_other_provider_reasoning(body: Mapping[str, object]) -> tuple[object, ...] | None: try: items: Final = _INPUT_ITEMS.validate_python(body.get("input")) @@ -145,8 +170,10 @@ async def _responses( body: Final = _JSON_OBJECT.validate_python(await read_request_body(request)) if _routes_to_litellm(body.get("model"), llm_router, user_api_key_dict.team_id): + litellm_body: Final = _with_provider_default_reasoning(body, llm_router, user_api_key_dict.team_id) + raw_body: Final = await request.body() if litellm_body is body else json.dumps(litellm_body).encode() return await responses_api( - _as_litellm_responses_request(request, await request.body()), + _as_litellm_responses_request(request, raw_body, litellm_body), fastapi_response, user_api_key_dict.model_copy(update={"request_route": _LITELLM_RESPONSES_PATH}), ) diff --git a/tests/unit/proxy/common_utils/test_codex_model_catalog.py b/tests/unit/proxy/common_utils/test_codex_model_catalog.py index 318a5595f30..70647fbbc4c 100644 --- a/tests/unit/proxy/common_utils/test_codex_model_catalog.py +++ b/tests/unit/proxy/common_utils/test_codex_model_catalog.py @@ -12,6 +12,7 @@ from litellm.proxy.common_utils.codex_model_catalog import ( CodexStockUpgrade, bundled_codex_models, codex_catalog_rows, + codex_entry_names_default_reasoning_level, codex_model_list_body, codex_models_response_json, configured_service_tiers, @@ -554,3 +555,20 @@ def test_catalog_rows_without_a_router_carry_only_the_listing(): listing = ({"id": "plain", "object": "model", "created": 0, "owned_by": "openai", "mode": "chat"},) assert codex_catalog_rows(listing, (("plain", "plain"),), None) == (CodexCatalogRow(id="plain", mode="chat"),) + + +def test_entry_names_a_default_reasoning_level_exactly_when_the_served_entry_carries_one(): + router = litellm.Router( + model_list=[ + {"model_name": "open-weights", "litellm_params": {"model": "fireworks_ai/open-weights"}}, + {"model_name": "coding-model", "litellm_params": {"model": "openai/gpt-5.5"}}, + ] + ) + names = ("open-weights", "coding-model") + listing = tuple({"id": name, "object": "model", "created": 0, "owned_by": "openai", "mode": "chat"} for name in names) + served = json.loads(codex_model_list_body(listing, tuple((name, name) for name in names), router))["models"] + + assert [codex_entry_names_default_reasoning_level(entry["slug"], router, None) for entry in served] == [ + "default_reasoning_level" in entry for entry in served + ] + assert ["default_reasoning_level" in entry for entry in served] == [False, True] diff --git a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py index f32f694bb8b..aeeb4f5bc83 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py @@ -213,10 +213,14 @@ _UPSTREAM_CATALOG: Final = { } -def _open_weights(model_name: str, mode: str | None = None) -> dict[str, object]: +def _open_weights(model_name: str, mode: str | None = None, upstream: str | None = None) -> dict[str, object]: return { "model_name": model_name, - "litellm_params": {"model": f"fireworks_ai/{model_name}", "api_base": _PROVIDER_BASE, "api_key": _PROVIDER_KEY}, + "litellm_params": { + "model": f"fireworks_ai/{upstream or model_name}", + "api_base": _PROVIDER_BASE, + "api_key": _PROVIDER_KEY, + }, "model_info": {"mode": mode} if mode else {}, } @@ -376,6 +380,43 @@ def test_litellm_model_on_native_route_streams_like_v1_without_chatgpt_credentia assert "account-a" not in native_call +@pytest.mark.parametrize( + ("model", "sent", "expected"), + [ + pytest.param("open-weights", "none", None, id="picker fills in none for a model without a default effort"), + pytest.param("open-weights", "high", "high", id="effort set for a model without a default effort"), + pytest.param("coding-model", "none", "none", id="none on a model whose entry names a default effort"), + ], +) +def test_litellm_model_on_native_route_gets_the_reasoning_effort_codex_m_would_send( + client: TestClient, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, model: str, sent: str, expected: str | None +) -> None: + monkeypatch.setattr( + proxy_server, + "llm_router", + Router(model_list=[_open_weights("open-weights"), _open_weights("coding-model", upstream="gpt-5.5")]), + ) + provider: Final = respx_mock.post(f"{_PROVIDER_BASE}/responses").mock( + side_effect=lambda _: httpx.Response( + 200, content=_provider_stream("Hi"), headers={"content-type": "text/event-stream"} + ) + ) + body: Final = {"model": model, "input": "Say hi", "stream": True, "store": False} + expected_reasoning: Final = {"summary": "auto"} if expected is None else {"effort": expected, "summary": "auto"} + native: Final = client.post( + "/backend-api/codex/responses", + json={**body, "reasoning": {"effort": sent, "summary": "auto"}}, + headers={**_HEADERS, "authorization": f"Bearer {_CHATGPT_JWT}"}, + ) + v1: Final = client.post( + "/v1/responses", json={**body, "reasoning": expected_reasoning}, headers={"authorization": f"Bearer {MASTER_KEY}"} + ) + assert native.status_code == v1.status_code == 200 + native_request, v1_request = (json.loads(call.request.content) for call in provider.calls) + assert native_request == v1_request + print("PROBE", native_request.get("reasoning"), native_request.get("reasoning_effort"), sorted(native_request)) + + @pytest.mark.parametrize( "model", ["gpt-unlisted", "chatgpt-subscription", _BUNDLED_SLUG, _PRICED_CHATGPT_SLUG, "newly-released"] ) From a355a7f8e12b4cc3b24774a753357799664709bb Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:05:34 -0700 Subject: [PATCH 03/20] fix(proxy): harden Codex mixed-model routing on the native ChatGPT routes Keep the ChatGPT credentials out of hook raw headers on the inner /v1/responses request, route to LiteLLM and merge LiteLLM catalog rows only for keys that may call /v1/responses, drop unpersisted reasoning ids on the passthrough when store is not true, keep any slug the caller's ChatGPT account lists on the subscription via a shared account catalog cache, and keep X-Models-Etag for keys whose catalog added no LiteLLM rows. The guide covers the new behavior and the /v1 wildcard precedence over a catch-all deployment. --- .../chatgpt_subscription_passthrough.md | 8 +- .../proxy/common_utils/codex_model_catalog.py | 9 + .../codex_passthrough_endpoints.py | 221 ++++++++++++- .../test_codex_passthrough_endpoints.py | 293 +++++++++++++++++- 4 files changed, 499 insertions(+), 32 deletions(-) diff --git a/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md b/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md index f0dc6481bd5..21c944c9f95 100644 --- a/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md +++ b/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md @@ -50,11 +50,13 @@ For a native Codex model catalog and request format, change Codex's `base_url` t ### Mix ChatGPT and LiteLLM models -On the native route, Codex's model picker shows OpenAI's catalog first and then the other chat and responses models the virtual key can use. Pick one of those and Codex's requests go through LiteLLM's `/v1/responses` with the key's usual routing, budgets, guardrails, and logging. The proxy removes the ChatGPT bearer and account header from that request, so the other provider and your callbacks never receive them. Only a model listed under its exact `model_name` on another provider is routed this way. A ChatGPT model, or a name that only a wildcard deployment matches, keeps the passthrough, so a model OpenAI adds later still uses the subscription. When the key can use many models, the picker lists the ones that fit in Codex's catalog size limit. +On the native route, Codex's model picker shows OpenAI's catalog first and then the other chat and responses models the virtual key can use. Pick one of those and Codex's requests go through LiteLLM's `/v1/responses` with the key's usual routing, budgets, guardrails, and logging. The proxy removes the ChatGPT bearer and account header from that request, so the other provider and your callbacks never receive them. Only a model listed under its exact `model_name` on another provider is routed this way. A ChatGPT model, a name OpenAI's catalog lists for the caller's ChatGPT account, or a name that only a wildcard deployment matches keeps the passthrough, so a model OpenAI adds later still uses the subscription even when a deployment of another provider has the same name. When the key can use many models, the picker lists the ones that fit in Codex's catalog size limit. + +Only a key that may call `/v1/responses` gets these models. A key whose `allowed_routes` covers only the native routes sees OpenAI's catalog unchanged, and every request it sends uses the subscription. Codex sends earlier reasoning items back with each turn. When a session switches from another provider's model to a ChatGPT model, the proxy drops the reasoning items that other provider produced before forwarding the request to OpenAI, which would reject them. Messages, tool calls, and ChatGPT's own reasoning are kept. -The merged catalog carries no ETag, so Codex picks up a newly allowed model on its next regular catalog refresh. +When the catalog adds LiteLLM models for a key, it carries no ETag, and that key's responses drop OpenAI's `X-Models-Etag` header, so Codex picks up a newly allowed model on its next regular catalog refresh. A key whose catalog adds no LiteLLM models keeps OpenAI's ETags. ### Restrict native models with an access group @@ -76,6 +78,8 @@ model_list: Then add `chatgpt-native` to the key's models, next to any other model names it should reach. The key can call every ChatGPT model that matches the wildcards, including ones the proxy configuration does not name. These wildcards are also served on `/v1`, where a request without a ChatGPT login falls back to the proxy's own saved login if it has one, so keep that login unset as described above. +On `/v1` the router picks the longest matching pattern, so for every caller these wildcards win over a `*` catch-all deployment, and a request for any `gpt-` or `codex-` name goes to ChatGPT instead of the catch-all's provider. Give a model an exact deployment if `/v1` users should reach it on another provider. + ### Configure Pi Sign into the built-in `openai-codex` provider in Pi. Merge this override into `~/.pi/agent/models.json` and export your LiteLLM virtual key as `LITELLM_API_KEY` before launching Pi: diff --git a/litellm/proxy/common_utils/codex_model_catalog.py b/litellm/proxy/common_utils/codex_model_catalog.py index 473c0ae9d1d..9e6f1de4bce 100644 --- a/litellm/proxy/common_utils/codex_model_catalog.py +++ b/litellm/proxy/common_utils/codex_model_catalog.py @@ -427,6 +427,15 @@ def merged_codex_catalog( return _compact_json({**top, "models": [*upstream, *(added_models[position] for position in kept)]}) +def upstream_codex_slugs(upstream_body: bytes) -> frozenset[str] | None: + try: + upstream: Final = _JSON_OBJECTS.validate_python(_JSON_OBJECT.validate_json(upstream_body).get("models")) + except ValidationError: + return None + slugs: Final = (model.get("slug") for model in upstream) + return frozenset(slug for slug in slugs if isinstance(slug, str)) + + def _catalog_row( row: ModelInfoResponse, lookup_id: str, label_id: str, llm_router: Router | None, team_id: str | None ) -> CodexCatalogRow: diff --git a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py index e8c69ee10ab..ae0d390b337 100644 --- a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py @@ -1,9 +1,12 @@ +import hashlib import hmac import json +import re from collections.abc import Iterator, Mapping from types import MappingProxyType -from typing import TYPE_CHECKING, Annotated, Final, Literal +from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol +import httpx from fastapi import APIRouter, Depends, HTTPException, Request, Response from fastapi.responses import StreamingResponse from pydantic import TypeAdapter, ValidationError @@ -11,12 +14,17 @@ from starlette.types import Message import litellm from litellm.llms.chatgpt.common_utils import CHATGPT_API_BASE +from litellm.llms.custom_httpx.http_handler import ( + get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared client factory has untyped params +) from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.codex_model_catalog import ( bundled_codex_models, codex_entry_names_default_reasoning_level, merged_codex_catalog, + upstream_codex_slugs, ) from litellm.proxy.common_utils.http_parsing_utils import ( read_request_body, # pyright: ignore[reportUnknownVariableType] # returns a bare dict, validated at the call @@ -24,6 +32,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( from litellm.proxy.pass_through_endpoints.pass_through_endpoints import pass_through_request from litellm.proxy.response_api_endpoints.endpoints import responses_api from litellm.responses.litellm_completion_transformation.reasoning_items import is_other_provider_reasoning_item +from litellm.types.llms.custom_http import httpxSpecialProvider from litellm.types.router import Deployment if TYPE_CHECKING: @@ -38,6 +47,14 @@ _LITELLM_RESPONSES_PATH: Final = "/v1/responses" _CODEX_NO_DEFAULT_EFFORT: Final = "none" _JSON_OBJECT: Final = TypeAdapter(dict[str, object]) _INPUT_ITEMS: Final = TypeAdapter(tuple[object, ...]) +_CACHED_SLUGS: Final = TypeAdapter(frozenset[str]) +_HEADER_DERIVED_STATE: Final = frozenset({"_cached_headers"}) +_ACCOUNT_CATALOG_HEADERS: Final = frozenset({"authorization", "chatgpt-account-id", "originator", "user-agent"}) +_CODEX_USER_AGENT_PRODUCT: Final = re.compile(r"[^/]+/(\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.+-]+)?)") +_ACCOUNT_CATALOG_TIMEOUT_SECONDS: Final = 5.0 +_SHARED_CACHE_TTL_SECONDS: Final = 3600 +_ACCOUNT_SLUGS_CACHE_PREFIX: Final = "codex_account_slugs:" +_MERGED_CATALOG_CACHE_PREFIX: Final = "codex_catalog_added_litellm_rows:" _HEADERS: Final = frozenset( { "authorization", @@ -86,6 +103,130 @@ def _routes_to_litellm(model: object, llm_router: "Router | None", team_id: str return bool(deployments) and all(_serves_by_name(deployment, names) for deployment in deployments) +class _SharedCache(Protocol): + async def async_get_cache(self, key: str) -> object: ... + + async def async_set_cache(self, key: str, value: object, *, ttl: int) -> None: ... + + +def codex_shared_cache() -> _SharedCache: + from litellm.proxy.proxy_server import proxy_logging_obj + + return proxy_logging_obj.internal_usage_cache.dual_cache + + +def _may_call_litellm_responses(user_api_key_dict: UserAPIKeyAuth) -> bool: + try: + _ = RouteChecks.should_call_route(route=_LITELLM_RESPONSES_PATH, valid_token=user_api_key_dict) + except HTTPException: + return False + return True + + +def _account_slugs_cache_key(account_id: str) -> str: + return f"{_ACCOUNT_SLUGS_CACHE_PREFIX}{hashlib.sha256(account_id.encode()).hexdigest()}" + + +def _cached_slugs(cached: object) -> frozenset[str] | None: + if cached is None: + return None + try: + return _CACHED_SLUGS.validate_python(cached) + except ValidationError: + return None + + +async def _remember_account_slugs(account_id: str, slugs: frozenset[str], cache: _SharedCache) -> None: + await cache.async_set_cache( + _account_slugs_cache_key(account_id), tuple(sorted(slugs)), ttl=_SHARED_CACHE_TTL_SECONDS + ) + + +def _codex_client_version(user_agent: str) -> str | None: + product: Final = _CODEX_USER_AGENT_PRODUCT.fullmatch(user_agent.partition(" ")[0]) + return product.group(1) if product is not None else None + + +async def _fetched_account_slugs(headers: Mapping[str, str]) -> frozenset[str] | None: + client_version: Final = _codex_client_version(headers.get("user-agent", "")) + if client_version is None: + return None + client: Final = get_async_httpx_client( + llm_provider=httpxSpecialProvider.PassThroughEndpoint, + params={"timeout": _ACCOUNT_CATALOG_TIMEOUT_SECONDS}, + ) + try: + response: Final = await client.client.get( + f"{CHATGPT_API_BASE}/models", + params={"client_version": client_version}, + headers={name: value for name, value in headers.items() if name in _ACCOUNT_CATALOG_HEADERS}, + ) + except httpx.HTTPError: + return None + return upstream_codex_slugs(response.content) if response.status_code == 200 else None + + +async def _account_slugs(headers: Mapping[str, str], cache: _SharedCache) -> frozenset[str] | None: + account_id: Final = headers.get("chatgpt-account-id") + if not account_id: + return None + cached: Final = _cached_slugs(await cache.async_get_cache(_account_slugs_cache_key(account_id))) + if cached is not None: + return cached + fetched: Final = await _fetched_account_slugs(headers) + if fetched is not None: + await _remember_account_slugs(account_id, fetched, cache) + return fetched + + +async def _serves_through_litellm( + model: object, + llm_router: "Router | None", + user_api_key_dict: UserAPIKeyAuth, + headers: Mapping[str, str], + cache: _SharedCache, +) -> bool: + if not _routes_to_litellm(model, llm_router, user_api_key_dict.team_id) or not _may_call_litellm_responses( + user_api_key_dict + ): + return False + account_slugs: Final = await _account_slugs(headers, cache) + return account_slugs is None or model not in account_slugs + + +def _merged_catalog_cache_key(token: str) -> str: + return f"{_MERGED_CATALOG_CACHE_PREFIX}{token}" + + +async def _remember_catalog( + headers: Mapping[str, str], + slugs: frozenset[str], + user_api_key_dict: UserAPIKeyAuth, + added_litellm_rows: bool, + cache: _SharedCache, +) -> None: + account_id: Final = headers.get("chatgpt-account-id") + if account_id: + await _remember_account_slugs(account_id, slugs, cache) + if user_api_key_dict.token is not None: + await cache.async_set_cache( + _merged_catalog_cache_key(user_api_key_dict.token), added_litellm_rows, ttl=_SHARED_CACHE_TTL_SECONDS + ) + + +async def _catalog_may_have_litellm_rows(user_api_key_dict: UserAPIKeyAuth, cache: _SharedCache) -> bool: + if user_api_key_dict.token is None: + return True + return await cache.async_get_cache(_merged_catalog_cache_key(user_api_key_dict.token)) is not False + + +def _scope_state(request: Request) -> Mapping[str, object]: + try: + return _JSON_OBJECT.validate_python(request.scope.get("state")) + except ValidationError: + return MappingProxyType({}) + + def _as_litellm_responses_request(request: Request, body: bytes, parsed_body: Mapping[str, object]) -> Request: replayed: Final[Iterator[Message]] = iter(({"type": "http.request", "body": body, "more_body": False},)) @@ -101,6 +242,9 @@ def _as_litellm_responses_request(request: Request, body: bytes, parsed_body: Ma (name, value) for name, value in request.headers.raw if name not in _CHATGPT_CREDENTIAL_HEADERS ], "parsed_body": (tuple(parsed_body), parsed_body), + "state": { + name: value for name, value in _scope_state(request).items() if name not in _HEADER_DERIVED_STATE + }, }, receive, ) @@ -124,17 +268,53 @@ def _with_provider_default_reasoning( return {**body, "reasoning": {key: value for key, value in reasoning.items() if key != "effort"}} -def _input_without_other_provider_reasoning(body: Mapping[str, object]) -> tuple[object, ...] | None: +def _is_unpersisted_reasoning_reference(item: object) -> bool: + try: + fields: Final = _JSON_OBJECT.validate_python(item) + except ValidationError: + return False + item_id: Final = fields.get("id") + encrypted_content: Final = fields.get("encrypted_content") + return ( + fields.get("type") == "reasoning" + and isinstance(item_id, str) + and bool(item_id) + and not (isinstance(encrypted_content, str) and bool(encrypted_content)) + ) + + +def _rejected_by_chatgpt(item: object, stored: bool) -> bool: + return is_other_provider_reasoning_item(item) or (not stored and _is_unpersisted_reasoning_reference(item)) + + +def _input_chatgpt_accepts(body: Mapping[str, object]) -> tuple[object, ...] | None: try: items: Final = _INPUT_ITEMS.validate_python(body.get("input")) except ValidationError: return None - kept: Final = tuple(item for item in items if not is_other_provider_reasoning_item(item)) + stored: Final = body.get("store") is True + kept: Final = tuple(item for item in items if not _rejected_by_chatgpt(item, stored)) return kept if len(kept) < len(items) else None -async def _catalog(request: Request, user_api_key_dict: UserAPIKeyAuth, headers: Mapping[str, str]) -> Response: - from litellm.proxy.proxy_server import codex_catalog_rows_for_key, llm_router +async def _merged_catalog( + upstream_body: bytes, user_api_key_dict: UserAPIKeyAuth, llm_router: "Router | None" +) -> str | None: + from litellm.proxy.proxy_server import codex_catalog_rows_for_key + + if llm_router is None or not _may_call_litellm_responses(user_api_key_dict): + return None + rows: Final = await codex_catalog_rows_for_key(user_api_key_dict) + return merged_codex_catalog( + upstream_body, + tuple(row for row in rows if _routes_to_litellm(row.id, llm_router, user_api_key_dict.team_id)), + ) + + +async def _catalog( + request: Request, user_api_key_dict: UserAPIKeyAuth, headers: Mapping[str, str], cache: _SharedCache +) -> Response: + from litellm.proxy.proxy_server import llm_router upstream: Final = await pass_through_request( request=request, @@ -144,13 +324,14 @@ async def _catalog(request: Request, user_api_key_dict: UserAPIKeyAuth, headers: custom_llm_provider=_CHATGPT_PROVIDER, upstream_headers_filter=lambda _: headers, ) - if llm_router is None or upstream.status_code != 200 or isinstance(upstream, StreamingResponse): + if upstream.status_code != 200 or isinstance(upstream, StreamingResponse): return upstream - rows: Final = await codex_catalog_rows_for_key(user_api_key_dict) - merged: Final = merged_codex_catalog( - bytes(upstream.body), - tuple(row for row in rows if _routes_to_litellm(row.id, llm_router, user_api_key_dict.team_id)), - ) + upstream_body: Final = bytes(upstream.body) + slugs: Final = upstream_codex_slugs(upstream_body) + if slugs is None: + return upstream + merged: Final = await _merged_catalog(upstream_body, user_api_key_dict, llm_router) + await _remember_catalog(headers, slugs, user_api_key_dict, merged is not None, cache) if merged is None: return upstream return Response( @@ -164,12 +345,16 @@ async def _catalog(request: Request, user_api_key_dict: UserAPIKeyAuth, headers: async def _responses( - request: Request, fastapi_response: Response, user_api_key_dict: UserAPIKeyAuth, headers: Mapping[str, str] + request: Request, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth, + headers: Mapping[str, str], + cache: _SharedCache, ) -> object: from litellm.proxy.proxy_server import llm_router body: Final = _JSON_OBJECT.validate_python(await read_request_body(request)) - if _routes_to_litellm(body.get("model"), llm_router, user_api_key_dict.team_id): + if await _serves_through_litellm(body.get("model"), llm_router, user_api_key_dict, headers, cache): litellm_body: Final = _with_provider_default_reasoning(body, llm_router, user_api_key_dict.team_id) raw_body: Final = await request.body() if litellm_body is body else json.dumps(litellm_body).encode() return await responses_api( @@ -177,7 +362,7 @@ async def _responses( fastapi_response, user_api_key_dict.model_copy(update={"request_route": _LITELLM_RESPONSES_PATH}), ) - kept_input: Final = _input_without_other_provider_reasoning(body) + kept_input: Final = _input_chatgpt_accepts(body) upstream: Final = await pass_through_request( request=request, target=f"{CHATGPT_API_BASE}/responses", @@ -187,7 +372,8 @@ async def _responses( custom_llm_provider=_CHATGPT_PROVIDER, upstream_headers_filter=lambda _: headers, ) - del upstream.headers[_MODELS_ETAG_HEADER] + if await _catalog_may_have_litellm_rows(user_api_key_dict, cache): + del upstream.headers[_MODELS_ETAG_HEADER] return upstream @@ -217,6 +403,7 @@ async def codex_passthrough_route( request: Request, fastapi_response: Response, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + cache: Annotated[_SharedCache, Depends(codex_shared_cache)], ) -> object: """Forward a client's ChatGPT subscription with separate LiteLLM authentication, listing and serving the key's other chat models through LiteLLM.""" @@ -243,5 +430,5 @@ async def codex_passthrough_route( headers: Final = MappingProxyType({name: value for name, value in request.headers.items() if name in _HEADERS}) if endpoint == "models": - return await _catalog(request, user_api_key_dict, headers) - return await _responses(request, fastapi_response, user_api_key_dict, headers) + return await _catalog(request, user_api_key_dict, headers, cache) + return await _responses(request, fastapi_response, user_api_key_dict, headers, cache) diff --git a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py index aeeb4f5bc83..9e781e4524b 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py @@ -12,11 +12,13 @@ from fastapi.testclient import TestClient import litellm from litellm import Router +from litellm.caching.dual_cache import DualCache from litellm.integrations.custom_logger import CustomLogger from litellm.llms.chatgpt.common_utils import OPENAI_AUTH_CLAIM_KEY from litellm.proxy import proxy_server from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.codex_model_catalog import bundled_codex_models +from litellm.proxy.pass_through_endpoints.codex_passthrough_endpoints import codex_shared_cache from litellm.proxy.utils import hash_token from litellm.responses.litellm_completion_transformation.reasoning_items import ( encode_thinking_blocks, @@ -33,6 +35,8 @@ _HEADERS: Final = {"authorization": f"Bearer {_TOKEN}", "x-litellm-api-key": MAS @pytest.fixture def client(monkeypatch: pytest.MonkeyPatch, httpx_transport: None) -> Iterator[TestClient]: monkeypatch.setattr(proxy_server, "master_key", MASTER_KEY) + shared_cache: Final = DualCache() + monkeypatch.setitem(proxy_server.app.dependency_overrides, codex_shared_cache, lambda: shared_cache) yield TestClient(proxy_server.app) @@ -197,10 +201,13 @@ async def test_usage_logged_without_credentials( _PROVIDER_BASE: Final = "https://open-weights.test/v1" -_CHATGPT_JWT: Final = ".".join( - base64.urlsafe_b64encode(json.dumps(part).encode()).decode().rstrip("=") - for part in ({"alg": "none"}, {OPENAI_AUTH_CLAIM_KEY: {"chatgpt_account_id": "account-a"}}) -) + "." +_CHATGPT_JWT: Final = ( + ".".join( + base64.urlsafe_b64encode(json.dumps(part).encode()).decode().rstrip("=") + for part in ({"alg": "none"}, {OPENAI_AUTH_CLAIM_KEY: {"chatgpt_account_id": "account-a"}}) + ) + + "." +) _PROVIDER_KEY: Final = "test-provider-key" _BUNDLED_SLUG: Final = next(iter(bundled_codex_models())) _PRICED_CHATGPT_SLUG: Final = "priced-subscription-model" @@ -339,14 +346,33 @@ class PreCallRecorder(CustomLogger): def __init__(self) -> None: super().__init__() self.requests: Final[list[tuple[str | None, str]]] = [] + self.raw_header_names: Final[list[frozenset[str]]] = [] async def async_pre_call_hook( self, user_api_key_dict: UserAPIKeyAuth, cache: object, data: dict[str, object], call_type: str ) -> dict[str, object]: self.requests.append((user_api_key_dict.request_route, repr(data))) + secret_fields: Final = data.get("secret_fields") + raw_headers: Final = secret_fields.get("raw_headers") if isinstance(secret_fields, dict) else None + self.raw_header_names.append(frozenset(raw_headers) if isinstance(raw_headers, dict) else frozenset()) return data +def _open_weights_provider(respx_mock: respx.MockRouter) -> respx.Route: + return respx_mock.post(f"{_PROVIDER_BASE}/responses").mock( + side_effect=lambda _: httpx.Response( + 200, content=_provider_stream("Hi"), headers={"content-type": "text/event-stream"} + ) + ) + + +def _chatgpt_responses(respx_mock: respx.MockRouter) -> respx.Route: + return respx_mock.post(f"{_BASE}/responses").respond( + content=b"event: response.completed\ndata: {}\n\n", + headers={"content-type": "text/event-stream", "x-models-etag": '"upstream-v2"'}, + ) + + def test_litellm_model_on_native_route_streams_like_v1_without_chatgpt_credentials( client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -380,6 +406,24 @@ def test_litellm_model_on_native_route_streams_like_v1_without_chatgpt_credentia assert "account-a" not in native_call +def test_litellm_model_on_native_route_keeps_chatgpt_credentials_out_of_hook_raw_headers( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + recorder: Final = PreCallRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + provider: Final = _open_weights_provider(respx_mock) + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": "open-weights", "input": "Say hi", "stream": True, "store": False}, + headers={**_HEADERS, "authorization": f"Bearer {_CHATGPT_JWT}", "chatgpt-account-id": "account-a"}, + ) + assert response.status_code == 200 + assert provider.called + (raw_header_names,) = recorder.raw_header_names + assert "x-litellm-api-key" in raw_header_names + assert not raw_header_names & {"authorization", "chatgpt-account-id"}, sorted(raw_header_names) + + @pytest.mark.parametrize( ("model", "sent", "expected"), [ @@ -389,7 +433,12 @@ def test_litellm_model_on_native_route_streams_like_v1_without_chatgpt_credentia ], ) def test_litellm_model_on_native_route_gets_the_reasoning_effort_codex_m_would_send( - client: TestClient, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, model: str, sent: str, expected: str | None + client: TestClient, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + model: str, + sent: str, + expected: str | None, ) -> None: monkeypatch.setattr( proxy_server, @@ -409,12 +458,13 @@ def test_litellm_model_on_native_route_gets_the_reasoning_effort_codex_m_would_s headers={**_HEADERS, "authorization": f"Bearer {_CHATGPT_JWT}"}, ) v1: Final = client.post( - "/v1/responses", json={**body, "reasoning": expected_reasoning}, headers={"authorization": f"Bearer {MASTER_KEY}"} + "/v1/responses", + json={**body, "reasoning": expected_reasoning}, + headers={"authorization": f"Bearer {MASTER_KEY}"}, ) assert native.status_code == v1.status_code == 200 native_request, v1_request = (json.loads(call.request.content) for call in provider.calls) assert native_request == v1_request - print("PROBE", native_request.get("reasoning"), native_request.get("reasoning_effort"), sorted(native_request)) @pytest.mark.parametrize( @@ -466,13 +516,14 @@ def test_chatgpt_model_keeps_passthrough_without_other_providers_reasoning( @pytest.fixture -def restricted_key(monkeypatch: pytest.MonkeyPatch) -> Callable[[list[str]], str]: +def restricted_key(monkeypatch: pytest.MonkeyPatch) -> Callable[..., str]: monkeypatch.setattr(proxy_server, "prisma_client", "test-database") - def mint(models: list[str]) -> str: - key: Final = f"sk-test-{'-'.join(models)}" + def mint(models: list[str], allowed_routes: tuple[str, ...] = ()) -> str: + key: Final = f"sk-test-{'-'.join((*models, *allowed_routes))}" proxy_server.user_api_key_cache.set_cache( - key=hash_token(key), value=UserAPIKeyAuth(token=hash_token(key), models=models) + key=hash_token(key), + value=UserAPIKeyAuth(token=hash_token(key), models=models, allowed_routes=list(allowed_routes)), ) return key @@ -490,7 +541,7 @@ def test_native_catalog_adds_only_the_litellm_models_the_key_can_call( client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router, - restricted_key: Callable[[list[str]], str], + restricted_key: Callable[..., str], models: list[str], added: list[str], ) -> None: @@ -519,7 +570,7 @@ def test_native_route_serves_only_the_models_the_key_allows( client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router, - restricted_key: Callable[[list[str]], str], + restricted_key: Callable[..., str], models: list[str], model: str, allowed: bool, @@ -536,3 +587,219 @@ def test_native_route_serves_only_the_models_the_key_allows( assert (response.status_code == 200) is allowed assert chatgpt.called is allowed assert not provider.called + + +_NATIVE_ROUTES: Final = ("/backend-api/codex/models", "/backend-api/codex/responses") + + +@pytest.mark.parametrize( + ("allowed_routes", "added"), + [ + pytest.param(_NATIVE_ROUTES, [], id="key limited to the native routes"), + pytest.param((*_NATIVE_ROUTES, "/v1/responses"), ["open-weights"], id="key that may call /v1/responses"), + ], +) +def test_native_catalog_adds_litellm_models_only_for_a_key_that_may_call_v1_responses( + client: TestClient, + respx_mock: respx.MockRouter, + mixed_router: Router, + restricted_key: Callable[..., str], + allowed_routes: tuple[str, ...], + added: list[str], +) -> None: + respx_mock.get(f"{_BASE}/models").respond(json=_UPSTREAM_CATALOG) + response: Final = client.get( + "/backend-api/codex/models?client_version=1.0.0", + headers={**_HEADERS, "x-litellm-api-key": restricted_key(["chatgpt-native", "open-weights"], allowed_routes)}, + ) + assert response.status_code == 200, response.text + assert [model["slug"] for model in response.json()["models"]] == [ + *(model["slug"] for model in _UPSTREAM_CATALOG["models"]), + *added, + ] + + +@pytest.mark.parametrize( + ("allowed_routes", "through_litellm"), + [ + pytest.param(_NATIVE_ROUTES, False, id="key limited to the native routes"), + pytest.param((*_NATIVE_ROUTES, "/v1/responses"), True, id="key that may call /v1/responses"), + ], +) +def test_native_responses_reach_litellm_models_only_for_a_key_that_may_call_v1_responses( + client: TestClient, + respx_mock: respx.MockRouter, + mixed_router: Router, + restricted_key: Callable[..., str], + allowed_routes: tuple[str, ...], + through_litellm: bool, +) -> None: + provider: Final = _open_weights_provider(respx_mock) + chatgpt: Final = _chatgpt_responses(respx_mock) + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": "open-weights", "input": "Hello", "stream": True}, + headers={**_HEADERS, "x-litellm-api-key": restricted_key(["chatgpt-native", "open-weights"], allowed_routes)}, + ) + assert response.status_code == 200, response.text + assert provider.called is through_litellm + assert chatgpt.called is not through_litellm + + +_USER_TURN: Final = {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Hello"}]} +_ASSISTANT_TURN: Final = {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "Hi"}]} +_FIREWORKS_REASONING: Final = { + "type": "reasoning", + "id": f"rs_{'0f' * 16}", + "summary": [{"type": "summary_text", "text": "Thinking it over"}], +} + + +@pytest.mark.parametrize( + ("item", "store", "forwarded"), + [ + pytest.param(_FIREWORKS_REASONING, False, False, id="another provider's summary-only item"), + pytest.param({**_FIREWORKS_REASONING, "encrypted_content": None}, None, False, id="null blob, store unset"), + pytest.param( + {"type": "reasoning", "id": "rs_upstream", "summary": [], "encrypted_content": "gAAAAB-test-blob"}, + False, + True, + id="ChatGPT's own item", + ), + pytest.param({**_FIREWORKS_REASONING, "id": None}, False, True, id="summary-only item without an id"), + pytest.param(_FIREWORKS_REASONING, True, True, id="stored conversation"), + ], +) +def test_chatgpt_passthrough_drops_reasoning_ids_chatgpt_never_persisted( + client: TestClient, + respx_mock: respx.MockRouter, + item: dict[str, object], + store: bool | None, + forwarded: bool, +) -> None: + chatgpt: Final = _chatgpt_responses(respx_mock) + body: Final = { + "model": "subscription-model", + "input": [_USER_TURN, item, _ASSISTANT_TURN], + "stream": True, + **({} if store is None else {"store": store}), + } + response: Final = client.post("/backend-api/codex/responses", json=body, headers=_HEADERS) + assert response.status_code == 200 + assert json.loads(chatgpt.calls.last.request.content)["input"] == ( + [_USER_TURN, item, _ASSISTANT_TURN] if forwarded else [_USER_TURN, _ASSISTANT_TURN] + ) + + +_CODEX_USER_AGENT: Final = "codex_cli_rs/0.165.0 (Ubuntu 24.4.0; x86_64) xterm-256color" +_ACCOUNT_HEADERS: Final = { + **_HEADERS, + "authorization": f"Bearer {_CHATGPT_JWT}", + "chatgpt-account-id": "account-a", + "originator": "codex_cli_rs", + "user-agent": _CODEX_USER_AGENT, +} + + +def test_exact_deployment_named_like_an_account_slug_stays_on_the_subscription_after_the_catalog_read( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router +) -> None: + catalog: Final = respx_mock.get(f"{_BASE}/models").respond(json=_UPSTREAM_CATALOG) + provider: Final = _open_weights_provider(respx_mock) + chatgpt: Final = _chatgpt_responses(respx_mock) + assert client.get("/backend-api/codex/models?client_version=0.165.0", headers=_ACCOUNT_HEADERS).status_code == 200 + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": "shared-name", "input": "Hello", "stream": True}, + headers=_ACCOUNT_HEADERS, + ) + assert response.status_code == 200 + assert chatgpt.called + assert not provider.called + assert catalog.call_count == 1 + + +@pytest.mark.parametrize( + ("model", "user_agent", "catalog_answer", "subscription"), + [ + pytest.param("shared-name", _CODEX_USER_AGENT, httpx.Response(200, json=_UPSTREAM_CATALOG), True, id="listed"), + pytest.param("shared-name", _CODEX_USER_AGENT, httpx.Response(503), False, id="catalog unavailable"), + pytest.param( + "shared-name", "codex_cli_rs", httpx.Response(200, json=_UPSTREAM_CATALOG), False, id="no version" + ), + pytest.param( + "open-weights", _CODEX_USER_AGENT, httpx.Response(200, json=_UPSTREAM_CATALOG), False, id="unlisted" + ), + ], +) +def test_native_responses_ask_chatgpt_which_models_the_account_lists_on_a_cache_miss( + client: TestClient, + respx_mock: respx.MockRouter, + mixed_router: Router, + model: str, + user_agent: str, + catalog_answer: httpx.Response, + subscription: bool, +) -> None: + catalog: Final = respx_mock.get(f"{_BASE}/models").mock(return_value=catalog_answer) + provider: Final = _open_weights_provider(respx_mock) + chatgpt: Final = _chatgpt_responses(respx_mock) + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": model, "input": "Hello", "stream": True}, + headers={**_ACCOUNT_HEADERS, "user-agent": user_agent}, + ) + assert response.status_code == 200 + assert chatgpt.called is subscription + assert provider.called is not subscription + assert catalog.called is (user_agent == _CODEX_USER_AGENT) + + +def test_account_catalog_lookup_sends_the_callers_chatgpt_login_and_codex_version( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router +) -> None: + catalog: Final = respx_mock.get(f"{_BASE}/models").respond(json=_UPSTREAM_CATALOG) + _chatgpt_responses(respx_mock) + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": "shared-name", "input": "Hello", "stream": True}, + headers=_ACCOUNT_HEADERS, + ) + assert response.status_code == 200 + fetched: Final = catalog.calls.last.request + assert fetched.url.params["client_version"] == "0.165.0" + assert {name: fetched.headers[name] for name in ("authorization", "chatgpt-account-id", "originator")} == { + "authorization": f"Bearer {_CHATGPT_JWT}", + "chatgpt-account-id": "account-a", + "originator": "codex_cli_rs", + } + assert fetched.headers["user-agent"] == _CODEX_USER_AGENT + assert "x-litellm-api-key" not in fetched.headers + + +@pytest.mark.parametrize( + ("models", "etag_kept"), + [ + pytest.param(["chatgpt-native", "open-weights"], False, id="catalog with LiteLLM models"), + pytest.param(["chatgpt-native"], True, id="catalog of ChatGPT models only"), + ], +) +def test_native_responses_keep_the_models_etag_only_for_a_key_whose_catalog_was_not_merged( + client: TestClient, + respx_mock: respx.MockRouter, + mixed_router: Router, + restricted_key: Callable[..., str], + models: list[str], + etag_kept: bool, +) -> None: + respx_mock.get(f"{_BASE}/models").respond(json=_UPSTREAM_CATALOG) + _chatgpt_responses(respx_mock) + headers: Final = {**_HEADERS, "x-litellm-api-key": restricted_key(models)} + assert client.get("/backend-api/codex/models?client_version=1.0.0", headers=headers).status_code == 200 + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": "gpt-unlisted", "input": "Hello", "stream": True}, + headers=headers, + ) + assert response.status_code == 200 + assert response.headers.get("x-models-etag") == ('"upstream-v2"' if etag_kept else None) From 32a3975a2e3fe53ec288579334e8d3825f1f6ba6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 01:14:49 -0700 Subject: [PATCH 04/20] fix(proxy): keep Codex's native route on the subscription when the account list is unknown A name OpenAI lists for the caller's ChatGPT account now goes to LiteLLM only when the account's list is known and leaves it out. A failed catalog read is remembered for 60 s and the Codex version comes only from a User-Agent whose product matches the originator header. The merged catalog serves a stable ETag digest of OpenAI's ETag and the key's LiteLLM rows, and /responses replaces X-Models-Etag with the same value, memoized per key for 300 s, so Codex stops refetching or dropping its catalog cache. LiteLLM-routed native replies wrap their reasoning with the model id, so an OpenAI or Azure blob is dropped before chatgpt.com and unwrapped for LiteLLM, and the chat-completions bridge decodes wrapped thinking blocks with their signature. The access-group recipe uses chatgpt/ model names so /v1 routing does not change, and the native route checks the chatgpt/ form of a name the key holds only through that group. A none reasoning effort is dropped only under drop_params. The guide covers app tools, every routing case, and the ETag, and the catalog logs how many models the byte limit left out. --- .../chatgpt_subscription_passthrough.md | 20 +- litellm/proxy/auth/user_api_key_auth.py | 29 ++ .../proxy/common_utils/codex_model_catalog.py | 16 +- .../codex_passthrough_endpoints.py | 251 +++++++++---- .../transformation.py | 8 +- .../common_utils/test_codex_model_catalog.py | 21 +- .../test_codex_passthrough_endpoints.py | 331 ++++++++++++++---- .../test_reasoning_items.py | 26 +- 8 files changed, 556 insertions(+), 146 deletions(-) diff --git a/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md b/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md index 21c944c9f95..034b58e8642 100644 --- a/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md +++ b/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md @@ -50,35 +50,39 @@ For a native Codex model catalog and request format, change Codex's `base_url` t ### Mix ChatGPT and LiteLLM models -On the native route, Codex's model picker shows OpenAI's catalog first and then the other chat and responses models the virtual key can use. Pick one of those and Codex's requests go through LiteLLM's `/v1/responses` with the key's usual routing, budgets, guardrails, and logging. The proxy removes the ChatGPT bearer and account header from that request, so the other provider and your callbacks never receive them. Only a model listed under its exact `model_name` on another provider is routed this way. A ChatGPT model, a name OpenAI's catalog lists for the caller's ChatGPT account, or a name that only a wildcard deployment matches keeps the passthrough, so a model OpenAI adds later still uses the subscription even when a deployment of another provider has the same name. When the key can use many models, the picker lists the ones that fit in Codex's catalog size limit. +On the native route, Codex's model picker shows OpenAI's catalog first and then the other chat and responses models the virtual key can use. Pick one of those and Codex's requests go through LiteLLM's `/v1/responses` with the key's usual routing, budgets, guardrails, and logging. The proxy removes the ChatGPT bearer and account header from that request, so the other provider and your callbacks never receive them. A name is routed this way when another provider serves it as an exact `model_name`, through a router `model_group_alias`, or as a team's public model name. A ChatGPT model, a name that only a wildcard deployment matches, or a name OpenAI's catalog lists for the caller's ChatGPT account keeps the passthrough, so a model OpenAI adds later still uses the subscription even when a deployment of another provider has the same name. The proxy learns the account's list from the catalog Codex fetches, or asks OpenAI with the caller's login. When it cannot get the list, for example because OpenAI is down, the request times out, or the client is not Codex, every request keeps the passthrough for the next minute, as it did before LiteLLM models were added to the picker. When the key can use many models, the picker lists the ones that fit in Codex's catalog size limit, and the proxy logs a warning with how many it left out. + +Codex's picker sends a `none` reasoning effort for a LiteLLM model whose catalog entry lists no effort levels. With `litellm_settings: drop_params: true`, the proxy drops it and the provider applies its own default. Without `drop_params`, the effort reaches the provider unchanged. + +Codex also sends its ChatGPT app tools with every turn, which on a LiteLLM model adds a few hundred thousand prompt tokens to each turn. Set `apps = false` under `[features]` in Codex's `config.toml` to turn them off. Only a key that may call `/v1/responses` gets these models. A key whose `allowed_routes` covers only the native routes sees OpenAI's catalog unchanged, and every request it sends uses the subscription. -Codex sends earlier reasoning items back with each turn. When a session switches from another provider's model to a ChatGPT model, the proxy drops the reasoning items that other provider produced before forwarding the request to OpenAI, which would reject them. Messages, tool calls, and ChatGPT's own reasoning are kept. +Codex sends earlier reasoning items back with each turn. The proxy tags the reasoning that a LiteLLM model returns on this route, OpenAI and Azure deployments included. When a session switches to a ChatGPT model, the proxy drops those items before forwarding the request to OpenAI, which would reject them. Messages, tool calls, and ChatGPT's own reasoning are kept, and a LiteLLM model still gets its own reasoning back. -When the catalog adds LiteLLM models for a key, it carries no ETag, and that key's responses drop OpenAI's `X-Models-Etag` header, so Codex picks up a newly allowed model on its next regular catalog refresh. A key whose catalog adds no LiteLLM models keeps OpenAI's ETags. +When the catalog adds LiteLLM models for a key, its `ETag` is a digest of OpenAI's ETag and those models, and that key's responses carry the same value in `X-Models-Etag`. Codex keeps its cached catalog while the two match and fetches it again when OpenAI's catalog or the key's models change. A key whose catalog adds no LiteLLM models keeps OpenAI's ETags. ### Restrict native models with an access group -A key with a model allowlist needs a rule for ChatGPT names that OpenAI adds over time. Put the subscription wildcards in an access group: +A key with a model allowlist needs a rule for ChatGPT names that OpenAI adds over time. Put the subscription wildcards in an access group under `chatgpt/` names: ```yaml model_list: - - model_name: gpt-* + - model_name: chatgpt/gpt-* litellm_params: model: chatgpt/gpt-* model_info: access_groups: ["chatgpt-native"] - - model_name: codex-* + - model_name: chatgpt/codex-* litellm_params: model: chatgpt/codex-* model_info: access_groups: ["chatgpt-native"] ``` -Then add `chatgpt-native` to the key's models, next to any other model names it should reach. The key can call every ChatGPT model that matches the wildcards, including ones the proxy configuration does not name. These wildcards are also served on `/v1`, where a request without a ChatGPT login falls back to the proxy's own saved login if it has one, so keep that login unset as described above. +Then add `chatgpt-native` to the key's models, next to any other model names it should reach. On the native route, a key that holds only the `chatgpt/` form of a name is checked as that form, so a request for `gpt-5.6-sol` is allowed when the key may call `chatgpt/gpt-5.6-sol`. The key can call every ChatGPT model that matches the wildcards, including ones the proxy configuration does not name. -On `/v1` the router picks the longest matching pattern, so for every caller these wildcards win over a `*` catch-all deployment, and a request for any `gpt-` or `codex-` name goes to ChatGPT instead of the catch-all's provider. Give a model an exact deployment if `/v1` users should reach it on another provider. +On `/v1` these wildcards match only a request that asks for a `chatgpt/` name, so a request for a plain `gpt-` or `codex-` name still goes where it went before, such as a `*` catch-all deployment. A `/v1` request for a `chatgpt/` name without a ChatGPT login falls back to the proxy's own saved login if it has one, so keep that login unset as described above. ### Configure Pi diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 79ad4824c92..656824c8bd9 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -340,6 +340,33 @@ async def _resolve_router_settings_model_group_alias( rewrite_request_model(request_data, request, target) +_CODEX_NATIVE_RESPONSES_ROUTE: Final = "/backend-api/codex/responses" + + +def _resolve_codex_native_chatgpt_model( + request_data: dict[str, object], # mutable-ok: the request body is rewritten in place for every downstream reader + valid_token: UserAPIKeyAuth, + request: Request | None, + route: str, +) -> None: + from litellm.proxy.pass_through_endpoints.codex_passthrough_endpoints import native_chatgpt_model + from litellm.proxy.proxy_server import llm_router + + requested: Final = request_data.get("model") + if ( + route != _CODEX_NATIVE_RESPONSES_ROUTE + or request is None + or llm_router is None + or not isinstance(requested, str) + ): + return + target: Final = native_chatgpt_model(requested, llm_router, valid_token) + if target is None: + return + request.scope.setdefault(CLIENT_REQUESTED_MODEL_SCOPE_KEY, requested) + rewrite_request_model(request_data, request, target) + + def _get_model_names_for_budget_checks( model: str | list[str] | None, ) -> list[str]: @@ -4075,6 +4102,7 @@ async def _authorize_authenticated_request( RouteChecks.should_call_route(route=route, valid_token=user_api_key_auth_obj, request=request) await _normalize_claude_model(request_data, user_api_key_auth_obj, request, route) await _resolve_router_settings_model_group_alias(request_data, user_api_key_auth_obj, request, route) + _resolve_codex_native_chatgpt_model(request_data, user_api_key_auth_obj, request, route) # Single authorization point. Builder paths MUST NOT call common_checks. # Route through the same exception handler the builder uses so @@ -4542,6 +4570,7 @@ async def enforce_key_and_fallback_model_access( """ await _normalize_claude_model(request_data, valid_token, request, route) await _resolve_router_settings_model_group_alias(request_data, valid_token, request, route) + _resolve_codex_native_chatgpt_model(request_data, valid_token, request, route) config: Final = valid_token.config if config != {}: diff --git a/litellm/proxy/common_utils/codex_model_catalog.py b/litellm/proxy/common_utils/codex_model_catalog.py index 9e6f1de4bce..de58ccc5c39 100644 --- a/litellm/proxy/common_utils/codex_model_catalog.py +++ b/litellm/proxy/common_utils/codex_model_catalog.py @@ -18,9 +18,10 @@ a newer tag. from __future__ import annotations +import hashlib import json from collections.abc import Mapping, Sequence -from dataclasses import dataclass +from dataclasses import asdict, dataclass from functools import cache from itertools import accumulate, pairwise from pathlib import Path @@ -422,11 +423,24 @@ def merged_codex_catalog( serialized: Final = tuple(_compact_json(model) for model in added_models) budget: Final = byte_limit - len(_compact_json({**top, "models": upstream})) kept: Final = tuple(sorted(_kept_positions(entries, serialized, budget))) + if len(kept) < len(entries): + verbose_proxy_logger.warning( + "Codex native model catalog left out %d of %d LiteLLM models to stay within Codex's %d byte limit", + len(entries) - len(kept), + len(entries), + byte_limit, + ) if not kept: return None return _compact_json({**top, "models": [*upstream, *(added_models[position] for position in kept)]}) +def codex_catalog_etag(upstream_etag: str, rows: Sequence[CodexCatalogRow]) -> str: + canonical: Final = json.dumps([asdict(row) for row in rows], sort_keys=True, separators=(",", ":"), default=repr) + digest: Final = hashlib.sha256(json.dumps([upstream_etag, canonical]).encode()).hexdigest() + return f'"{digest}"' + + def upstream_codex_slugs(upstream_body: bytes) -> frozenset[str] | None: try: upstream: Final = _JSON_OBJECTS.validate_python(_JSON_OBJECT.validate_json(upstream_body).get("models")) diff --git a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py index ae0d390b337..4963b324dc7 100644 --- a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py @@ -13,20 +13,25 @@ from pydantic import TypeAdapter, ValidationError from starlette.types import Message import litellm +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params from litellm.llms.chatgpt.common_utils import CHATGPT_API_BASE from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared client factory has untyped params ) from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.auth_checks import check_model_access_helper from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.codex_model_catalog import ( + CodexCatalogRow, bundled_codex_models, + codex_catalog_etag, codex_entry_names_default_reasoning_level, merged_codex_catalog, upstream_codex_slugs, ) from litellm.proxy.common_utils.http_parsing_utils import ( + get_client_requested_model, read_request_body, # pyright: ignore[reportUnknownVariableType] # returns a bare dict, validated at the call ) from litellm.proxy.pass_through_endpoints.pass_through_endpoints import pass_through_request @@ -42,19 +47,26 @@ router: Final = APIRouter() _CHATGPT_PROVIDER: Final = "chatgpt" _CHATGPT_CREDENTIAL_HEADERS: Final = frozenset({b"authorization", b"chatgpt-account-id"}) _MERGED_CATALOG_DROPPED_HEADERS: Final = frozenset({"etag", "content-length", "content-type"}) +_ETAG_HEADER: Final = "etag" _MODELS_ETAG_HEADER: Final = "x-models-etag" +_ENCRYPTED_CONTENT_AFFINITY_FLAG: Final = "encrypted_content_affinity_enabled" _LITELLM_RESPONSES_PATH: Final = "/v1/responses" _CODEX_NO_DEFAULT_EFFORT: Final = "none" _JSON_OBJECT: Final = TypeAdapter(dict[str, object]) _INPUT_ITEMS: Final = TypeAdapter(tuple[object, ...]) _CACHED_SLUGS: Final = TypeAdapter(frozenset[str]) +_CACHED_MODELS_ETAG: Final = TypeAdapter(dict[Literal["etag"], str]) +_MODEL_NAMES: Final = TypeAdapter(list[str]) _HEADER_DERIVED_STATE: Final = frozenset({"_cached_headers"}) _ACCOUNT_CATALOG_HEADERS: Final = frozenset({"authorization", "chatgpt-account-id", "originator", "user-agent"}) -_CODEX_USER_AGENT_PRODUCT: Final = re.compile(r"[^/]+/(\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.+-]+)?)") +_CODEX_USER_AGENT_PRODUCT: Final = re.compile(r"([^/]+)/(\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.+-]+)?)") _ACCOUNT_CATALOG_TIMEOUT_SECONDS: Final = 5.0 _SHARED_CACHE_TTL_SECONDS: Final = 3600 +_ACCOUNT_SLUGS_UNAVAILABLE: Final = "unavailable" +_ACCOUNT_SLUGS_UNAVAILABLE_TTL_SECONDS: Final = 60 _ACCOUNT_SLUGS_CACHE_PREFIX: Final = "codex_account_slugs:" -_MERGED_CATALOG_CACHE_PREFIX: Final = "codex_catalog_added_litellm_rows:" +_MODELS_ETAG_CACHE_PREFIX: Final = "codex_models_etag:" +_MODELS_ETAG_TTL_SECONDS: Final = 300 _HEADERS: Final = frozenset( { "authorization", @@ -87,6 +99,13 @@ def _serves_by_name(deployment: Deployment, names: frozenset[str]) -> bool: ) +def _candidate_deployments(model: str, llm_router: "Router", team_id: str | None) -> tuple[Deployment, ...]: + candidates: Final = tuple( + llm_router.get_deployment(model_id) for model_id in llm_router.get_candidate_model_ids_for_route(model, team_id) + ) + return tuple(deployment for deployment in candidates if deployment is not None) + + def _routes_to_litellm(model: object, llm_router: "Router | None", team_id: str | None) -> bool: if ( not isinstance(model, str) @@ -95,14 +114,32 @@ def _routes_to_litellm(model: object, llm_router: "Router | None", team_id: str or f"{_CHATGPT_PROVIDER}/{model}" in litellm.model_cost # pyright: ignore[reportUnknownMemberType] # registry is dynamically typed ): return False - candidates: Final = tuple( - llm_router.get_deployment(model_id) for model_id in llm_router.get_candidate_model_ids_for_route(model, team_id) - ) - deployments: Final = tuple(deployment for deployment in candidates if deployment is not None) + deployments: Final = _candidate_deployments(model, llm_router, team_id) names: Final = frozenset({model, llm_router.get_model_from_alias(model=model) or model}) return bool(deployments) and all(_serves_by_name(deployment, names) for deployment in deployments) +def _key_and_team_allow(model: str, llm_router: "Router", user_api_key_dict: UserAPIKeyAuth) -> bool: + allowed_lists: Final = ( + _MODEL_NAMES.validate_python(user_api_key_dict.models), + _MODEL_NAMES.validate_python(user_api_key_dict.team_models), + ) + return all( + check_model_access_helper(model, llm_router, allowed, team_id=user_api_key_dict.team_id) + for allowed in allowed_lists + ) + + +def native_chatgpt_model(model: str, llm_router: "Router", user_api_key_dict: UserAPIKeyAuth) -> str | None: + if model.startswith(f"{_CHATGPT_PROVIDER}/") or _key_and_team_allow(model, llm_router, user_api_key_dict): + return None + chatgpt_model: Final = f"{_CHATGPT_PROVIDER}/{model}" + deployments: Final = _candidate_deployments(chatgpt_model, llm_router, user_api_key_dict.team_id) + if not deployments or not all(_is_chatgpt(deployment) for deployment in deployments): + return None + return chatgpt_model if _key_and_team_allow(chatgpt_model, llm_router, user_api_key_dict) else None + + class _SharedCache(Protocol): async def async_get_cache(self, key: str) -> object: ... @@ -136,21 +173,27 @@ def _cached_slugs(cached: object) -> frozenset[str] | None: return None -async def _remember_account_slugs(account_id: str, slugs: frozenset[str], cache: _SharedCache) -> None: +async def _remember_account_slugs(account_id: str, slugs: frozenset[str] | None, cache: _SharedCache) -> None: + if slugs is None: + await cache.async_set_cache( + _account_slugs_cache_key(account_id), + _ACCOUNT_SLUGS_UNAVAILABLE, + ttl=_ACCOUNT_SLUGS_UNAVAILABLE_TTL_SECONDS, + ) + return await cache.async_set_cache( _account_slugs_cache_key(account_id), tuple(sorted(slugs)), ttl=_SHARED_CACHE_TTL_SECONDS ) -def _codex_client_version(user_agent: str) -> str | None: - product: Final = _CODEX_USER_AGENT_PRODUCT.fullmatch(user_agent.partition(" ")[0]) - return product.group(1) if product is not None else None - - -async def _fetched_account_slugs(headers: Mapping[str, str]) -> frozenset[str] | None: - client_version: Final = _codex_client_version(headers.get("user-agent", "")) - if client_version is None: +def _codex_client_version(headers: Mapping[str, str]) -> str | None: + product: Final = _CODEX_USER_AGENT_PRODUCT.fullmatch(headers.get("user-agent", "").partition(" ")[0]) + if product is None or product.group(1) != headers.get("originator"): return None + return product.group(2) + + +async def _fetched_account_slugs(headers: Mapping[str, str], client_version: str) -> frozenset[str] | None: client: Final = get_async_httpx_client( llm_provider=httpxSpecialProvider.PassThroughEndpoint, params={"timeout": _ACCOUNT_CATALOG_TIMEOUT_SECONDS}, @@ -170,12 +213,17 @@ async def _account_slugs(headers: Mapping[str, str], cache: _SharedCache) -> fro account_id: Final = headers.get("chatgpt-account-id") if not account_id: return None - cached: Final = _cached_slugs(await cache.async_get_cache(_account_slugs_cache_key(account_id))) + cached_value: Final = await cache.async_get_cache(_account_slugs_cache_key(account_id)) + if cached_value == _ACCOUNT_SLUGS_UNAVAILABLE: + return None + cached: Final = _cached_slugs(cached_value) if cached is not None: return cached - fetched: Final = await _fetched_account_slugs(headers) - if fetched is not None: - await _remember_account_slugs(account_id, fetched, cache) + client_version: Final = _codex_client_version(headers) + if client_version is None: + return None + fetched: Final = await _fetched_account_slugs(headers, client_version) + await _remember_account_slugs(account_id, fetched, cache) return fetched @@ -191,40 +239,18 @@ async def _serves_through_litellm( ): return False account_slugs: Final = await _account_slugs(headers, cache) - return account_slugs is None or model not in account_slugs + return account_slugs is not None and model not in account_slugs -def _merged_catalog_cache_key(token: str) -> str: - return f"{_MERGED_CATALOG_CACHE_PREFIX}{token}" - - -async def _remember_catalog( - headers: Mapping[str, str], - slugs: frozenset[str], - user_api_key_dict: UserAPIKeyAuth, - added_litellm_rows: bool, - cache: _SharedCache, -) -> None: - account_id: Final = headers.get("chatgpt-account-id") - if account_id: - await _remember_account_slugs(account_id, slugs, cache) - if user_api_key_dict.token is not None: - await cache.async_set_cache( - _merged_catalog_cache_key(user_api_key_dict.token), added_litellm_rows, ttl=_SHARED_CACHE_TTL_SECONDS - ) - - -async def _catalog_may_have_litellm_rows(user_api_key_dict: UserAPIKeyAuth, cache: _SharedCache) -> bool: - if user_api_key_dict.token is None: - return True - return await cache.async_get_cache(_merged_catalog_cache_key(user_api_key_dict.token)) is not False +def _json_object(value: object) -> Mapping[str, object]: + try: + return _JSON_OBJECT.validate_python(value) + except ValidationError: + return MappingProxyType({}) def _scope_state(request: Request) -> Mapping[str, object]: - try: - return _JSON_OBJECT.validate_python(request.scope.get("state")) - except ValidationError: - return MappingProxyType({}) + return _json_object(request.scope.get("state")) def _as_litellm_responses_request(request: Request, body: bytes, parsed_body: Mapping[str, object]) -> Request: @@ -250,6 +276,14 @@ def _as_litellm_responses_request(request: Request, body: bytes, parsed_body: Ma ) +def _drops_params(body: Mapping[str, object], model: str, llm_router: "Router", team_id: str | None) -> bool: + deployments: Final = _candidate_deployments(model, llm_router, team_id) + return drops_unsupported_params(body.get("drop_params")) or ( + bool(deployments) + and all(normalize_drop_params(deployment.litellm_params.drop_params) is True for deployment in deployments) + ) + + def _with_provider_default_reasoning( body: Mapping[str, object], llm_router: "Router | None", team_id: str | None ) -> Mapping[str, object]: @@ -262,12 +296,20 @@ def _with_provider_default_reasoning( reasoning.get("effort") != _CODEX_NO_DEFAULT_EFFORT or not isinstance(model, str) or llm_router is None + or not _drops_params(body, model, llm_router, team_id) or codex_entry_names_default_reasoning_level(model, llm_router, team_id) ): return body return {**body, "reasoning": {key: value for key, value in reasoning.items() if key != "effort"}} +def _with_wrapped_reasoning(body: Mapping[str, object]) -> Mapping[str, object]: + return { + **body, + "litellm_metadata": {**_json_object(body.get("litellm_metadata")), _ENCRYPTED_CONTENT_AFFINITY_FLAG: True}, + } + + def _is_unpersisted_reasoning_reference(item: object) -> bool: try: fields: Final = _JSON_OBJECT.validate_python(item) @@ -297,17 +339,76 @@ def _input_chatgpt_accepts(body: Mapping[str, object]) -> tuple[object, ...] | N return kept if len(kept) < len(items) else None -async def _merged_catalog( - upstream_body: bytes, user_api_key_dict: UserAPIKeyAuth, llm_router: "Router | None" -) -> str | None: +def _chatgpt_body(body: Mapping[str, object], client_model: str | None) -> Mapping[str, object] | None: + kept_input: Final = _input_chatgpt_accepts(body) + restored_model: Final = client_model is not None and body.get("model") == f"{_CHATGPT_PROVIDER}/{client_model}" + if kept_input is None and not restored_model: + return None + return { + **body, + **({"input": list(kept_input)} if kept_input is not None else {}), + **({"model": client_model} if restored_model else {}), + } + + +async def _litellm_catalog_rows( + user_api_key_dict: UserAPIKeyAuth, llm_router: "Router | None" +) -> tuple[CodexCatalogRow, ...]: from litellm.proxy.proxy_server import codex_catalog_rows_for_key if llm_router is None or not _may_call_litellm_responses(user_api_key_dict): - return None + return () rows: Final = await codex_catalog_rows_for_key(user_api_key_dict) - return merged_codex_catalog( - upstream_body, - tuple(row for row in rows if _routes_to_litellm(row.id, llm_router, user_api_key_dict.team_id)), + return tuple(row for row in rows if _routes_to_litellm(row.id, llm_router, user_api_key_dict.team_id)) + + +def _models_etag_cache_key(token: str, upstream_etag: str) -> str: + return f"{_MODELS_ETAG_CACHE_PREFIX}{hashlib.sha256(json.dumps([token, upstream_etag]).encode()).hexdigest()}" + + +async def _remembered_models_etag( + user_api_key_dict: UserAPIKeyAuth, upstream_etag: str, rows: tuple[CodexCatalogRow, ...], cache: _SharedCache +) -> str: + etag: Final = codex_catalog_etag(upstream_etag, rows) if rows else upstream_etag + if user_api_key_dict.token is not None: + await cache.async_set_cache( + _models_etag_cache_key(user_api_key_dict.token, upstream_etag), + {"etag": etag}, + ttl=_MODELS_ETAG_TTL_SECONDS, + ) + return etag + + +async def _cached_models_etag(token: str | None, upstream_etag: str, cache: _SharedCache) -> str | None: + if token is None: + return None + try: + return _CACHED_MODELS_ETAG.validate_python( + await cache.async_get_cache(_models_etag_cache_key(token, upstream_etag)) + )["etag"] + except ValidationError: + return None + + +async def _key_models_etag( + upstream_etag: str, user_api_key_dict: UserAPIKeyAuth, llm_router: "Router | None", cache: _SharedCache +) -> str: + cached: Final = await _cached_models_etag(user_api_key_dict.token, upstream_etag, cache) + if cached is not None: + return cached + rows: Final = await _litellm_catalog_rows(user_api_key_dict, llm_router) + return await _remembered_models_etag(user_api_key_dict, upstream_etag, rows, cache) + + +def _with_etag(response: Response, content: bytes | str, etag: str | None) -> Response: + kept_headers: Final = { + name: value for name, value in response.headers.items() if name not in _MERGED_CATALOG_DROPPED_HEADERS + } + return Response( + content=content, + status_code=response.status_code, + headers={**kept_headers, **({_ETAG_HEADER: etag} if etag is not None else {})}, + media_type="application/json", ) @@ -330,18 +431,18 @@ async def _catalog( slugs: Final = upstream_codex_slugs(upstream_body) if slugs is None: return upstream - merged: Final = await _merged_catalog(upstream_body, user_api_key_dict, llm_router) - await _remember_catalog(headers, slugs, user_api_key_dict, merged is not None, cache) - if merged is None: - return upstream - return Response( - content=merged, - status_code=upstream.status_code, - headers={ - name: value for name, value in upstream.headers.items() if name not in _MERGED_CATALOG_DROPPED_HEADERS - }, - media_type="application/json", + account_id: Final = headers.get("chatgpt-account-id") + if account_id: + await _remember_account_slugs(account_id, slugs, cache) + rows: Final = await _litellm_catalog_rows(user_api_key_dict, llm_router) + upstream_etag: Final = upstream.headers.get(_ETAG_HEADER) + etag: Final = ( + None if upstream_etag is None else await _remembered_models_etag(user_api_key_dict, upstream_etag, rows, cache) ) + if not rows: + return upstream + merged: Final = merged_codex_catalog(upstream_body, rows) + return _with_etag(upstream, merged if merged is not None else upstream_body, etag) async def _responses( @@ -355,25 +456,29 @@ async def _responses( body: Final = _JSON_OBJECT.validate_python(await read_request_body(request)) if await _serves_through_litellm(body.get("model"), llm_router, user_api_key_dict, headers, cache): - litellm_body: Final = _with_provider_default_reasoning(body, llm_router, user_api_key_dict.team_id) - raw_body: Final = await request.body() if litellm_body is body else json.dumps(litellm_body).encode() + litellm_body: Final = _with_wrapped_reasoning( + _with_provider_default_reasoning(body, llm_router, user_api_key_dict.team_id) + ) return await responses_api( - _as_litellm_responses_request(request, raw_body, litellm_body), + _as_litellm_responses_request(request, json.dumps(litellm_body).encode(), litellm_body), fastapi_response, user_api_key_dict.model_copy(update={"request_route": _LITELLM_RESPONSES_PATH}), ) - kept_input: Final = _input_chatgpt_accepts(body) + chatgpt_body: Final = _chatgpt_body(body, get_client_requested_model(request)) upstream: Final = await pass_through_request( request=request, target=f"{CHATGPT_API_BASE}/responses", custom_headers=dict(headers), user_api_key_dict=user_api_key_dict, - custom_body={**body, "input": list(kept_input)} if kept_input is not None else None, + custom_body=dict(chatgpt_body) if chatgpt_body is not None else None, custom_llm_provider=_CHATGPT_PROVIDER, upstream_headers_filter=lambda _: headers, ) - if await _catalog_may_have_litellm_rows(user_api_key_dict, cache): - del upstream.headers[_MODELS_ETAG_HEADER] + upstream_models_etag: Final = upstream.headers.get(_MODELS_ETAG_HEADER) + if upstream_models_etag is not None: + upstream.headers[_MODELS_ETAG_HEADER] = await _key_models_etag( + upstream_models_etag, user_api_key_dict, llm_router, cache + ) return upstream diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index bc92bc9cb3a..75cc967d752 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -65,6 +65,7 @@ from litellm.responses.bridge_check import chat_target_accepts_reasoning_summary from litellm.responses.litellm_completion_transformation.session_handler import ( ResponsesSessionHandler, ) +from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.anthropic import AnthropicToolSearchToolRegex from litellm.types.llms.base import CachedTokensDetails from litellm.types.llms.openai import ( @@ -2355,7 +2356,12 @@ class LiteLLMCompletionResponsesConfig: Returns None for anything this deployment did not write, so a genuinely opaque blob is still skipped rather than forwarded as garbage. """ - decoded: Final = decode_thinking_blocks(input_item.get("encrypted_content")) + encrypted_content: Final = input_item.get("encrypted_content") + decoded: Final = decode_thinking_blocks( + ResponsesAPIRequestUtils.unwrap_encrypted_content_with_model_id(encrypted_content)[1] + if isinstance(encrypted_content, str) + else encrypted_content + ) if decoded is None: return None return tuple( diff --git a/tests/unit/proxy/common_utils/test_codex_model_catalog.py b/tests/unit/proxy/common_utils/test_codex_model_catalog.py index 70647fbbc4c..395e73d6464 100644 --- a/tests/unit/proxy/common_utils/test_codex_model_catalog.py +++ b/tests/unit/proxy/common_utils/test_codex_model_catalog.py @@ -1,9 +1,11 @@ import json +import logging from types import MappingProxyType import pytest import litellm +from litellm._logging import verbose_proxy_logger from litellm.proxy.common_utils.codex_model_catalog import ( CODEX_CATALOG_BYTE_LIMIT, CodexCatalogRow, @@ -382,6 +384,21 @@ def test_merged_catalog_keeps_upstream_whole_and_adds_only_the_entries_that_fit_ assert merged(*rows, byte_limit=upstream_only) is None +def test_merged_catalog_warns_once_with_how_many_models_the_byte_limit_left_out(caplog, monkeypatch): + upstream = json.dumps({"models": [{"slug": "upstream-model", "priority": 0}]}).encode() + rows = tuple(_row(f"model-{index}") for index in range(4)) + limit = len(merged_codex_catalog(upstream, rows[:1], stock=_STOCK, instructions=_PROMPT, byte_limit=10**6).encode()) + monkeypatch.setattr(verbose_proxy_logger, "propagate", True) + + with caplog.at_level(logging.WARNING, logger=verbose_proxy_logger.name): + merged_codex_catalog(upstream, rows, stock=_STOCK, instructions=_PROMPT, byte_limit=limit) + merged_codex_catalog(upstream, rows[:1], stock=_STOCK, instructions=_PROMPT, byte_limit=limit) + + assert [record.getMessage() for record in caplog.records] == [ + f"Codex native model catalog left out 3 of 4 LiteLLM models to stay within Codex's {limit} byte limit" + ] + + def test_unlimited_body_keeps_every_entry(): rows = tuple(_row(f"model-{index}") for index in range(3)) @@ -565,7 +582,9 @@ def test_entry_names_a_default_reasoning_level_exactly_when_the_served_entry_car ] ) names = ("open-weights", "coding-model") - listing = tuple({"id": name, "object": "model", "created": 0, "owned_by": "openai", "mode": "chat"} for name in names) + listing = tuple( + {"id": name, "object": "model", "created": 0, "owned_by": "openai", "mode": "chat"} for name in names + ) served = json.loads(codex_model_list_body(listing, tuple((name, name) for name in names), router))["models"] assert [codex_entry_names_default_reasoning_level(entry["slug"], router, None) for entry in served] == [ diff --git a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py index 9e781e4524b..f9e8cca4198 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py @@ -209,6 +209,14 @@ _CHATGPT_JWT: Final = ( + "." ) _PROVIDER_KEY: Final = "test-provider-key" +_CODEX_USER_AGENT: Final = "codex_cli_rs/0.165.0 (Ubuntu 24.4.0; x86_64) xterm-256color" +_ACCOUNT_HEADERS: Final = { + **_HEADERS, + "authorization": f"Bearer {_CHATGPT_JWT}", + "chatgpt-account-id": "account-a", + "originator": "codex_cli_rs", + "user-agent": _CODEX_USER_AGENT, +} _BUNDLED_SLUG: Final = next(iter(bundled_codex_models())) _PRICED_CHATGPT_SLUG: Final = "priced-subscription-model" _UPSTREAM_CATALOG: Final = { @@ -237,12 +245,12 @@ def mixed_router(monkeypatch: pytest.MonkeyPatch) -> Router: router: Final = Router( model_list=[ { - "model_name": "gpt-*", + "model_name": "chatgpt/gpt-*", "litellm_params": {"model": "chatgpt/gpt-*"}, "model_info": {"access_groups": ["chatgpt-native"]}, }, { - "model_name": "codex-*", + "model_name": "chatgpt/codex-*", "litellm_params": {"model": "chatgpt/codex-*"}, "model_info": {"access_groups": ["chatgpt-native"]}, }, @@ -279,7 +287,6 @@ def test_native_catalog_lists_the_keys_other_chat_models_after_chatgpts( assert [model["slug"] for model in added] == ["open-weights"] assert added[0]["visibility"] == "list" assert added[0]["priority"] > max(model["priority"] for model in _UPSTREAM_CATALOG["models"]) - assert "etag" not in response.headers @pytest.mark.parametrize( @@ -300,8 +307,8 @@ def test_native_catalog_relays_an_unmergeable_upstream_answer_unchanged( assert response.headers["etag"] == '"e"' -def _provider_stream(text: str) -> bytes: - completed: Final = { +def _provider_completed(text: str) -> dict[str, object]: + return { "id": "resp_open_weights", "object": "response", "created_at": 1700000000, @@ -318,6 +325,10 @@ def _provider_stream(text: str) -> bytes: ], "usage": {"input_tokens": 3, "output_tokens": 1, "total_tokens": 4}, } + + +def _provider_stream(text: str) -> bytes: + completed: Final = _provider_completed(text) events: Final = ( { "type": "response.created", @@ -358,26 +369,36 @@ class PreCallRecorder(CustomLogger): return data +def _provider_reply(request: httpx.Request) -> httpx.Response: + if json.loads(request.content).get("stream") is not True: + return httpx.Response(200, json=_provider_completed("Hi")) + return httpx.Response(200, content=_provider_stream("Hi"), headers={"content-type": "text/event-stream"}) + + def _open_weights_provider(respx_mock: respx.MockRouter) -> respx.Route: - return respx_mock.post(f"{_PROVIDER_BASE}/responses").mock( - side_effect=lambda _: httpx.Response( - 200, content=_provider_stream("Hi"), headers={"content-type": "text/event-stream"} - ) - ) + return respx_mock.post(f"{_PROVIDER_BASE}/responses").mock(side_effect=_provider_reply) + + +_UPSTREAM_MODELS_ETAG: Final = '"upstream-v2"' def _chatgpt_responses(respx_mock: respx.MockRouter) -> respx.Route: return respx_mock.post(f"{_BASE}/responses").respond( content=b"event: response.completed\ndata: {}\n\n", - headers={"content-type": "text/event-stream", "x-models-etag": '"upstream-v2"'}, + headers={"content-type": "text/event-stream", "x-models-etag": _UPSTREAM_MODELS_ETAG}, ) +def _account_catalog(respx_mock: respx.MockRouter) -> respx.Route: + return respx_mock.get(f"{_BASE}/models").respond(json=_UPSTREAM_CATALOG) + + def test_litellm_model_on_native_route_streams_like_v1_without_chatgpt_credentials( client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router, monkeypatch: pytest.MonkeyPatch ) -> None: recorder: Final = PreCallRecorder() monkeypatch.setattr(litellm, "callbacks", [recorder]) + _account_catalog(respx_mock) chatgpt: Final = respx_mock.post(f"{_BASE}/responses").respond(json={}) provider: Final = respx_mock.post(f"{_PROVIDER_BASE}/responses").mock( side_effect=lambda _: httpx.Response( @@ -385,11 +406,7 @@ def test_litellm_model_on_native_route_streams_like_v1_without_chatgpt_credentia ) ) body: Final = {"model": "open-weights", "input": "Say hi", "stream": True, "store": False} - native: Final = client.post( - "/backend-api/codex/responses", - json=body, - headers={**_HEADERS, "authorization": f"Bearer {_CHATGPT_JWT}", "chatgpt-account-id": "account-a"}, - ) + native: Final = client.post("/backend-api/codex/responses", json=body, headers=_ACCOUNT_HEADERS) v1: Final = client.post("/v1/responses", json=body, headers={"authorization": f"Bearer {MASTER_KEY}"}) assert native.status_code == v1.status_code == 200 assert native.headers["content-type"].startswith("text/event-stream") @@ -411,11 +428,12 @@ def test_litellm_model_on_native_route_keeps_chatgpt_credentials_out_of_hook_raw ) -> None: recorder: Final = PreCallRecorder() monkeypatch.setattr(litellm, "callbacks", [recorder]) + _account_catalog(respx_mock) provider: Final = _open_weights_provider(respx_mock) response: Final = client.post( "/backend-api/codex/responses", json={"model": "open-weights", "input": "Say hi", "stream": True, "store": False}, - headers={**_HEADERS, "authorization": f"Bearer {_CHATGPT_JWT}", "chatgpt-account-id": "account-a"}, + headers=_ACCOUNT_HEADERS, ) assert response.status_code == 200 assert provider.called @@ -425,11 +443,14 @@ def test_litellm_model_on_native_route_keeps_chatgpt_credentials_out_of_hook_raw @pytest.mark.parametrize( - ("model", "sent", "expected"), + ("model", "sent", "drop_params", "expected"), [ - pytest.param("open-weights", "none", None, id="picker fills in none for a model without a default effort"), - pytest.param("open-weights", "high", "high", id="effort set for a model without a default effort"), - pytest.param("coding-model", "none", "none", id="none on a model whose entry names a default effort"), + pytest.param("open-weights", "none", None, "none", id="none without drop_params"), + pytest.param("open-weights", "none", "global", None, id="none under litellm_settings drop_params"), + pytest.param("open-weights", "none", "deployment", None, id="none under the deployment's drop_params"), + pytest.param("open-weights", "none", "request", None, id="none under the request's drop_params"), + pytest.param("open-weights", "high", "global", "high", id="effort set for a model without a default effort"), + pytest.param("coding-model", "none", "global", "none", id="none on a model whose entry names a default effort"), ], ) def test_litellm_model_on_native_route_gets_the_reasoning_effort_codex_m_would_send( @@ -438,24 +459,42 @@ def test_litellm_model_on_native_route_gets_the_reasoning_effort_codex_m_would_s monkeypatch: pytest.MonkeyPatch, model: str, sent: str, + drop_params: str | None, expected: str | None, ) -> None: + monkeypatch.setattr(litellm, "drop_params", drop_params == "global") + open_weights: Final = _open_weights("open-weights") monkeypatch.setattr( proxy_server, "llm_router", - Router(model_list=[_open_weights("open-weights"), _open_weights("coding-model", upstream="gpt-5.5")]), + Router( + model_list=[ + { + **open_weights, + "litellm_params": {**open_weights["litellm_params"], "drop_params": drop_params == "deployment"}, + }, + _open_weights("coding-model", upstream="gpt-5.5"), + ] + ), ) + _account_catalog(respx_mock) provider: Final = respx_mock.post(f"{_PROVIDER_BASE}/responses").mock( side_effect=lambda _: httpx.Response( 200, content=_provider_stream("Hi"), headers={"content-type": "text/event-stream"} ) ) - body: Final = {"model": model, "input": "Say hi", "stream": True, "store": False} + body: Final = { + "model": model, + "input": "Say hi", + "stream": True, + "store": False, + **({"drop_params": True} if drop_params == "request" else {}), + } expected_reasoning: Final = {"summary": "auto"} if expected is None else {"effort": expected, "summary": "auto"} native: Final = client.post( "/backend-api/codex/responses", json={**body, "reasoning": {"effort": sent, "summary": "auto"}}, - headers={**_HEADERS, "authorization": f"Bearer {_CHATGPT_JWT}"}, + headers=_ACCOUNT_HEADERS, ) v1: Final = client.post( "/v1/responses", @@ -512,7 +551,6 @@ def test_chatgpt_model_keeps_passthrough_without_other_providers_reasoning( **body, "input": [user_turn, chatgpt_reasoning, assistant_turn], } - assert "x-models-etag" not in response.headers @pytest.fixture @@ -555,7 +593,7 @@ def test_native_catalog_adds_only_the_litellm_models_the_key_can_call( *(model["slug"] for model in _UPSTREAM_CATALOG["models"]), *added, ] - assert ("etag" in response.headers) is not added + assert (response.headers["etag"] == '"upstream-v1"') is not added @pytest.mark.parametrize( @@ -587,6 +625,22 @@ def test_native_route_serves_only_the_models_the_key_allows( assert (response.status_code == 200) is allowed assert chatgpt.called is allowed assert not provider.called + assert [json.loads(call.request.content)["model"] for call in chatgpt.calls] == ([model] if allowed else []) + + +def test_v1_request_for_a_gpt_name_still_reaches_the_catch_all_with_the_access_group_recipe( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router +) -> None: + provider: Final = _open_weights_provider(respx_mock) + chatgpt: Final = _chatgpt_responses(respx_mock) + response: Final = client.post( + "/v1/responses", + json={"model": "gpt-unlisted", "input": "Hello", "stream": True}, + headers={"authorization": f"Bearer {MASTER_KEY}"}, + ) + assert response.status_code == 200, response.text + assert provider.called + assert not chatgpt.called _NATIVE_ROUTES: Final = ("/backend-api/codex/models", "/backend-api/codex/responses") @@ -634,12 +688,16 @@ def test_native_responses_reach_litellm_models_only_for_a_key_that_may_call_v1_r allowed_routes: tuple[str, ...], through_litellm: bool, ) -> None: + _account_catalog(respx_mock) provider: Final = _open_weights_provider(respx_mock) chatgpt: Final = _chatgpt_responses(respx_mock) response: Final = client.post( "/backend-api/codex/responses", json={"model": "open-weights", "input": "Hello", "stream": True}, - headers={**_HEADERS, "x-litellm-api-key": restricted_key(["chatgpt-native", "open-weights"], allowed_routes)}, + headers={ + **_ACCOUNT_HEADERS, + "x-litellm-api-key": restricted_key(["chatgpt-native", "open-weights"], allowed_routes), + }, ) assert response.status_code == 200, response.text assert provider.called is through_litellm @@ -691,16 +749,6 @@ def test_chatgpt_passthrough_drops_reasoning_ids_chatgpt_never_persisted( ) -_CODEX_USER_AGENT: Final = "codex_cli_rs/0.165.0 (Ubuntu 24.4.0; x86_64) xterm-256color" -_ACCOUNT_HEADERS: Final = { - **_HEADERS, - "authorization": f"Bearer {_CHATGPT_JWT}", - "chatgpt-account-id": "account-a", - "originator": "codex_cli_rs", - "user-agent": _CODEX_USER_AGENT, -} - - def test_exact_deployment_named_like_an_account_slug_stays_on_the_subscription_after_the_catalog_read( client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router ) -> None: @@ -719,17 +767,18 @@ def test_exact_deployment_named_like_an_account_slug_stays_on_the_subscription_a assert catalog.call_count == 1 +_LISTED: Final = httpx.Response(200, json=_UPSTREAM_CATALOG) + + @pytest.mark.parametrize( ("model", "user_agent", "catalog_answer", "subscription"), [ - pytest.param("shared-name", _CODEX_USER_AGENT, httpx.Response(200, json=_UPSTREAM_CATALOG), True, id="listed"), - pytest.param("shared-name", _CODEX_USER_AGENT, httpx.Response(503), False, id="catalog unavailable"), - pytest.param( - "shared-name", "codex_cli_rs", httpx.Response(200, json=_UPSTREAM_CATALOG), False, id="no version" - ), - pytest.param( - "open-weights", _CODEX_USER_AGENT, httpx.Response(200, json=_UPSTREAM_CATALOG), False, id="unlisted" - ), + pytest.param("shared-name", _CODEX_USER_AGENT, _LISTED, True, id="listed"), + pytest.param("open-weights", _CODEX_USER_AGENT, _LISTED, False, id="unlisted"), + pytest.param("open-weights", _CODEX_USER_AGENT, httpx.Response(503), True, id="catalog unavailable"), + pytest.param("open-weights", _CODEX_USER_AGENT, httpx.ReadTimeout("slow"), True, id="catalog timeout"), + pytest.param("open-weights", "codex_cli_rs", _LISTED, True, id="no version"), + pytest.param("open-weights", "curl/8.5.0", _LISTED, True, id="user agent of another client"), ], ) def test_native_responses_ask_chatgpt_which_models_the_account_lists_on_a_cache_miss( @@ -738,10 +787,10 @@ def test_native_responses_ask_chatgpt_which_models_the_account_lists_on_a_cache_ mixed_router: Router, model: str, user_agent: str, - catalog_answer: httpx.Response, + catalog_answer: httpx.Response | httpx.TimeoutException, subscription: bool, ) -> None: - catalog: Final = respx_mock.get(f"{_BASE}/models").mock(return_value=catalog_answer) + catalog: Final = respx_mock.get(f"{_BASE}/models").mock(side_effect=[catalog_answer]) provider: Final = _open_weights_provider(respx_mock) chatgpt: Final = _chatgpt_responses(respx_mock) response: Final = client.post( @@ -755,6 +804,26 @@ def test_native_responses_ask_chatgpt_which_models_the_account_lists_on_a_cache_ assert catalog.called is (user_agent == _CODEX_USER_AGENT) +def test_a_failed_account_catalog_read_keeps_the_next_turns_on_the_subscription_without_asking_again( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router +) -> None: + catalog: Final = respx_mock.get(f"{_BASE}/models").respond(503) + provider: Final = _open_weights_provider(respx_mock) + chatgpt: Final = _chatgpt_responses(respx_mock) + responses: Final = tuple( + client.post( + "/backend-api/codex/responses", + json={"model": "open-weights", "input": "Hello", "stream": True}, + headers=_ACCOUNT_HEADERS, + ) + for _ in range(2) + ) + assert [response.status_code for response in responses] == [200, 200] + assert chatgpt.call_count == 2 + assert not provider.called + assert catalog.call_count == 1 + + def test_account_catalog_lookup_sends_the_callers_chatgpt_login_and_codex_version( client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router ) -> None: @@ -777,29 +846,169 @@ def test_account_catalog_lookup_sends_the_callers_chatgpt_login_and_codex_versio assert "x-litellm-api-key" not in fetched.headers +def _models_etags(client: TestClient, headers: dict[str, str]) -> tuple[str | None, str | None]: + catalog: Final = client.get("/backend-api/codex/models?client_version=0.165.0", headers=headers) + responses: Final = client.post( + "/backend-api/codex/responses", + json={"model": "gpt-unlisted", "input": "Hello", "stream": True}, + headers=headers, + ) + assert catalog.status_code == responses.status_code == 200, (catalog.text, responses.text) + return catalog.headers.get("etag"), responses.headers.get("x-models-etag") + + @pytest.mark.parametrize( - ("models", "etag_kept"), + ("models", "merged"), [ - pytest.param(["chatgpt-native", "open-weights"], False, id="catalog with LiteLLM models"), - pytest.param(["chatgpt-native"], True, id="catalog of ChatGPT models only"), + pytest.param(["chatgpt-native", "open-weights"], True, id="catalog with LiteLLM models"), + pytest.param(["chatgpt-native"], False, id="catalog of ChatGPT models only"), ], ) -def test_native_responses_keep_the_models_etag_only_for_a_key_whose_catalog_was_not_merged( +def test_native_catalog_and_responses_carry_the_same_models_etag( client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router, restricted_key: Callable[..., str], models: list[str], - etag_kept: bool, + merged: bool, ) -> None: - respx_mock.get(f"{_BASE}/models").respond(json=_UPSTREAM_CATALOG) + respx_mock.get(f"{_BASE}/models").respond(json=_UPSTREAM_CATALOG, headers={"etag": _UPSTREAM_MODELS_ETAG}) _chatgpt_responses(respx_mock) - headers: Final = {**_HEADERS, "x-litellm-api-key": restricted_key(models)} - assert client.get("/backend-api/codex/models?client_version=1.0.0", headers=headers).status_code == 200 - response: Final = client.post( - "/backend-api/codex/responses", - json={"model": "gpt-unlisted", "input": "Hello", "stream": True}, - headers=headers, + catalog_etag, responses_etag = _models_etags(client, {**_HEADERS, "x-litellm-api-key": restricted_key(models)}) + assert catalog_etag is not None + assert catalog_etag == responses_etag + assert (catalog_etag != _UPSTREAM_MODELS_ETAG) is merged + + +def test_native_models_etag_is_the_same_on_replicas_with_empty_caches( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setitem(proxy_server.app.dependency_overrides, codex_shared_cache, lambda: DualCache()) + respx_mock.get(f"{_BASE}/models").respond(json=_UPSTREAM_CATALOG, headers={"etag": _UPSTREAM_MODELS_ETAG}) + _chatgpt_responses(respx_mock) + first_replica: Final = _models_etags(client, _HEADERS) + second_replica: Final = _models_etags(client, _HEADERS) + assert first_replica[0] not in (None, _UPSTREAM_MODELS_ETAG) + assert len({*first_replica, *second_replica}) == 1 + + +def test_native_models_etag_changes_with_the_keys_litellm_models_and_with_upstreams_etag( + client: TestClient, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + open_weights: Final = _open_weights("open-weights") + wider_window: Final = {**open_weights, "model_info": {"max_input_tokens": 64000}} + + def catalog_etag(deployment: dict[str, object], upstream_etag: str) -> str | None: + monkeypatch.setattr(proxy_server, "llm_router", Router(model_list=[deployment])) + respx_mock.get(f"{_BASE}/models").respond(json=_UPSTREAM_CATALOG, headers={"etag": upstream_etag}) + return client.get("/backend-api/codex/models?client_version=0.165.0", headers=_HEADERS).headers.get("etag") + + baseline: Final = catalog_etag(open_weights, '"upstream-v1"') + assert baseline == catalog_etag(open_weights, '"upstream-v1"') + assert ( + len({baseline, catalog_etag(wider_window, '"upstream-v1"'), catalog_etag(open_weights, '"upstream-v3"')}) == 3 ) - assert response.status_code == 200 - assert response.headers.get("x-models-etag") == ('"upstream-v2"' if etag_kept else None) + assert baseline not in (None, '"upstream-v1"') + + +_OPENAI_BASE: Final = "https://openai-api.test/v1" +_OPENAI_BLOB: Final = "gAAAAB-openai-reasoning-blob" +_OPENAI_REASONING: Final = {"type": "reasoning", "id": "rs_openai", "summary": [], "encrypted_content": _OPENAI_BLOB} + + +def _openai_reply(request: httpx.Request) -> httpx.Response: + completed: Final = { + "id": "resp_openai", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "gpt-5.6", + "output": [ + _OPENAI_REASONING, + { + "type": "message", + "id": "msg_openai", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "Hi", "annotations": []}], + }, + ], + "usage": {"input_tokens": 3, "output_tokens": 1, "total_tokens": 4}, + } + if json.loads(request.content).get("stream") is not True: + return httpx.Response(200, json=completed) + events: Final = ( + { + "type": "response.created", + "sequence_number": 0, + "response": {**completed, "status": "in_progress", "output": []}, + }, + {"type": "response.output_item.done", "sequence_number": 1, "output_index": 0, "item": _OPENAI_REASONING}, + {"type": "response.completed", "sequence_number": 2, "response": completed}, + ) + return httpx.Response( + 200, + content="".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events).encode(), + headers={"content-type": "text/event-stream"}, + ) + + +def _reasoning_item_of(response: httpx.Response, stream: bool) -> dict[str, object]: + if not stream: + return response.json()["output"][0] + payloads: Final = ( + json.loads(line.removeprefix(b"data: ")) + for line in response.content.splitlines() + if line.startswith(b"data: {") + ) + return next(event["item"] for event in payloads if event["type"] == "response.output_item.done") + + +@pytest.mark.parametrize("stream", [False, True]) +def test_openai_reasoning_from_the_native_route_is_dropped_for_chatgpt_and_unwrapped_for_litellm( + client: TestClient, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, stream: bool +) -> None: + monkeypatch.setattr( + proxy_server, + "llm_router", + Router( + model_list=[ + { + "model_name": "api-model", + "litellm_params": { + "model": "openai/gpt-5.6", + "api_base": _OPENAI_BASE, + "api_key": "test-openai-key", + }, + } + ] + ), + ) + _account_catalog(respx_mock) + provider: Final = respx_mock.post(f"{_OPENAI_BASE}/responses").mock(side_effect=_openai_reply) + chatgpt: Final = _chatgpt_responses(respx_mock) + turn: Final = {"model": "api-model", "input": [_USER_TURN], "stream": stream, "store": False} + first: Final = client.post("/backend-api/codex/responses", json=turn, headers=_ACCOUNT_HEADERS) + assert first.status_code == 200, first.text + reasoning: Final = _reasoning_item_of(first, stream) + model_id, blob = ResponsesAPIRequestUtils.unwrap_encrypted_content_with_model_id( + str(reasoning["encrypted_content"]) + ) + assert (model_id is not None, blob) == (True, _OPENAI_BLOB) + history: Final = [_USER_TURN, reasoning, _ASSISTANT_TURN, _USER_TURN] + to_chatgpt: Final = client.post( + "/backend-api/codex/responses", + json={**turn, "model": "subscription-model", "input": history, "stream": True}, + headers=_ACCOUNT_HEADERS, + ) + back_to_litellm: Final = client.post( + "/backend-api/codex/responses", json={**turn, "input": history}, headers=_ACCOUNT_HEADERS + ) + v1: Final = client.post( + "/v1/responses", json={**turn, "stream": False}, headers={"authorization": f"Bearer {MASTER_KEY}"} + ) + assert to_chatgpt.status_code == back_to_litellm.status_code == v1.status_code == 200 + assert json.loads(chatgpt.calls.last.request.content)["input"] == [_USER_TURN, _ASSISTANT_TURN, _USER_TURN] + replayed: Final = json.loads(provider.calls[1].request.content)["input"][1] + assert (replayed["id"], replayed["encrypted_content"]) == ("rs_openai", _OPENAI_BLOB) + assert v1.json()["output"][0]["encrypted_content"] == _OPENAI_BLOB diff --git a/tests/unit/responses/litellm_completion_transformation/test_reasoning_items.py b/tests/unit/responses/litellm_completion_transformation/test_reasoning_items.py index 85ec4be2636..580ee672500 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_reasoning_items.py +++ b/tests/unit/responses/litellm_completion_transformation/test_reasoning_items.py @@ -10,7 +10,9 @@ from litellm.responses.litellm_completion_transformation.reasoning_items import is_other_provider_reasoning_item, mint_reasoning_item_id, ) +from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig from litellm.responses.utils import ResponsesAPIRequestUtils +from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams A_PROVIDER_OWNED_REASONING_ITEM_ID = "rs_08d3a89dbb92277a006abf04f4266087d0b4eedacd7848f306" A_PROVIDER_OWNED_ENCRYPTED_BLOB = "gAAAAABo-opaque-provider-blob" @@ -62,7 +64,9 @@ def test_a_reasoning_item_is_litellm_minted_by_its_id_or_by_its_encoded_thinking "encrypted_content": A_PROVIDER_OWNED_ENCRYPTED_BLOB, } ) - assert not is_litellm_minted_reasoning_item({"type": "message", "id": mint_reasoning_item_id(), "role": "assistant"}) + assert not is_litellm_minted_reasoning_item( + {"type": "message", "id": mint_reasoning_item_id(), "role": "assistant"} + ) assert not is_litellm_minted_reasoning_item("a bare string input") @@ -119,3 +123,23 @@ def test_reasoning_items_another_provider_produced_are_recognized(item): ) def test_reasoning_items_the_provider_can_replay_are_kept(item): assert not is_other_provider_reasoning_item(item) + + +def test_signed_thinking_survives_the_model_id_wrap_on_its_way_back_to_the_chat_completions_bridge(): + wrapped = ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id( + encode_thinking_blocks([SIGNED_BLOCK]), "anthropic-deployment" + ) + messages = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model="anthropic/claude-sonnet-5-5", + input=[ + {"role": "user", "content": "Capital of France?"}, + {"type": "reasoning", "id": mint_reasoning_item_id(), "summary": [], "encrypted_content": wrapped}, + {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "Paris."}]}, + {"role": "user", "content": "And of Italy?"}, + ], + responses_api_request=ResponsesAPIOptionalRequestParams(), + )["messages"] + + assert [message.get("thinking_blocks") for message in messages if message["role"] == "assistant"] == [ + [SIGNED_BLOCK] + ] From 062cfada8bd7cabe0885ed41152bcf77d54f6c4a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 05:18:23 -0700 Subject: [PATCH 05/20] fix(fireworks_ai): move Codex additional_tools items into tools on the Responses API --- .../fireworks_ai/responses/transformation.py | 30 +++++++---- ...t_fireworks_ai_responses_transformation.py | 53 +++++++++++++++++++ 2 files changed, 74 insertions(+), 9 deletions(-) diff --git a/litellm/llms/fireworks_ai/responses/transformation.py b/litellm/llms/fireworks_ai/responses/transformation.py index db234c09930..c6fbb130ed9 100644 --- a/litellm/llms/fireworks_ai/responses/transformation.py +++ b/litellm/llms/fireworks_ai/responses/transformation.py @@ -1,6 +1,6 @@ from collections.abc import Mapping, Sequence from types import MappingProxyType -from typing import TYPE_CHECKING, Final +from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # the base signature leaves the params bag untyped from urllib.parse import unquote import httpx @@ -16,8 +16,9 @@ from litellm.llms.fireworks_ai.common_utils import ( without_caller_user, ) from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.responses.additional_tools import hoist_additional_tools from litellm.secret_managers.main import get_secret_str -from litellm.types.llms.openai import ResponseInputParam +from litellm.types.llms.openai import ALL_RESPONSES_API_TOOL_PARAMS, ResponseInputParam from litellm.types.responses.main import DeleteResponseResult from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders @@ -57,9 +58,11 @@ def _role(item: ResponseInputItemParam) -> str | None: def _developer_item_as_system(item: ResponseInputItemParam) -> ResponseInputItemParam: - if "role" not in item or item["role"] != "developer": - return item - return EasyInputMessageParam(role="system", content=item["content"], type="message") + match item: + case {"role": "developer", "content": content}: + return EasyInputMessageParam(role="system", content=content, type="message") + case _: + return item def _developer_items_as_system(input: str | ResponseInputParam) -> str | ResponseInputParam: @@ -162,19 +165,28 @@ class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig): litellm_params: GenericLiteLLMParams, headers: dict, # mutable-ok: overrides the base class signature ) -> dict: # mutable-ok: overrides the base class signature - instructions_param: Final[object] = response_api_optional_request_params.get("instructions") + optional_params: Final = cast( # cast-ok: the base signature leaves the params bag untyped + "Mapping[str, object]", response_api_optional_request_params + ) + instructions_param: Final = optional_params.get("instructions") validated_input: Final = self._validate_input_param(input) + tools_param: Final = cast( # cast-ok: the base signature leaves the params bag untyped + "Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None", optional_params.get("tools") + ) + hoisted: Final = hoist_additional_tools(validated_input, tools_param) instructions, folded_input = ( - _with_instruction_items_folded(validated_input, instructions_param) + _with_instruction_items_folded(hoisted.input, instructions_param) if isinstance(instructions_param, str | None) - else (instructions_param, _developer_items_as_system(validated_input)) + else (instructions_param, _developer_items_as_system(hoisted.input)) ) instruction_entries: Final = () if instructions is None else (("instructions", instructions),) + tool_entries: Final = (("tools", list(hoisted.tools)),) if hoisted.hoisted else () folded_params: Final = { key: value for key, value in ( - *((key, value) for key, value in response_api_optional_request_params.items() if key != "instructions"), + *((key, value) for key, value in optional_params.items() if key != "instructions"), *instruction_entries, + *tool_entries, ) } request: Final = super().transform_responses_api_request( diff --git a/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py b/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py index 878e0285cd1..258d4bfa9cc 100644 --- a/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py +++ b/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py @@ -352,6 +352,59 @@ def test_transform_request_forwards_non_string_instructions_and_input_untouched( ) +CODEX_CLOCK_NAMESPACE_TOOL: Final = { + "type": "namespace", + "name": "clock", + "description": "Clock tools.", + "tools": [ + { + "type": "function", + "name": "sleep", + "description": "Sleep for some seconds.", + "parameters": {"type": "object", "properties": {"seconds": {"type": "number"}}, "required": ["seconds"]}, + } + ], +} +CODEX_ADDITIONAL_TOOLS_ITEM: Final = { + "type": "additional_tools", + "role": "developer", + "tools": [CODEX_CLOCK_NAMESPACE_TOOL], +} + + +@pytest.mark.parametrize("instructions", [None, "Answer with exactly one word."], ids=["no-instructions", "instructions"]) +def test_transform_request_moves_codex_additional_tools_item_into_tools(instructions: str | None) -> None: + user_item: Final = {"role": "user", "content": "Sleep for two seconds."} + request: Final = FireworksAIResponsesAPIConfig().transform_responses_api_request( + model="accounts/fireworks/models/glm-5p3", + input=cast(ResponseInputParam, [CODEX_ADDITIONAL_TOOLS_ITEM, user_item]), + response_api_optional_request_params={} if instructions is None else {"instructions": instructions}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert tuple(request["tools"]) == (CODEX_CLOCK_NAMESPACE_TOOL,) + assert tuple(request["input"]) == (user_item,) + assert request.get("instructions") == instructions + + +def test_transform_request_merges_codex_additional_tools_into_existing_tools_once() -> None: + weather_tool: Final = { + "type": "function", + "name": "get_weather", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + } + user_item: Final = {"role": "user", "content": "Sleep for two seconds, then check the weather in Paris."} + request: Final = FireworksAIResponsesAPIConfig().transform_responses_api_request( + model="accounts/fireworks/models/glm-5p3", + input=cast(ResponseInputParam, [CODEX_ADDITIONAL_TOOLS_ITEM, user_item, CODEX_ADDITIONAL_TOOLS_ITEM]), + response_api_optional_request_params={"tools": [weather_tool]}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert tuple(request["tools"]) == (weather_tool, CODEX_CLOCK_NAMESPACE_TOOL) + assert tuple(request["input"]) == (user_item,) + + def test_responses_call_maps_pydantic_developer_items_and_replays_pydantic_output_items() -> None: client: Final = _mock_http_client(_fireworks_response("accounts/fireworks/models/kimi-k3")) pydantic_input: Final = cast( From e6350d6394eedba1a3daa80cfba77dbf9da4f225 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 07:17:29 -0700 Subject: [PATCH 06/20] fix(proxy): keep Codex native turns working when the key's model listing fails A listing callback that raises no longer turns chatgpt.com's catalog or responses answer into a 500. The catalog keeps upstream's body and ETag, and responses keep upstream's X-Models-Etag. Under drop_params, a `none` reasoning effort is now dropped only when a candidate deployment does not explicitly support it, so an alias to a model like openai/gpt-5.4 keeps it. The account slug list and its unavailable marker are keyed on the account id plus the caller's ChatGPT Authorization header, so a failed fetch with one bearer cannot turn off LiteLLM routing for another caller on the same account. The auth step checks the Codex route before importing the Codex module. --- .../chatgpt_subscription_passthrough.md | 2 +- litellm/proxy/auth/user_api_key_auth.py | 11 +-- .../codex_passthrough_endpoints.py | 61 ++++++++++----- .../test_codex_passthrough_endpoints.py | 74 ++++++++++++++++++- 4 files changed, 120 insertions(+), 28 deletions(-) diff --git a/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md b/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md index 034b58e8642..538a5f8a404 100644 --- a/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md +++ b/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md @@ -52,7 +52,7 @@ For a native Codex model catalog and request format, change Codex's `base_url` t On the native route, Codex's model picker shows OpenAI's catalog first and then the other chat and responses models the virtual key can use. Pick one of those and Codex's requests go through LiteLLM's `/v1/responses` with the key's usual routing, budgets, guardrails, and logging. The proxy removes the ChatGPT bearer and account header from that request, so the other provider and your callbacks never receive them. A name is routed this way when another provider serves it as an exact `model_name`, through a router `model_group_alias`, or as a team's public model name. A ChatGPT model, a name that only a wildcard deployment matches, or a name OpenAI's catalog lists for the caller's ChatGPT account keeps the passthrough, so a model OpenAI adds later still uses the subscription even when a deployment of another provider has the same name. The proxy learns the account's list from the catalog Codex fetches, or asks OpenAI with the caller's login. When it cannot get the list, for example because OpenAI is down, the request times out, or the client is not Codex, every request keeps the passthrough for the next minute, as it did before LiteLLM models were added to the picker. When the key can use many models, the picker lists the ones that fit in Codex's catalog size limit, and the proxy logs a warning with how many it left out. -Codex's picker sends a `none` reasoning effort for a LiteLLM model whose catalog entry lists no effort levels. With `litellm_settings: drop_params: true`, the proxy drops it and the provider applies its own default. Without `drop_params`, the effort reaches the provider unchanged. +Codex's picker sends a `none` reasoning effort for a LiteLLM model whose catalog entry lists no effort levels. With `litellm_settings: drop_params: true`, the proxy drops it for a model that does not support `none`, and the provider applies its own default. A model whose cost map entry or `model_info` sets `supports_none_reasoning_effort: true` keeps it. Without `drop_params`, the effort reaches the provider unchanged. Codex also sends its ChatGPT app tools with every turn, which on a LiteLLM model adds a few hundred thousand prompt tokens to each turn. Set `apps = false` under `[features]` in Codex's `config.toml` to turn them off. diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 32d4bfe91de..ff29eca8da3 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -351,16 +351,13 @@ def _resolve_codex_native_chatgpt_model( request: Request | None, route: str, ) -> None: + requested: Final = request_data.get("model") + if route != _CODEX_NATIVE_RESPONSES_ROUTE or request is None or not isinstance(requested, str): + return from litellm.proxy.pass_through_endpoints.codex_passthrough_endpoints import native_chatgpt_model from litellm.proxy.proxy_server import llm_router - requested: Final = request_data.get("model") - if ( - route != _CODEX_NATIVE_RESPONSES_ROUTE - or request is None - or llm_router is None - or not isinstance(requested, str) - ): + if llm_router is None: return target: Final = native_chatgpt_model(requested, llm_router, valid_token) if target is None: diff --git a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py index 4963b324dc7..97fddf2b5ce 100644 --- a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py @@ -13,6 +13,7 @@ from pydantic import TypeAdapter, ValidationError from starlette.types import Message import litellm +from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params from litellm.llms.chatgpt.common_utils import CHATGPT_API_BASE from litellm.llms.custom_httpx.http_handler import ( @@ -39,6 +40,7 @@ from litellm.proxy.response_api_endpoints.endpoints import responses_api from litellm.responses.litellm_completion_transformation.reasoning_items import is_other_provider_reasoning_item from litellm.types.llms.custom_http import httpxSpecialProvider from litellm.types.router import Deployment +from litellm.utils import supports_none_reasoning_effort if TYPE_CHECKING: from litellm.router import Router @@ -52,6 +54,7 @@ _MODELS_ETAG_HEADER: Final = "x-models-etag" _ENCRYPTED_CONTENT_AFFINITY_FLAG: Final = "encrypted_content_affinity_enabled" _LITELLM_RESPONSES_PATH: Final = "/v1/responses" _CODEX_NO_DEFAULT_EFFORT: Final = "none" +_NONE_EFFORT_SUPPORT_FLAG: Final = "supports_none_reasoning_effort" _JSON_OBJECT: Final = TypeAdapter(dict[str, object]) _INPUT_ITEMS: Final = TypeAdapter(tuple[object, ...]) _CACHED_SLUGS: Final = TypeAdapter(frozenset[str]) @@ -160,8 +163,12 @@ def _may_call_litellm_responses(user_api_key_dict: UserAPIKeyAuth) -> bool: return True -def _account_slugs_cache_key(account_id: str) -> str: - return f"{_ACCOUNT_SLUGS_CACHE_PREFIX}{hashlib.sha256(account_id.encode()).hexdigest()}" +def _account_slugs_cache_key(headers: Mapping[str, str]) -> str | None: + account_id: Final = headers.get("chatgpt-account-id") + if not account_id: + return None + caller: Final = json.dumps([account_id, headers.get("authorization", "")]) + return f"{_ACCOUNT_SLUGS_CACHE_PREFIX}{hashlib.sha256(caller.encode()).hexdigest()}" def _cached_slugs(cached: object) -> frozenset[str] | None: @@ -173,17 +180,11 @@ def _cached_slugs(cached: object) -> frozenset[str] | None: return None -async def _remember_account_slugs(account_id: str, slugs: frozenset[str] | None, cache: _SharedCache) -> None: +async def _remember_account_slugs(cache_key: str, slugs: frozenset[str] | None, cache: _SharedCache) -> None: if slugs is None: - await cache.async_set_cache( - _account_slugs_cache_key(account_id), - _ACCOUNT_SLUGS_UNAVAILABLE, - ttl=_ACCOUNT_SLUGS_UNAVAILABLE_TTL_SECONDS, - ) + await cache.async_set_cache(cache_key, _ACCOUNT_SLUGS_UNAVAILABLE, ttl=_ACCOUNT_SLUGS_UNAVAILABLE_TTL_SECONDS) return - await cache.async_set_cache( - _account_slugs_cache_key(account_id), tuple(sorted(slugs)), ttl=_SHARED_CACHE_TTL_SECONDS - ) + await cache.async_set_cache(cache_key, tuple(sorted(slugs)), ttl=_SHARED_CACHE_TTL_SECONDS) def _codex_client_version(headers: Mapping[str, str]) -> str | None: @@ -210,10 +211,10 @@ async def _fetched_account_slugs(headers: Mapping[str, str], client_version: str async def _account_slugs(headers: Mapping[str, str], cache: _SharedCache) -> frozenset[str] | None: - account_id: Final = headers.get("chatgpt-account-id") - if not account_id: + cache_key: Final = _account_slugs_cache_key(headers) + if cache_key is None: return None - cached_value: Final = await cache.async_get_cache(_account_slugs_cache_key(account_id)) + cached_value: Final = await cache.async_get_cache(cache_key) if cached_value == _ACCOUNT_SLUGS_UNAVAILABLE: return None cached: Final = _cached_slugs(cached_value) @@ -223,7 +224,7 @@ async def _account_slugs(headers: Mapping[str, str], cache: _SharedCache) -> fro if client_version is None: return None fetched: Final = await _fetched_account_slugs(headers, client_version) - await _remember_account_slugs(account_id, fetched, cache) + await _remember_account_slugs(cache_key, fetched, cache) return fetched @@ -284,6 +285,20 @@ def _drops_params(body: Mapping[str, object], model: str, llm_router: "Router", ) +def _deployment_supports_no_effort(deployment: Deployment) -> bool: + configured: Final = _json_object(deployment.model_info.model_extra).get(_NONE_EFFORT_SUPPORT_FLAG) + if isinstance(configured, bool): + return configured + return supports_none_reasoning_effort( + model=deployment.litellm_params.model, custom_llm_provider=deployment.litellm_params.custom_llm_provider + ) + + +def _supports_no_effort(model: str, llm_router: "Router", team_id: str | None) -> bool: + deployments: Final = _candidate_deployments(model, llm_router, team_id) + return bool(deployments) and all(_deployment_supports_no_effort(deployment) for deployment in deployments) + + def _with_provider_default_reasoning( body: Mapping[str, object], llm_router: "Router | None", team_id: str | None ) -> Mapping[str, object]: @@ -298,6 +313,7 @@ def _with_provider_default_reasoning( or llm_router is None or not _drops_params(body, model, llm_router, team_id) or codex_entry_names_default_reasoning_level(model, llm_router, team_id) + or _supports_no_effort(model, llm_router, team_id) ): return body return {**body, "reasoning": {key: value for key, value in reasoning.items() if key != "effort"}} @@ -358,7 +374,14 @@ async def _litellm_catalog_rows( if llm_router is None or not _may_call_litellm_responses(user_api_key_dict): return () - rows: Final = await codex_catalog_rows_for_key(user_api_key_dict) + try: + rows: Final = await codex_catalog_rows_for_key(user_api_key_dict) + except Exception as error: # noqa: BLE001 # a failed listing must never fail the ChatGPT answer it extends + verbose_proxy_logger.warning( + "Codex catalog keeps only ChatGPT's models: listing the key's LiteLLM models raised %s", + type(error).__name__, + ) + return () return tuple(row for row in rows if _routes_to_litellm(row.id, llm_router, user_api_key_dict.team_id)) @@ -431,9 +454,9 @@ async def _catalog( slugs: Final = upstream_codex_slugs(upstream_body) if slugs is None: return upstream - account_id: Final = headers.get("chatgpt-account-id") - if account_id: - await _remember_account_slugs(account_id, slugs, cache) + account_slugs_key: Final = _account_slugs_cache_key(headers) + if account_slugs_key is not None: + await _remember_account_slugs(account_slugs_key, slugs, cache) rows: Final = await _litellm_catalog_rows(user_api_key_dict, llm_router) upstream_etag: Final = upstream.headers.get(_ETAG_HEADER) etag: Final = ( diff --git a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py index f9e8cca4198..02c6229b9df 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py @@ -1,7 +1,7 @@ import asyncio import base64 import json -from collections.abc import Callable, Iterator +from collections.abc import Callable, Iterator, Sequence from datetime import datetime from typing import Final @@ -451,6 +451,9 @@ def test_litellm_model_on_native_route_keeps_chatgpt_credentials_out_of_hook_raw pytest.param("open-weights", "none", "request", None, id="none under the request's drop_params"), pytest.param("open-weights", "high", "global", "high", id="effort set for a model without a default effort"), pytest.param("coding-model", "none", "global", "none", id="none on a model whose entry names a default effort"), + pytest.param("api-model", "none", "global", "none", id="none on a model that supports it"), + pytest.param("glm-model", "none", "global", None, id="none on a model that does not support it"), + pytest.param("declared-model", "none", "global", "none", id="none on a deployment whose model_info allows it"), ], ) def test_litellm_model_on_native_route_gets_the_reasoning_effort_codex_m_would_send( @@ -474,6 +477,12 @@ def test_litellm_model_on_native_route_gets_the_reasoning_effort_codex_m_would_s "litellm_params": {**open_weights["litellm_params"], "drop_params": drop_params == "deployment"}, }, _open_weights("coding-model", upstream="gpt-5.5"), + _open_weights("glm-model", upstream="glm-5p3"), + {**_open_weights("declared-model"), "model_info": {"supports_none_reasoning_effort": True}}, + { + "model_name": "api-model", + "litellm_params": {"model": "openai/gpt-5.4", "api_base": _PROVIDER_BASE, "api_key": _PROVIDER_KEY}, + }, ] ), ) @@ -504,6 +513,7 @@ def test_litellm_model_on_native_route_gets_the_reasoning_effort_codex_m_would_s assert native.status_code == v1.status_code == 200 native_request, v1_request = (json.loads(call.request.content) for call in provider.calls) assert native_request == v1_request + assert native_request["reasoning"].get("effort") == expected @pytest.mark.parametrize( @@ -824,6 +834,35 @@ def test_a_failed_account_catalog_read_keeps_the_next_turns_on_the_subscription_ assert catalog.call_count == 1 +def test_a_failed_account_catalog_read_with_another_login_leaves_the_accounts_own_list_reachable( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router +) -> None: + own_login: Final = f"Bearer {_CHATGPT_JWT}" + catalog: Final = respx_mock.get(f"{_BASE}/models").mock( + side_effect=lambda request: ( + httpx.Response(200, json=_UPSTREAM_CATALOG) + if request.headers["authorization"] == own_login + else httpx.Response(401) + ) + ) + provider: Final = _open_weights_provider(respx_mock) + chatgpt: Final = _chatgpt_responses(respx_mock) + turn: Final = {"model": "open-weights", "input": "Hello", "stream": True} + other_login: Final = client.post( + "/backend-api/codex/responses", + json=turn, + headers={**_ACCOUNT_HEADERS, "authorization": "Bearer test-someone-elses-login"}, + ) + own: Final = client.post("/backend-api/codex/responses", json=turn, headers=_ACCOUNT_HEADERS) + assert other_login.status_code == own.status_code == 200 + assert chatgpt.call_count == 1 + assert provider.call_count == 1 + assert [call.request.headers["authorization"] for call in catalog.calls] == [ + "Bearer test-someone-elses-login", + own_login, + ] + + def test_account_catalog_lookup_sends_the_callers_chatgpt_login_and_codex_version( client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router ) -> None: @@ -911,6 +950,39 @@ def test_native_models_etag_changes_with_the_keys_litellm_models_and_with_upstre assert baseline not in (None, '"upstream-v1"') +class FailingModelListing(CustomLogger): + async def async_filter_listed_models( + self, user_api_key_dict: UserAPIKeyAuth, model_names: Sequence[str] + ) -> Sequence[str]: + raise RuntimeError("model listing backend unavailable") + + +def test_native_catalog_relays_chatgpts_answer_when_listing_the_keys_models_fails( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "callbacks", [FailingModelListing()]) + respx_mock.get(f"{_BASE}/models").respond(json=_UPSTREAM_CATALOG, headers={"etag": '"upstream-v1"'}) + response: Final = client.get("/backend-api/codex/models?client_version=0.165.0", headers=_HEADERS) + assert response.status_code == 200 + assert response.json() == _UPSTREAM_CATALOG + assert response.headers["etag"] == '"upstream-v1"' + + +def test_native_responses_keep_chatgpts_models_etag_when_listing_the_keys_models_fails( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "callbacks", [FailingModelListing()]) + chatgpt: Final = _chatgpt_responses(respx_mock) + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": "gpt-unlisted", "input": "Hello", "stream": True}, + headers=_HEADERS, + ) + assert response.status_code == 200 + assert chatgpt.called + assert response.headers["x-models-etag"] == _UPSTREAM_MODELS_ETAG + + _OPENAI_BASE: Final = "https://openai-api.test/v1" _OPENAI_BLOB: Final = "gAAAAB-openai-reasoning-blob" _OPENAI_REASONING: Final = {"type": "reasoning", "id": "rs_openai", "summary": [], "encrypted_content": _OPENAI_BLOB} From 6ac4cfe9cf7317895510bc5c680fee022768351b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 08:51:30 -0700 Subject: [PATCH 07/20] fix(proxy): keep Codex model switches working across LiteLLM providers On the Codex native route, a LiteLLM model now gets only the encrypted reasoning every deployment it can route to is able to decrypt, so ChatGPT or another deployment's blobs no longer make xAI answer 400 Codex additional_tools items are hoisted into tools for every provider except the OpenAI API, and namespace tools become flat tools named mcp__docs__search, the shape xAI accepts A JWT caller's models ETag is cached on a hash of its LiteLLM credential, so ChatGPT turns stop rebuilding the catalog rows --- .../codex_passthrough_endpoints.py | 139 ++++++++-- litellm/responses/additional_tools.py | 42 ++- .../count_request.py | 19 +- .../encrypted_content_affinity_check.py | 4 +- .../test_codex_passthrough_endpoints.py | 262 +++++++++++++++++- 5 files changed, 416 insertions(+), 50 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py index 97fddf2b5ce..8ef4c64dd92 100644 --- a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py @@ -14,7 +14,9 @@ from starlette.types import Message import litellm from litellm._logging import verbose_proxy_logger +from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.llms.chatgpt.common_utils import CHATGPT_API_BASE from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared client factory has untyped params @@ -37,10 +39,13 @@ from litellm.proxy.common_utils.http_parsing_utils import ( ) from litellm.proxy.pass_through_endpoints.pass_through_endpoints import pass_through_request from litellm.proxy.response_api_endpoints.endpoints import responses_api +from litellm.responses.additional_tools import with_additional_tools_hoisted, with_namespace_members_as_tools from litellm.responses.litellm_completion_transformation.reasoning_items import is_other_provider_reasoning_item +from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import EncryptedContentAffinityCheck from litellm.types.llms.custom_http import httpxSpecialProvider from litellm.types.router import Deployment -from litellm.utils import supports_none_reasoning_effort +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager, supports_none_reasoning_effort if TYPE_CHECKING: from litellm.router import Router @@ -70,6 +75,7 @@ _ACCOUNT_SLUGS_UNAVAILABLE_TTL_SECONDS: Final = 60 _ACCOUNT_SLUGS_CACHE_PREFIX: Final = "codex_account_slugs:" _MODELS_ETAG_CACHE_PREFIX: Final = "codex_models_etag:" _MODELS_ETAG_TTL_SECONDS: Final = 300 +_OPENAI_API_PROVIDERS: Final = frozenset({LlmProviders.OPENAI.value, LlmProviders.AZURE.value}) _HEADERS: Final = frozenset( { "authorization", @@ -326,6 +332,80 @@ def _with_wrapped_reasoning(body: Mapping[str, object]) -> Mapping[str, object]: } +def _router_pins_encrypted_reasoning(model: str, llm_router: "Router") -> bool: + return any( + isinstance(callback, EncryptedContentAffinityCheck) and callback.is_enabled_for_model_group(model) + for callback in llm_router.optional_callbacks or () + ) + + +def _with_reasoning_the_targets_can_decrypt( + body: Mapping[str, object], model: str, llm_router: "Router", deployments: tuple[Deployment, ...] +) -> Mapping[str, object]: + try: + items: Final = _INPUT_ITEMS.validate_python(body.get("input")) + except ValidationError: + return body + targets: Final = tuple(deployment.model_dump(exclude_none=True) for deployment in deployments) + target_groups: Final = ( + (targets,) if _router_pins_encrypted_reasoning(model, llm_router) else tuple((target,) for target in targets) + ) + kept: Final = list(items) + for group in target_groups: + EncryptedContentAffinityCheck.strip_reasoning_the_targets_cannot_decrypt( + llm_router, kept, None, group, unmarked_origin=None + ) + return {**body, "input": kept} + + +def _model_and_provider(deployment: Deployment) -> tuple[str, str] | None: + try: + model, provider, _, _ = get_llm_provider( + model=deployment.litellm_params.model, custom_llm_provider=deployment.litellm_params.custom_llm_provider + ) + except BadRequestError: + return None + return model, provider + + +def _is_openai_api(deployment: Deployment) -> bool: + resolved: Final = _model_and_provider(deployment) + return resolved is not None and resolved[1] in _OPENAI_API_PROVIDERS + + +def _bridges_to_chat_completions(deployment: Deployment) -> bool: + resolved: Final = _model_and_provider(deployment) + return ( + resolved is not None + and ProviderConfigManager.get_provider_responses_api_config( + provider=resolved[1], model=resolved[0], api_base=deployment.litellm_params.api_base + ) + is None + ) + + +def _with_tools_the_targets_accept( + body: Mapping[str, object], deployments: tuple[Deployment, ...] +) -> Mapping[str, object]: + if all(_is_openai_api(deployment) for deployment in deployments): + return body + hoisted: Final = with_additional_tools_hoisted(body) + if all(_bridges_to_chat_completions(deployment) for deployment in deployments): + return hoisted + return with_namespace_members_as_tools(hoisted) + + +def _litellm_body(body: Mapping[str, object], llm_router: "Router | None", team_id: str | None) -> Mapping[str, object]: + wrapped: Final = _with_wrapped_reasoning(_with_provider_default_reasoning(body, llm_router, team_id)) + model: Final = body.get("model") + if not isinstance(model, str) or llm_router is None: + return wrapped + deployments: Final = _candidate_deployments(model, llm_router, team_id) + return _with_tools_the_targets_accept( + _with_reasoning_the_targets_can_decrypt(wrapped, model, llm_router, deployments), deployments + ) + + def _is_unpersisted_reasoning_reference(item: object) -> bool: try: fields: Final = _JSON_OBJECT.validate_python(item) @@ -385,42 +465,45 @@ async def _litellm_catalog_rows( return tuple(row for row in rows if _routes_to_litellm(row.id, llm_router, user_api_key_dict.team_id)) -def _models_etag_cache_key(token: str, upstream_etag: str) -> str: - return f"{_MODELS_ETAG_CACHE_PREFIX}{hashlib.sha256(json.dumps([token, upstream_etag]).encode()).hexdigest()}" +def _models_etag_caller(user_api_key_dict: UserAPIKeyAuth, gateway_token: str) -> str: + return user_api_key_dict.token or hashlib.sha256(gateway_token.encode()).hexdigest() + + +def _models_etag_cache_key(caller: str, upstream_etag: str) -> str: + return f"{_MODELS_ETAG_CACHE_PREFIX}{hashlib.sha256(json.dumps([caller, upstream_etag]).encode()).hexdigest()}" async def _remembered_models_etag( - user_api_key_dict: UserAPIKeyAuth, upstream_etag: str, rows: tuple[CodexCatalogRow, ...], cache: _SharedCache + caller: str, upstream_etag: str, rows: tuple[CodexCatalogRow, ...], cache: _SharedCache ) -> str: etag: Final = codex_catalog_etag(upstream_etag, rows) if rows else upstream_etag - if user_api_key_dict.token is not None: - await cache.async_set_cache( - _models_etag_cache_key(user_api_key_dict.token, upstream_etag), - {"etag": etag}, - ttl=_MODELS_ETAG_TTL_SECONDS, - ) + await cache.async_set_cache( + _models_etag_cache_key(caller, upstream_etag), {"etag": etag}, ttl=_MODELS_ETAG_TTL_SECONDS + ) return etag -async def _cached_models_etag(token: str | None, upstream_etag: str, cache: _SharedCache) -> str | None: - if token is None: - return None +async def _cached_models_etag(caller: str, upstream_etag: str, cache: _SharedCache) -> str | None: try: return _CACHED_MODELS_ETAG.validate_python( - await cache.async_get_cache(_models_etag_cache_key(token, upstream_etag)) + await cache.async_get_cache(_models_etag_cache_key(caller, upstream_etag)) )["etag"] except ValidationError: return None async def _key_models_etag( - upstream_etag: str, user_api_key_dict: UserAPIKeyAuth, llm_router: "Router | None", cache: _SharedCache + upstream_etag: str, + user_api_key_dict: UserAPIKeyAuth, + caller: str, + llm_router: "Router | None", + cache: _SharedCache, ) -> str: - cached: Final = await _cached_models_etag(user_api_key_dict.token, upstream_etag, cache) + cached: Final = await _cached_models_etag(caller, upstream_etag, cache) if cached is not None: return cached rows: Final = await _litellm_catalog_rows(user_api_key_dict, llm_router) - return await _remembered_models_etag(user_api_key_dict, upstream_etag, rows, cache) + return await _remembered_models_etag(caller, upstream_etag, rows, cache) def _with_etag(response: Response, content: bytes | str, etag: str | None) -> Response: @@ -436,7 +519,11 @@ def _with_etag(response: Response, content: bytes | str, etag: str | None) -> Re async def _catalog( - request: Request, user_api_key_dict: UserAPIKeyAuth, headers: Mapping[str, str], cache: _SharedCache + request: Request, + user_api_key_dict: UserAPIKeyAuth, + caller: str, + headers: Mapping[str, str], + cache: _SharedCache, ) -> Response: from litellm.proxy.proxy_server import llm_router @@ -459,9 +546,7 @@ async def _catalog( await _remember_account_slugs(account_slugs_key, slugs, cache) rows: Final = await _litellm_catalog_rows(user_api_key_dict, llm_router) upstream_etag: Final = upstream.headers.get(_ETAG_HEADER) - etag: Final = ( - None if upstream_etag is None else await _remembered_models_etag(user_api_key_dict, upstream_etag, rows, cache) - ) + etag: Final = None if upstream_etag is None else await _remembered_models_etag(caller, upstream_etag, rows, cache) if not rows: return upstream merged: Final = merged_codex_catalog(upstream_body, rows) @@ -472,6 +557,7 @@ async def _responses( request: Request, fastapi_response: Response, user_api_key_dict: UserAPIKeyAuth, + caller: str, headers: Mapping[str, str], cache: _SharedCache, ) -> object: @@ -479,9 +565,7 @@ async def _responses( body: Final = _JSON_OBJECT.validate_python(await read_request_body(request)) if await _serves_through_litellm(body.get("model"), llm_router, user_api_key_dict, headers, cache): - litellm_body: Final = _with_wrapped_reasoning( - _with_provider_default_reasoning(body, llm_router, user_api_key_dict.team_id) - ) + litellm_body: Final = _litellm_body(body, llm_router, user_api_key_dict.team_id) return await responses_api( _as_litellm_responses_request(request, json.dumps(litellm_body).encode(), litellm_body), fastapi_response, @@ -500,7 +584,7 @@ async def _responses( upstream_models_etag: Final = upstream.headers.get(_MODELS_ETAG_HEADER) if upstream_models_etag is not None: upstream.headers[_MODELS_ETAG_HEADER] = await _key_models_etag( - upstream_models_etag, user_api_key_dict, llm_router, cache + upstream_models_etag, user_api_key_dict, caller, llm_router, cache ) return upstream @@ -557,6 +641,7 @@ async def codex_passthrough_route( raise HTTPException(status_code=401, detail="ChatGPT and LiteLLM credentials must be separate") headers: Final = MappingProxyType({name: value for name, value in request.headers.items() if name in _HEADERS}) + caller: Final = _models_etag_caller(user_api_key_dict, gateway_token) if endpoint == "models": - return await _catalog(request, user_api_key_dict, headers, cache) - return await _responses(request, fastapi_response, user_api_key_dict, headers, cache) + return await _catalog(request, user_api_key_dict, caller, headers, cache) + return await _responses(request, fastapi_response, user_api_key_dict, caller, headers, cache) diff --git a/litellm/responses/additional_tools.py b/litellm/responses/additional_tools.py index ce530447c0d..03972fbd441 100644 --- a/litellm/responses/additional_tools.py +++ b/litellm/responses/additional_tools.py @@ -1,17 +1,20 @@ import json from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass -from itertools import groupby +from itertools import chain, groupby from types import MappingProxyType from typing import Final, cast # noqa: TID251 # validating the openai tool union strips vendor keys from raw tools -from pydantic import ValidationError +from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_logger from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import ALL_RESPONSES_API_TOOL_PARAMS, ResponseInputParam ADDITIONAL_TOOLS_INPUT_ITEM_TYPE: Final = "additional_tools" +_NAMESPACE_TOOL_TYPE: Final = "namespace" +_QUALIFIED_NAME_SEPARATOR: Final = "__" +_TOOLS: Final = TypeAdapter(tuple[object, ...]) class _InputItemType(LiteLLMBaseModel): @@ -61,7 +64,7 @@ def _members_of_copies(copies: Sequence[Mapping[str, object]]) -> Iterator[Mappi def _merged(copies: Sequence[Mapping[str, object]]) -> Mapping[str, object]: newest: Final = copies[-1] - if newest.get("type") != "namespace" or len(copies) == 1: + if newest.get("type") != _NAMESPACE_TOOL_TYPE or len(copies) == 1: return newest members: Final = tuple(_members_of_copies(copies)) return {**newest, "tools": list(_unique_tools(members))} if members else newest @@ -108,3 +111,36 @@ def hoist_additional_tools( for tool in _unique_tools((*existing, *hoisted)) ) return HoistedAdditionalTools(input=remaining_input, tools=unique, hoisted=hoisted) + + +def with_additional_tools_hoisted(request: Mapping[str, object]) -> Mapping[str, object]: + request_input: Final = request.get("input") + if not isinstance(request_input, list): + return request + input_items: Final = cast(ResponseInputParam, request_input) # cast-ok: forwarded as sent, as inference does + top_level_tools: Final = cast( # cast-ok: tools are forwarded as sent, as inference does + "tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]", _TOOLS.validate_python(request.get("tools") or ()) + ) + hoisted: Final = hoist_additional_tools(input_items, top_level_tools) + if not hoisted.hoisted: + return request + return {**request, "input": hoisted.input, "tools": list(hoisted.tools)} + + +def _qualified_member(namespace: str, member: Mapping[str, object]) -> Mapping[str, object]: + prefix: Final = f"{namespace.removesuffix(_QUALIFIED_NAME_SEPARATOR)}{_QUALIFIED_NAME_SEPARATOR}" + return {**member, "name": f"{prefix}{member.get('name')}"} + + +def _members_as_tools(tool: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: + if tool.get("type") != _NAMESPACE_TOOL_TYPE: + return (tool,) + namespace: Final = str(tool.get("name") or "") + return tuple(_qualified_member(namespace, member) for member in _tool_list_of(tool)) + + +def with_namespace_members_as_tools(request: Mapping[str, object]) -> Mapping[str, object]: + tools: Final = _tool_list_of(request) + if not any(tool.get("type") == _NAMESPACE_TOOL_TYPE for tool in tools): + return request + return {**request, "tools": list(chain.from_iterable(map(_members_as_tools, tools)))} diff --git a/litellm/responses/litellm_completion_transformation/count_request.py b/litellm/responses/litellm_completion_transformation/count_request.py index 8e8e4066bb8..2206d9127c5 100644 --- a/litellm/responses/litellm_completion_transformation/count_request.py +++ b/litellm/responses/litellm_completion_transformation/count_request.py @@ -12,7 +12,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( ) from litellm.litellm_core_utils.tool_fields import DropPolicy from litellm.llms.base_llm.files.batch_records import responses_batch_body_to_chat_body -from litellm.responses.additional_tools import hoist_additional_tools +from litellm.responses.additional_tools import with_additional_tools_hoisted from litellm.responses.file_search.emulated_handler import replace_file_search_tools from litellm.responses.litellm_completion_transformation.compaction_items import holds_compaction_summary from litellm.responses.litellm_completion_transformation.transformation import ( @@ -20,7 +20,6 @@ from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) from litellm.responses.main import previous_response_client_ids -from litellm.types.llms.openai import ALL_RESPONSES_API_TOOL_PARAMS, ResponseInputParam from litellm.utils import ProviderConfigManager _COUNTED_CHAT_PARAMS: Final = ( @@ -114,20 +113,6 @@ def with_provider_file_ids( } -def _with_hoisted_additional_tools(responses_request: Mapping[str, object]) -> Mapping[str, object]: - responses_input: Final = responses_request.get("input") - if not isinstance(responses_input, list): - return responses_request - input_items: Final = cast(ResponseInputParam, responses_input) # cast-ok: forwarded as sent, as inference does - top_level_tools: Final = cast( # cast-ok: tools are forwarded as sent, as inference does - "tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]", _ITEMS.validate_python(responses_request.get("tools") or ()) - ) - hoisted: Final = hoist_additional_tools(input_items, top_level_tools) - if not hoisted.hoisted: - return responses_request - return {**responses_request, "input": hoisted.input, "tools": list(hoisted.tools)} - - def _with_emulated_file_search(responses_request: Mapping[str, object]) -> Mapping[str, object]: tools: Final = responses_request.get("tools") if tools is None: @@ -196,7 +181,7 @@ async def chat_count_request( chat_body: Final = await _with_previous_turns( responses_batch_body_to_chat_body( _with_compaction_as_summary_text( - _with_emulated_file_search(_with_hoisted_additional_tools(responses_request)) + _with_emulated_file_search(with_additional_tools_hoisted(responses_request)) ), custom_llm_provider, policy.drop_params, diff --git a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py index 806c3db7601..86aa7663c49 100644 --- a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py @@ -98,7 +98,7 @@ class EncryptedContentAffinityCheck(CustomLogger): return any("encrypted_content_affinity" in checks for checks in model_group_affinity_config.values()) - def _is_enabled_for_model_group(self, model_group: str) -> bool: + def is_enabled_for_model_group(self, model_group: str) -> bool: group_checks: Final = self.model_group_affinity_config.get(model_group) return self.enable_global_affinity or ( group_checks is not None and "encrypted_content_affinity" in group_checks @@ -352,7 +352,7 @@ class EncryptedContentAffinityCheck(CustomLogger): """ request_kwargs = request_kwargs or {} typed_healthy_deployments: Final = cast(list[dict[str, object]], healthy_deployments) - if not self._is_enabled_for_model_group(model): + if not self.is_enabled_for_model_group(model): return typed_healthy_deployments typed_request_kwargs: Final = _REQUEST_KWARGS_ADAPTER.validate_python(request_kwargs) target_order: Final = typed_request_kwargs.get("_target_order") diff --git a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py index 02c6229b9df..07dcc2859d6 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py @@ -16,7 +16,8 @@ from litellm.caching.dual_cache import DualCache from litellm.integrations.custom_logger import CustomLogger from litellm.llms.chatgpt.common_utils import OPENAI_AUTH_CLAIM_KEY from litellm.proxy import proxy_server -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.codex_model_catalog import bundled_codex_models from litellm.proxy.pass_through_endpoints.codex_passthrough_endpoints import codex_shared_cache from litellm.proxy.utils import hash_token @@ -1084,3 +1085,262 @@ def test_openai_reasoning_from_the_native_route_is_dropped_for_chatgpt_and_unwra replayed: Final = json.loads(provider.calls[1].request.content)["input"][1] assert (replayed["id"], replayed["encrypted_content"]) == ("rs_openai", _OPENAI_BLOB) assert v1.json()["output"][0]["encrypted_content"] == _OPENAI_BLOB + + +_XAI_BASE: Final = "https://xai-api.test/v1" +_LUNA_ID: Final = "oai-luna-deployment" +_LUNA_BLOB: Final = "gAAAAB-luna-reasoning-blob" +_LUNA_REASONING: Final = { + "type": "reasoning", + "summary": [], + "encrypted_content": ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id(_LUNA_BLOB, _LUNA_ID), +} +_CHATGPT_REASONING: Final = {"type": "reasoning", "summary": [], "encrypted_content": "gAAAAB-chatgpt-reasoning-blob"} + + +@pytest.fixture +def xai_and_openai_router(monkeypatch: pytest.MonkeyPatch) -> Router: + router: Final = Router( + model_list=[ + { + "model_name": "grok-4.7", + "litellm_params": {"model": "xai/grok-4.7", "api_base": _XAI_BASE, "api_key": "test-xai-key"}, + }, + { + "model_name": "oai-luna", + "litellm_params": { + "model": "openai/gpt-6-luna", + "api_base": _OPENAI_BASE, + "api_key": "test-openai-key", + }, + "model_info": {"id": _LUNA_ID}, + }, + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + return router + + +@pytest.mark.parametrize( + ("model", "replayed", "expected"), + [ + pytest.param("grok-4.7", _CHATGPT_REASONING, None, id="chatgpt reasoning to xai is dropped"), + pytest.param("grok-4.7", _LUNA_REASONING, None, id="another deployment's reasoning to xai is dropped"), + pytest.param("oai-luna", _LUNA_REASONING, _LUNA_BLOB, id="the minting deployment's reasoning is kept"), + ], +) +def test_litellm_model_on_native_route_gets_only_the_reasoning_it_can_decrypt( + client: TestClient, + respx_mock: respx.MockRouter, + xai_and_openai_router: Router, + model: str, + replayed: dict[str, object], + expected: str | None, +) -> None: + _account_catalog(respx_mock) + xai: Final = respx_mock.post(f"{_XAI_BASE}/responses").mock(side_effect=_provider_reply) + luna: Final = respx_mock.post(f"{_OPENAI_BASE}/responses").mock(side_effect=_provider_reply) + response: Final = client.post( + "/backend-api/codex/responses", + json={ + "model": model, + "input": [_USER_TURN, replayed, _ASSISTANT_TURN, _USER_TURN], + "stream": False, + "store": False, + }, + headers=_ACCOUNT_HEADERS, + ) + assert response.status_code == 200, response.text + provider: Final = xai if model == "grok-4.7" else luna + sent: Final = json.loads(provider.calls.last.request.content)["input"] + assert [item.get("encrypted_content") for item in sent if item.get("type") == "reasoning"] == ( + [] if expected is None else [expected] + ) + + +def _namespace(name: str, *members: str) -> dict[str, object]: + return { + "type": "namespace", + "name": name, + "description": "", + "tools": [{"type": "function", "name": member, "parameters": {"type": "object"}} for member in members], + } + + +_EXEC_TOOL: Final = {"type": "function", "name": "exec_command", "parameters": {"type": "object"}} +_ADDITIONAL_TOOLS: Final = { + "type": "additional_tools", + "role": "developer", + "tools": [_namespace("collaboration", "followup_task"), _namespace("mcp__docs__", "search")], +} + + +@pytest.mark.parametrize( + ("model", "expected_input", "expected_tool_names"), + [ + pytest.param( + "grok-4.7", + [_USER_TURN], + ["exec_command", "collaboration__followup_task", "mcp__docs__search"], + id="xai gets the item's tools as top-level functions", + ), + pytest.param("oai-luna", [_ADDITIONAL_TOOLS, _USER_TURN], ["exec_command"], id="the openai api keeps the item"), + ], +) +def test_codex_additional_tools_item_reaches_each_provider_in_a_shape_it_accepts( + client: TestClient, + respx_mock: respx.MockRouter, + xai_and_openai_router: Router, + model: str, + expected_input: list[dict[str, object]], + expected_tool_names: list[str], +) -> None: + _account_catalog(respx_mock) + xai: Final = respx_mock.post(f"{_XAI_BASE}/responses").mock(side_effect=_provider_reply) + luna: Final = respx_mock.post(f"{_OPENAI_BASE}/responses").mock(side_effect=_provider_reply) + response: Final = client.post( + "/backend-api/codex/responses", + json={ + "model": model, + "input": [_ADDITIONAL_TOOLS, _USER_TURN], + "tools": [_EXEC_TOOL], + "stream": False, + "store": False, + }, + headers=_ACCOUNT_HEADERS, + ) + assert response.status_code == 200, response.text + sent: Final = json.loads((xai if model == "grok-4.7" else luna).calls.last.request.content) + assert sent["input"] == expected_input + assert [tool["name"] for tool in sent["tools"]] == expected_tool_names + + +_ANTHROPIC_BASE: Final = "https://anthropic-api.test" + + +def test_codex_namespace_tool_called_through_the_chat_bridge_comes_back_with_its_namespace( + client: TestClient, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr( + proxy_server, + "llm_router", + Router( + model_list=[ + { + "model_name": "claude-sonnet-5-5", + "litellm_params": { + "model": "anthropic/claude-sonnet-5-5", + "api_base": _ANTHROPIC_BASE, + "api_key": "test-anthropic-key", + }, + } + ] + ), + ) + _account_catalog(respx_mock) + anthropic: Final = respx_mock.post(f"{_ANTHROPIC_BASE}/v1/messages").respond( + json={ + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [ + {"type": "tool_use", "id": "toolu_test", "name": "mcp__docs____search", "input": {"query": "proxy"}} + ], + "stop_reason": "tool_use", + "usage": {"input_tokens": 3, "output_tokens": 1}, + } + ) + response: Final = client.post( + "/backend-api/codex/responses", + json={ + "model": "claude-sonnet-5-5", + "input": [_ADDITIONAL_TOOLS, _USER_TURN], + "tools": [_EXEC_TOOL], + "stream": False, + "store": False, + }, + headers=_ACCOUNT_HEADERS, + ) + assert response.status_code == 200, response.text + assert anthropic.called + (call,) = (item for item in response.json()["output"] if item["type"] == "function_call") + assert (call["namespace"], call["name"]) == ("mcp__docs__", "search") + + +def test_encrypted_reasoning_stays_only_when_every_deployment_the_router_may_pick_can_decrypt_it( + client: TestClient, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + second_base: Final = "https://second-openai-org.test/v1" + + def replayed_blobs(optional_pre_call_checks: list[str]) -> list[object]: + monkeypatch.setattr( + proxy_server, + "llm_router", + Router( + model_list=[ + { + "model_name": "oai-pair", + "litellm_params": {"model": "openai/gpt-6-luna", "api_base": base, "api_key": "test-key"}, + "model_info": {"id": model_id}, + } + for base, model_id in ((_OPENAI_BASE, _LUNA_ID), (second_base, "second-org-deployment")) + ], + optional_pre_call_checks=optional_pre_call_checks, + ), + ) + providers: Final = tuple( + respx_mock.post(f"{base}/responses").mock(side_effect=_provider_reply) + for base in (_OPENAI_BASE, second_base) + ) + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": "oai-pair", "input": [_USER_TURN, _LUNA_REASONING, _USER_TURN], "store": False}, + headers=_ACCOUNT_HEADERS, + ) + assert response.status_code == 200, response.text + (sent,) = (json.loads(call.request.content)["input"] for provider in providers for call in provider.calls) + return [item.get("encrypted_content") for item in sent if item.get("type") == "reasoning"] + + _account_catalog(respx_mock) + assert replayed_blobs([]) == [] + respx_mock.reset() + _account_catalog(respx_mock) + assert replayed_blobs(["encrypted_content_affinity"]) == [_LUNA_BLOB] + + +class ModelListingCounter(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.listings: Final[list[tuple[str, ...]]] = [] + + async def async_filter_listed_models( + self, user_api_key_dict: UserAPIKeyAuth, model_names: Sequence[str] + ) -> Sequence[str]: + self.listings.append(tuple(model_names)) + return model_names + + +def test_jwt_caller_builds_the_models_etag_once_across_chatgpt_turns( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + counter: Final = ModelListingCounter() + monkeypatch.setattr(litellm, "callbacks", [counter]) + monkeypatch.setitem( + proxy_server.app.dependency_overrides, + user_api_key_auth, + lambda: UserAPIKeyAuth(api_key=None, user_id="jwt-user", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + _chatgpt_responses(respx_mock) + turns: Final = tuple( + client.post( + "/backend-api/codex/responses", + json={"model": "gpt-unlisted", "input": "Hello", "stream": True}, + headers={**_HEADERS, "x-litellm-api-key": "test-jwt-credential"}, + ) + for _ in range(2) + ) + assert [turn.status_code for turn in turns] == [200, 200] + assert len(counter.listings) == 1 + first_etag, second_etag = (turn.headers["x-models-etag"] for turn in turns) + assert first_etag == second_etag != _UPSTREAM_MODELS_ETAG From c7d95a338e2c96e0acbce2e721fc28ad7edc3fa1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 11:44:37 -0700 Subject: [PATCH 08/20] fix(proxy): keep Codex namespace tool calls intact across LiteLLM providers Fireworks and Bedrock Mantle accept Codex namespace tools natively, so the native route now hoists them without flattening, like OpenAI and Azure and the chat completions bridge. Other native Responses providers still get flat names, now built with the bridge's namespace_tool_name_map, and each function_call they return gets its tool name and namespace back in the non-streaming body and in streamed output_item.added, output_item.done, and completed/incomplete/failed events. Every other event passes through unchanged --- .../codex_passthrough_endpoints.py | 186 +++++++++++++++--- litellm/responses/additional_tools.py | 65 ++++-- .../test_codex_passthrough_endpoints.py | 138 ++++++++++++- 3 files changed, 352 insertions(+), 37 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py index 8ef4c64dd92..ac73af18293 100644 --- a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py @@ -2,14 +2,15 @@ import hashlib import hmac import json import re -from collections.abc import Iterator, Mapping +from collections.abc import AsyncIterable, AsyncIterator, Iterator, Mapping +from dataclasses import dataclass from types import MappingProxyType from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol import httpx from fastapi import APIRouter, Depends, HTTPException, Request, Response from fastapi.responses import StreamingResponse -from pydantic import TypeAdapter, ValidationError +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from starlette.types import Message import litellm @@ -39,10 +40,13 @@ from litellm.proxy.common_utils.http_parsing_utils import ( ) from litellm.proxy.pass_through_endpoints.pass_through_endpoints import pass_through_request from litellm.proxy.response_api_endpoints.endpoints import responses_api -from litellm.responses.additional_tools import with_additional_tools_hoisted, with_namespace_members_as_tools +from litellm.responses.additional_tools import with_additional_tools_hoisted, with_namespaces_flattened from litellm.responses.litellm_completion_transformation.reasoning_items import is_other_provider_reasoning_item +from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import EncryptedContentAffinityCheck +from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.custom_http import httpxSpecialProvider +from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.router import Deployment from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager, supports_none_reasoning_effort @@ -76,6 +80,12 @@ _ACCOUNT_SLUGS_CACHE_PREFIX: Final = "codex_account_slugs:" _MODELS_ETAG_CACHE_PREFIX: Final = "codex_models_etag:" _MODELS_ETAG_TTL_SECONDS: Final = 300 _OPENAI_API_PROVIDERS: Final = frozenset({LlmProviders.OPENAI.value, LlmProviders.AZURE.value}) +_NAMESPACE_TOOL_PROVIDERS: Final = frozenset( + {*_OPENAI_API_PROVIDERS, LlmProviders.FIREWORKS_AI.value, LlmProviders.BEDROCK_MANTLE.value} +) +_TOOL_DEFINITIONS: Final = TypeAdapter(tuple[dict[str, object], ...]) +_NOTHING_FLATTENED: Final[Mapping[str, tuple[str, str]]] = MappingProxyType({}) +_SSE_DATA_FIELD: Final = "data:" _HEADERS: Final = frozenset( { "authorization", @@ -95,6 +105,23 @@ _HEADERS: Final = frozenset( ) +class _FunctionCallName(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + type: Literal["function_call"] + name: str + + +class _ResponseOutput(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + output: tuple[object, ...] = () + + +@dataclass(frozen=True, slots=True) +class _LiteLLMRequest: + body: Mapping[str, object] + flattened_names: Mapping[str, tuple[str, str]] + + def _is_chatgpt(deployment: Deployment) -> bool: return ( deployment.litellm_params.custom_llm_provider == _CHATGPT_PROVIDER @@ -373,39 +400,132 @@ def _is_openai_api(deployment: Deployment) -> bool: return resolved is not None and resolved[1] in _OPENAI_API_PROVIDERS -def _bridges_to_chat_completions(deployment: Deployment) -> bool: +def _takes_namespace_tools(deployment: Deployment) -> bool: resolved: Final = _model_and_provider(deployment) + if resolved is None: + return False + model, provider = resolved return ( - resolved is not None - and ProviderConfigManager.get_provider_responses_api_config( - provider=resolved[1], model=resolved[0], api_base=deployment.litellm_params.api_base + provider in _NAMESPACE_TOOL_PROVIDERS + or ProviderConfigManager.get_provider_responses_api_config( + provider=provider, model=model, api_base=deployment.litellm_params.api_base ) is None ) -def _with_tools_the_targets_accept( - body: Mapping[str, object], deployments: tuple[Deployment, ...] -) -> Mapping[str, object]: +def _flattened_names(body: Mapping[str, object]) -> Mapping[str, tuple[str, str]]: + try: + tools: Final = _TOOL_DEFINITIONS.validate_python(body.get("tools") or ()) + except ValidationError: + return _NOTHING_FLATTENED + names: Final = LiteLLMCompletionResponsesConfig.namespace_tool_name_map(tools) + return MappingProxyType( + {flat_name: (namespace, tool) for flat_name, (namespace, tool) in names.items() if flat_name != tool} + ) + + +def _with_tools_the_targets_accept(body: Mapping[str, object], deployments: tuple[Deployment, ...]) -> _LiteLLMRequest: if all(_is_openai_api(deployment) for deployment in deployments): - return body + return _LiteLLMRequest(body=body, flattened_names=_NOTHING_FLATTENED) hoisted: Final = with_additional_tools_hoisted(body) - if all(_bridges_to_chat_completions(deployment) for deployment in deployments): - return hoisted - return with_namespace_members_as_tools(hoisted) + if all(_takes_namespace_tools(deployment) for deployment in deployments): + return _LiteLLMRequest(body=hoisted, flattened_names=_NOTHING_FLATTENED) + flattened_names: Final = _flattened_names(hoisted) + return _LiteLLMRequest(body=with_namespaces_flattened(hoisted, flattened_names), flattened_names=flattened_names) -def _litellm_body(body: Mapping[str, object], llm_router: "Router | None", team_id: str | None) -> Mapping[str, object]: +def _litellm_request(body: Mapping[str, object], llm_router: "Router | None", team_id: str | None) -> _LiteLLMRequest: wrapped: Final = _with_wrapped_reasoning(_with_provider_default_reasoning(body, llm_router, team_id)) model: Final = body.get("model") if not isinstance(model, str) or llm_router is None: - return wrapped + return _LiteLLMRequest(body=wrapped, flattened_names=_NOTHING_FLATTENED) deployments: Final = _candidate_deployments(model, llm_router, team_id) return _with_tools_the_targets_accept( _with_reasoning_the_targets_can_decrypt(wrapped, model, llm_router, deployments), deployments ) +def _namespaced(item: object, flattened_names: Mapping[str, tuple[str, str]]) -> tuple[str, str] | None: + try: + call: Final = _FunctionCallName.model_validate(item) + except ValidationError: + return None + return flattened_names.get(call.name) + + +def _with_namespace(item: object, flattened_names: Mapping[str, tuple[str, str]]) -> object: + namespaced: Final = _namespaced(item, flattened_names) + if namespaced is None: + return item + namespace, name = namespaced + restored: Final = {"name": name, "namespace": namespace} + match item: + case BaseModel(): + return item.model_copy(update=restored) + case _: + return {**_json_object(item), **restored} + + +def _response_output(response: object) -> tuple[object, ...]: + try: + return _ResponseOutput.model_validate(response).output + except ValidationError: + return () + + +def _output_with_namespaces( + response: object, flattened_names: Mapping[str, tuple[str, str]] +) -> tuple[object, ...] | None: + output: Final = _response_output(response) + if all(_namespaced(item, flattened_names) is None for item in output): + return None + return tuple(_with_namespace(item, flattened_names) for item in output) + + +def _event_with_namespaces( + event: Mapping[str, object], flattened_names: Mapping[str, tuple[str, str]] +) -> Mapping[str, object] | None: + match event.get("type"): + case "response.output_item.added" | "response.output_item.done": + item: Final = event.get("item") + if _namespaced(item, flattened_names) is None: + return None + return {**event, "item": _with_namespace(item, flattened_names)} + case "response.completed" | "response.incomplete" | "response.failed": + response: Final = _json_object(event.get("response")) + output: Final = _output_with_namespaces(response, flattened_names) + return None if output is None else {**event, "response": {**response, "output": list(output)}} + case _: + return None + + +def _sse_line_with_namespaces(line: str, flattened_names: Mapping[str, tuple[str, str]]) -> str: + if not line.startswith(_SSE_DATA_FIELD): + return line + try: + event: Final = _JSON_OBJECT.validate_json(line.removeprefix(_SSE_DATA_FIELD)) + except ValidationError: + return line + restored: Final = _event_with_namespaces(event, flattened_names) + return line if restored is None else f"{_SSE_DATA_FIELD} {json.dumps(restored)}" + + +def _frame_with_namespaces( + frame: str | bytes | memoryview, flattened_names: Mapping[str, tuple[str, str]] +) -> str | bytes | memoryview: + if not isinstance(frame, str) or not any(flat_name in frame for flat_name in flattened_names): + return frame + return "\n".join(_sse_line_with_namespaces(line, flattened_names) for line in frame.split("\n")) + + +async def _frames_with_namespaces( + frames: AsyncIterable[str | bytes | memoryview], flattened_names: Mapping[str, tuple[str, str]] +) -> AsyncIterator[str | bytes | memoryview]: + async for frame in frames: + yield _frame_with_namespaces(frame, flattened_names) + + def _is_unpersisted_reasoning_reference(item: object) -> bool: try: fields: Final = _JSON_OBJECT.validate_python(item) @@ -553,6 +673,33 @@ async def _catalog( return _with_etag(upstream, merged if merged is not None else upstream_body, etag) +async def _litellm_responses( + request: Request, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth, + body: Mapping[str, object], + llm_router: "Router | None", +) -> object: + litellm_request: Final = _litellm_request(body, llm_router, user_api_key_dict.team_id) + served: Final = await responses_api( + _as_litellm_responses_request(request, json.dumps(litellm_request.body).encode(), litellm_request.body), + fastapi_response, + user_api_key_dict.model_copy(update={"request_route": _LITELLM_RESPONSES_PATH}), + ) + flattened_names: Final = litellm_request.flattened_names + if not flattened_names: + return served + match served: + case StreamingResponse(): + served.body_iterator = _frames_with_namespaces(served.body_iterator, flattened_names) + return served + case ResponsesAPIResponse(): + output: Final = _output_with_namespaces(served, flattened_names) + return served if output is None else served.model_copy(update={"output": list(output)}) + case _: + return served + + async def _responses( request: Request, fastapi_response: Response, @@ -565,12 +712,7 @@ async def _responses( body: Final = _JSON_OBJECT.validate_python(await read_request_body(request)) if await _serves_through_litellm(body.get("model"), llm_router, user_api_key_dict, headers, cache): - litellm_body: Final = _litellm_body(body, llm_router, user_api_key_dict.team_id) - return await responses_api( - _as_litellm_responses_request(request, json.dumps(litellm_body).encode(), litellm_body), - fastapi_response, - user_api_key_dict.model_copy(update={"request_route": _LITELLM_RESPONSES_PATH}), - ) + return await _litellm_responses(request, fastapi_response, user_api_key_dict, body, llm_router) chatgpt_body: Final = _chatgpt_body(body, get_client_requested_model(request)) upstream: Final = await pass_through_request( request=request, diff --git a/litellm/responses/additional_tools.py b/litellm/responses/additional_tools.py index 03972fbd441..8fa222f0bcc 100644 --- a/litellm/responses/additional_tools.py +++ b/litellm/responses/additional_tools.py @@ -3,9 +3,14 @@ from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass from itertools import chain, groupby from types import MappingProxyType -from typing import Final, cast # noqa: TID251 # validating the openai tool union strips vendor keys from raw tools +from typing import ( + Annotated, + Final, + Literal, + cast, # noqa: TID251 # validating the openai tool union strips vendor keys from raw tools +) -from pydantic import TypeAdapter, ValidationError +from pydantic import ConfigDict, Field, TypeAdapter, ValidationError from litellm._logging import verbose_logger from litellm.types.llms.base import LiteLLMBaseModel @@ -13,8 +18,8 @@ from litellm.types.llms.openai import ALL_RESPONSES_API_TOOL_PARAMS, ResponseInp ADDITIONAL_TOOLS_INPUT_ITEM_TYPE: Final = "additional_tools" _NAMESPACE_TOOL_TYPE: Final = "namespace" -_QUALIFIED_NAME_SEPARATOR: Final = "__" -_TOOLS: Final = TypeAdapter(tuple[object, ...]) +_ITEMS: Final = TypeAdapter(tuple[object, ...]) +_JSON_OBJECT: Final = TypeAdapter(dict[str, object]) class _InputItemType(LiteLLMBaseModel): @@ -25,6 +30,13 @@ class _ToolList(LiteLLMBaseModel): tools: tuple[Mapping[str, object], ...] = () +class _NamespacedFunctionCall(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + type: Literal["function_call"] + name: str + namespace: Annotated[str, Field(min_length=1)] + + @dataclass(frozen=True, slots=True) class HoistedAdditionalTools: input: str | ResponseInputParam @@ -119,7 +131,7 @@ def with_additional_tools_hoisted(request: Mapping[str, object]) -> Mapping[str, return request input_items: Final = cast(ResponseInputParam, request_input) # cast-ok: forwarded as sent, as inference does top_level_tools: Final = cast( # cast-ok: tools are forwarded as sent, as inference does - "tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]", _TOOLS.validate_python(request.get("tools") or ()) + "tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]", _ITEMS.validate_python(request.get("tools") or ()) ) hoisted: Final = hoist_additional_tools(input_items, top_level_tools) if not hoisted.hoisted: @@ -127,20 +139,45 @@ def with_additional_tools_hoisted(request: Mapping[str, object]) -> Mapping[str, return {**request, "input": hoisted.input, "tools": list(hoisted.tools)} -def _qualified_member(namespace: str, member: Mapping[str, object]) -> Mapping[str, object]: - prefix: Final = f"{namespace.removesuffix(_QUALIFIED_NAME_SEPARATOR)}{_QUALIFIED_NAME_SEPARATOR}" - return {**member, "name": f"{prefix}{member.get('name')}"} +def _flattened_member( + namespace: str, member: Mapping[str, object], flat_names: Mapping[tuple[str, str], str] +) -> Mapping[str, object]: + flat_name: Final = flat_names.get((namespace, str(member.get("name") or ""))) + return member if flat_name is None else {**member, "name": flat_name} -def _members_as_tools(tool: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: +def _members_as_tools( + tool: Mapping[str, object], flat_names: Mapping[tuple[str, str], str] +) -> tuple[Mapping[str, object], ...]: if tool.get("type") != _NAMESPACE_TOOL_TYPE: return (tool,) namespace: Final = str(tool.get("name") or "") - return tuple(_qualified_member(namespace, member) for member in _tool_list_of(tool)) + return tuple(_flattened_member(namespace, member, flat_names) for member in _tool_list_of(tool)) -def with_namespace_members_as_tools(request: Mapping[str, object]) -> Mapping[str, object]: - tools: Final = _tool_list_of(request) - if not any(tool.get("type") == _NAMESPACE_TOOL_TYPE for tool in tools): +def _flattened_call(item: object, flat_names: Mapping[tuple[str, str], str]) -> object: + try: + call: Final = _NamespacedFunctionCall.model_validate(item) + except ValidationError: + return item + flat_name: Final = flat_names.get((call.namespace, call.name)) + if flat_name is None: + return item + fields: Final = _JSON_OBJECT.validate_python(item) + return {**{key: value for key, value in fields.items() if key != "namespace"}, "name": flat_name} + + +def with_namespaces_flattened( + request: Mapping[str, object], namespaced_names: Mapping[str, tuple[str, str]] +) -> Mapping[str, object]: + if not namespaced_names: return request - return {**request, "tools": list(chain.from_iterable(map(_members_as_tools, tools)))} + flat_names: Final = MappingProxyType({namespaced: flat for flat, namespaced in namespaced_names.items()}) + tools: Final = chain.from_iterable(_members_as_tools(tool, flat_names) for tool in _tool_list_of(request)) + request_input: Final = request.get("input") + replayed: Final = ( + {"input": [_flattened_call(item, flat_names) for item in _ITEMS.validate_python(request_input)]} + if isinstance(request_input, list) + else {} + ) + return {**request, "tools": list(tools), **replayed} diff --git a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py index 07dcc2859d6..f05d6d62fcf 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py @@ -1181,7 +1181,7 @@ _ADDITIONAL_TOOLS: Final = { pytest.param( "grok-4.7", [_USER_TURN], - ["exec_command", "collaboration__followup_task", "mcp__docs__search"], + ["exec_command", "collaboration__followup_task", "mcp__docs____search"], id="xai gets the item's tools as top-level functions", ), pytest.param("oai-luna", [_ADDITIONAL_TOOLS, _USER_TURN], ["exec_command"], id="the openai api keeps the item"), @@ -1268,6 +1268,142 @@ def test_codex_namespace_tool_called_through_the_chat_bridge_comes_back_with_its assert (call["namespace"], call["name"]) == ("mcp__docs__", "search") +_CLOCK_NAMESPACE: Final = _namespace("clock", "sleep") +_CLOCK_TOOLS: Final = {"type": "additional_tools", "role": "developer", "tools": [_CLOCK_NAMESPACE]} +_FLAT_SLEEP_TOOL: Final = {"type": "function", "name": "clock__sleep", "parameters": {"type": "object"}} +_FLAT_SLEEP_CALL: Final = { + "type": "function_call", + "id": "fc_sleep", + "call_id": "call_sleep", + "name": "clock__sleep", + "arguments": '{"seconds":1}', + "status": "completed", +} + + +def test_codex_namespace_tool_reaches_fireworks_as_a_namespace( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router +) -> None: + _account_catalog(respx_mock) + provider: Final = _open_weights_provider(respx_mock) + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": "open-weights", "input": [_CLOCK_TOOLS, _USER_TURN], "stream": False, "store": False}, + headers=_ACCOUNT_HEADERS, + ) + assert response.status_code == 200, response.text + assert json.loads(provider.calls.last.request.content)["tools"] == [_CLOCK_NAMESPACE] + + +def test_xai_namespace_call_round_trips_to_codex_with_its_namespace( + client: TestClient, respx_mock: respx.MockRouter, xai_and_openai_router: Router +) -> None: + _account_catalog(respx_mock) + xai: Final = respx_mock.post(f"{_XAI_BASE}/responses").respond( + json={**_provider_completed(""), "output": [_FLAT_SLEEP_CALL]} + ) + replayed_call: Final = {**_FLAT_SLEEP_CALL, "name": "sleep", "namespace": "clock"} + replayed_output: Final = {"type": "function_call_output", "call_id": "call_sleep", "output": "slept"} + response: Final = client.post( + "/backend-api/codex/responses", + json={ + "model": "grok-4.7", + "input": [_CLOCK_TOOLS, _USER_TURN, replayed_call, replayed_output], + "stream": False, + "store": False, + }, + headers=_ACCOUNT_HEADERS, + ) + assert response.status_code == 200, response.text + sent: Final = json.loads(xai.calls.last.request.content) + assert sent["tools"] == [_FLAT_SLEEP_TOOL] + assert sent["input"][1:] == [_FLAT_SLEEP_CALL, replayed_output] + (call,) = (item for item in response.json()["output"] if item["type"] == "function_call") + assert (call["name"], call["namespace"], call["call_id"]) == ("sleep", "clock", "call_sleep") + + +def _sse_events(*events: dict[str, object]) -> bytes: + return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events).encode() + + +def _xai_sleep_call_stream() -> bytes: + completed: Final = {**_provider_completed(""), "output": [_FLAT_SLEEP_CALL]} + return _sse_events( + { + "type": "response.created", + "sequence_number": 0, + "response": {**completed, "status": "in_progress", "output": []}, + }, + { + "type": "response.output_item.added", + "sequence_number": 1, + "output_index": 0, + "item": {**_FLAT_SLEEP_CALL, "arguments": "", "status": "in_progress"}, + }, + { + "type": "response.function_call_arguments.delta", + "sequence_number": 2, + "item_id": "fc_sleep", + "output_index": 0, + "delta": '{"seconds":1}', + }, + {"type": "response.output_item.done", "sequence_number": 3, "output_index": 0, "item": _FLAT_SLEEP_CALL}, + {"type": "response.completed", "sequence_number": 4, "response": completed}, + ) + + +def _data_lines(stream: bytes) -> list[bytes]: + return [line.removeprefix(b"data: ") for line in stream.splitlines() if line.startswith(b"data: {")] + + +def test_xai_streamed_namespace_call_reaches_codex_with_its_namespace_and_other_events_unchanged( + client: TestClient, respx_mock: respx.MockRouter, xai_and_openai_router: Router +) -> None: + _account_catalog(respx_mock) + respx_mock.post(f"{_XAI_BASE}/responses").respond( + content=_xai_sleep_call_stream(), headers={"content-type": "text/event-stream"} + ) + native: Final = client.post( + "/backend-api/codex/responses", + json={"model": "grok-4.7", "input": [_CLOCK_TOOLS, _USER_TURN], "stream": True, "store": False}, + headers=_ACCOUNT_HEADERS, + ) + v1: Final = client.post( + "/v1/responses", + json={"model": "grok-4.7", "input": [_USER_TURN], "tools": [_FLAT_SLEEP_TOOL], "stream": True, "store": False}, + headers={"authorization": f"Bearer {MASTER_KEY}"}, + ) + assert native.status_code == v1.status_code == 200 + native_lines: Final = _data_lines(native.content) + v1_lines: Final = _data_lines(v1.content) + native_events: Final = [json.loads(line) for line in native_lines] + v1_events: Final = [json.loads(line) for line in v1_lines] + assert ( + [event["type"] for event in native_events] + == [event["type"] for event in v1_events] + == [ + "response.created", + "response.output_item.added", + "response.function_call_arguments.delta", + "response.output_item.done", + "response.completed", + ] + ) + restored: Final = {"name": "sleep", "namespace": "clock"} + for native_line, v1_line, native_event, v1_event in zip(native_lines, v1_lines, native_events, v1_events): + match v1_event["type"]: + case "response.output_item.added" | "response.output_item.done": + assert native_event == {**v1_event, "item": {**v1_event["item"], **restored}} + case "response.completed": + (v1_call,) = v1_event["response"]["output"] + assert native_event == { + **v1_event, + "response": {**v1_event["response"], "output": [{**v1_call, **restored}]}, + } + case _: + assert native_line == v1_line + + def test_encrypted_reasoning_stays_only_when_every_deployment_the_router_may_pick_can_decrypt_it( client: TestClient, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ) -> None: From 5611997f6affd3d310ae936eb0a1c0959783a5c2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 13:32:13 -0700 Subject: [PATCH 09/20] fix(proxy): bridge Codex turns for providers whose Responses API rejects Codex tools Codex sends its freeform exec tool as a custom tool on every turn, and xAI's Responses API answers 422 for custom, namespace and additional_tools shapes. On the Codex native route, a model whose deployments include a native Responses provider other than OpenAI, Azure, Fireworks or Bedrock Mantle now asks for the chat completions bridge with use_chat_completions_api, which already turns custom and namespace tools into functions, hoists additional_tools, and puts the namespace back on each returned call OpenAI and Azure still get the request unchanged, and Fireworks and Bedrock Mantle still get it on their own Responses API with additional_tools hoisted. The route's namespace flattening and the response-side namespace restore are removed, since no path sends flattened names anymore --- .../codex_passthrough_endpoints.py | 180 ++------------ litellm/responses/additional_tools.py | 63 +---- .../test_codex_passthrough_endpoints.py | 221 ++++++++---------- 3 files changed, 125 insertions(+), 339 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py index ac73af18293..7225c1fbf7e 100644 --- a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py @@ -2,15 +2,14 @@ import hashlib import hmac import json import re -from collections.abc import AsyncIterable, AsyncIterator, Iterator, Mapping -from dataclasses import dataclass +from collections.abc import Iterator, Mapping from types import MappingProxyType from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol import httpx from fastapi import APIRouter, Depends, HTTPException, Request, Response from fastapi.responses import StreamingResponse -from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError +from pydantic import TypeAdapter, ValidationError from starlette.types import Message import litellm @@ -40,13 +39,10 @@ from litellm.proxy.common_utils.http_parsing_utils import ( ) from litellm.proxy.pass_through_endpoints.pass_through_endpoints import pass_through_request from litellm.proxy.response_api_endpoints.endpoints import responses_api -from litellm.responses.additional_tools import with_additional_tools_hoisted, with_namespaces_flattened +from litellm.responses.additional_tools import with_additional_tools_hoisted from litellm.responses.litellm_completion_transformation.reasoning_items import is_other_provider_reasoning_item -from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import EncryptedContentAffinityCheck -from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.custom_http import httpxSpecialProvider -from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.router import Deployment from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager, supports_none_reasoning_effort @@ -80,12 +76,10 @@ _ACCOUNT_SLUGS_CACHE_PREFIX: Final = "codex_account_slugs:" _MODELS_ETAG_CACHE_PREFIX: Final = "codex_models_etag:" _MODELS_ETAG_TTL_SECONDS: Final = 300 _OPENAI_API_PROVIDERS: Final = frozenset({LlmProviders.OPENAI.value, LlmProviders.AZURE.value}) -_NAMESPACE_TOOL_PROVIDERS: Final = frozenset( +_CODEX_TOOL_PROVIDERS: Final = frozenset( {*_OPENAI_API_PROVIDERS, LlmProviders.FIREWORKS_AI.value, LlmProviders.BEDROCK_MANTLE.value} ) -_TOOL_DEFINITIONS: Final = TypeAdapter(tuple[dict[str, object], ...]) -_NOTHING_FLATTENED: Final[Mapping[str, tuple[str, str]]] = MappingProxyType({}) -_SSE_DATA_FIELD: Final = "data:" +_CHAT_COMPLETIONS_BRIDGE: Final = "use_chat_completions_api" _HEADERS: Final = frozenset( { "authorization", @@ -105,23 +99,6 @@ _HEADERS: Final = frozenset( ) -class _FunctionCallName(LiteLLMBaseModel): - model_config = ConfigDict(frozen=True, from_attributes=True) - type: Literal["function_call"] - name: str - - -class _ResponseOutput(LiteLLMBaseModel): - model_config = ConfigDict(frozen=True, from_attributes=True) - output: tuple[object, ...] = () - - -@dataclass(frozen=True, slots=True) -class _LiteLLMRequest: - body: Mapping[str, object] - flattened_names: Mapping[str, tuple[str, str]] - - def _is_chatgpt(deployment: Deployment) -> bool: return ( deployment.litellm_params.custom_llm_provider == _CHATGPT_PROVIDER @@ -400,13 +377,13 @@ def _is_openai_api(deployment: Deployment) -> bool: return resolved is not None and resolved[1] in _OPENAI_API_PROVIDERS -def _takes_namespace_tools(deployment: Deployment) -> bool: +def _takes_codex_tools(deployment: Deployment) -> bool: resolved: Final = _model_and_provider(deployment) if resolved is None: return False model, provider = resolved return ( - provider in _NAMESPACE_TOOL_PROVIDERS + provider in _CODEX_TOOL_PROVIDERS or ProviderConfigManager.get_provider_responses_api_config( provider=provider, model=model, api_base=deployment.litellm_params.api_base ) @@ -414,118 +391,27 @@ def _takes_namespace_tools(deployment: Deployment) -> bool: ) -def _flattened_names(body: Mapping[str, object]) -> Mapping[str, tuple[str, str]]: - try: - tools: Final = _TOOL_DEFINITIONS.validate_python(body.get("tools") or ()) - except ValidationError: - return _NOTHING_FLATTENED - names: Final = LiteLLMCompletionResponsesConfig.namespace_tool_name_map(tools) - return MappingProxyType( - {flat_name: (namespace, tool) for flat_name, (namespace, tool) in names.items() if flat_name != tool} - ) - - -def _with_tools_the_targets_accept(body: Mapping[str, object], deployments: tuple[Deployment, ...]) -> _LiteLLMRequest: +def _with_tools_the_targets_accept( + body: Mapping[str, object], deployments: tuple[Deployment, ...] +) -> Mapping[str, object]: if all(_is_openai_api(deployment) for deployment in deployments): - return _LiteLLMRequest(body=body, flattened_names=_NOTHING_FLATTENED) - hoisted: Final = with_additional_tools_hoisted(body) - if all(_takes_namespace_tools(deployment) for deployment in deployments): - return _LiteLLMRequest(body=hoisted, flattened_names=_NOTHING_FLATTENED) - flattened_names: Final = _flattened_names(hoisted) - return _LiteLLMRequest(body=with_namespaces_flattened(hoisted, flattened_names), flattened_names=flattened_names) + return body + if all(_takes_codex_tools(deployment) for deployment in deployments): + return with_additional_tools_hoisted(body) + return {**body, _CHAT_COMPLETIONS_BRIDGE: True} -def _litellm_request(body: Mapping[str, object], llm_router: "Router | None", team_id: str | None) -> _LiteLLMRequest: +def _litellm_body(body: Mapping[str, object], llm_router: "Router | None", team_id: str | None) -> Mapping[str, object]: wrapped: Final = _with_wrapped_reasoning(_with_provider_default_reasoning(body, llm_router, team_id)) model: Final = body.get("model") if not isinstance(model, str) or llm_router is None: - return _LiteLLMRequest(body=wrapped, flattened_names=_NOTHING_FLATTENED) + return wrapped deployments: Final = _candidate_deployments(model, llm_router, team_id) return _with_tools_the_targets_accept( _with_reasoning_the_targets_can_decrypt(wrapped, model, llm_router, deployments), deployments ) -def _namespaced(item: object, flattened_names: Mapping[str, tuple[str, str]]) -> tuple[str, str] | None: - try: - call: Final = _FunctionCallName.model_validate(item) - except ValidationError: - return None - return flattened_names.get(call.name) - - -def _with_namespace(item: object, flattened_names: Mapping[str, tuple[str, str]]) -> object: - namespaced: Final = _namespaced(item, flattened_names) - if namespaced is None: - return item - namespace, name = namespaced - restored: Final = {"name": name, "namespace": namespace} - match item: - case BaseModel(): - return item.model_copy(update=restored) - case _: - return {**_json_object(item), **restored} - - -def _response_output(response: object) -> tuple[object, ...]: - try: - return _ResponseOutput.model_validate(response).output - except ValidationError: - return () - - -def _output_with_namespaces( - response: object, flattened_names: Mapping[str, tuple[str, str]] -) -> tuple[object, ...] | None: - output: Final = _response_output(response) - if all(_namespaced(item, flattened_names) is None for item in output): - return None - return tuple(_with_namespace(item, flattened_names) for item in output) - - -def _event_with_namespaces( - event: Mapping[str, object], flattened_names: Mapping[str, tuple[str, str]] -) -> Mapping[str, object] | None: - match event.get("type"): - case "response.output_item.added" | "response.output_item.done": - item: Final = event.get("item") - if _namespaced(item, flattened_names) is None: - return None - return {**event, "item": _with_namespace(item, flattened_names)} - case "response.completed" | "response.incomplete" | "response.failed": - response: Final = _json_object(event.get("response")) - output: Final = _output_with_namespaces(response, flattened_names) - return None if output is None else {**event, "response": {**response, "output": list(output)}} - case _: - return None - - -def _sse_line_with_namespaces(line: str, flattened_names: Mapping[str, tuple[str, str]]) -> str: - if not line.startswith(_SSE_DATA_FIELD): - return line - try: - event: Final = _JSON_OBJECT.validate_json(line.removeprefix(_SSE_DATA_FIELD)) - except ValidationError: - return line - restored: Final = _event_with_namespaces(event, flattened_names) - return line if restored is None else f"{_SSE_DATA_FIELD} {json.dumps(restored)}" - - -def _frame_with_namespaces( - frame: str | bytes | memoryview, flattened_names: Mapping[str, tuple[str, str]] -) -> str | bytes | memoryview: - if not isinstance(frame, str) or not any(flat_name in frame for flat_name in flattened_names): - return frame - return "\n".join(_sse_line_with_namespaces(line, flattened_names) for line in frame.split("\n")) - - -async def _frames_with_namespaces( - frames: AsyncIterable[str | bytes | memoryview], flattened_names: Mapping[str, tuple[str, str]] -) -> AsyncIterator[str | bytes | memoryview]: - async for frame in frames: - yield _frame_with_namespaces(frame, flattened_names) - - def _is_unpersisted_reasoning_reference(item: object) -> bool: try: fields: Final = _JSON_OBJECT.validate_python(item) @@ -673,33 +559,6 @@ async def _catalog( return _with_etag(upstream, merged if merged is not None else upstream_body, etag) -async def _litellm_responses( - request: Request, - fastapi_response: Response, - user_api_key_dict: UserAPIKeyAuth, - body: Mapping[str, object], - llm_router: "Router | None", -) -> object: - litellm_request: Final = _litellm_request(body, llm_router, user_api_key_dict.team_id) - served: Final = await responses_api( - _as_litellm_responses_request(request, json.dumps(litellm_request.body).encode(), litellm_request.body), - fastapi_response, - user_api_key_dict.model_copy(update={"request_route": _LITELLM_RESPONSES_PATH}), - ) - flattened_names: Final = litellm_request.flattened_names - if not flattened_names: - return served - match served: - case StreamingResponse(): - served.body_iterator = _frames_with_namespaces(served.body_iterator, flattened_names) - return served - case ResponsesAPIResponse(): - output: Final = _output_with_namespaces(served, flattened_names) - return served if output is None else served.model_copy(update={"output": list(output)}) - case _: - return served - - async def _responses( request: Request, fastapi_response: Response, @@ -712,7 +571,12 @@ async def _responses( body: Final = _JSON_OBJECT.validate_python(await read_request_body(request)) if await _serves_through_litellm(body.get("model"), llm_router, user_api_key_dict, headers, cache): - return await _litellm_responses(request, fastapi_response, user_api_key_dict, body, llm_router) + litellm_body: Final = _litellm_body(body, llm_router, user_api_key_dict.team_id) + return await responses_api( + _as_litellm_responses_request(request, json.dumps(litellm_body).encode(), litellm_body), + fastapi_response, + user_api_key_dict.model_copy(update={"request_route": _LITELLM_RESPONSES_PATH}), + ) chatgpt_body: Final = _chatgpt_body(body, get_client_requested_model(request)) upstream: Final = await pass_through_request( request=request, diff --git a/litellm/responses/additional_tools.py b/litellm/responses/additional_tools.py index 8fa222f0bcc..d12f2a1abd4 100644 --- a/litellm/responses/additional_tools.py +++ b/litellm/responses/additional_tools.py @@ -1,16 +1,11 @@ import json from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass -from itertools import chain, groupby +from itertools import groupby from types import MappingProxyType -from typing import ( - Annotated, - Final, - Literal, - cast, # noqa: TID251 # validating the openai tool union strips vendor keys from raw tools -) +from typing import Final, cast # noqa: TID251 # validating the openai tool union strips vendor keys from raw tools -from pydantic import ConfigDict, Field, TypeAdapter, ValidationError +from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_logger from litellm.types.llms.base import LiteLLMBaseModel @@ -19,7 +14,6 @@ from litellm.types.llms.openai import ALL_RESPONSES_API_TOOL_PARAMS, ResponseInp ADDITIONAL_TOOLS_INPUT_ITEM_TYPE: Final = "additional_tools" _NAMESPACE_TOOL_TYPE: Final = "namespace" _ITEMS: Final = TypeAdapter(tuple[object, ...]) -_JSON_OBJECT: Final = TypeAdapter(dict[str, object]) class _InputItemType(LiteLLMBaseModel): @@ -30,13 +24,6 @@ class _ToolList(LiteLLMBaseModel): tools: tuple[Mapping[str, object], ...] = () -class _NamespacedFunctionCall(LiteLLMBaseModel): - model_config = ConfigDict(frozen=True) - type: Literal["function_call"] - name: str - namespace: Annotated[str, Field(min_length=1)] - - @dataclass(frozen=True, slots=True) class HoistedAdditionalTools: input: str | ResponseInputParam @@ -137,47 +124,3 @@ def with_additional_tools_hoisted(request: Mapping[str, object]) -> Mapping[str, if not hoisted.hoisted: return request return {**request, "input": hoisted.input, "tools": list(hoisted.tools)} - - -def _flattened_member( - namespace: str, member: Mapping[str, object], flat_names: Mapping[tuple[str, str], str] -) -> Mapping[str, object]: - flat_name: Final = flat_names.get((namespace, str(member.get("name") or ""))) - return member if flat_name is None else {**member, "name": flat_name} - - -def _members_as_tools( - tool: Mapping[str, object], flat_names: Mapping[tuple[str, str], str] -) -> tuple[Mapping[str, object], ...]: - if tool.get("type") != _NAMESPACE_TOOL_TYPE: - return (tool,) - namespace: Final = str(tool.get("name") or "") - return tuple(_flattened_member(namespace, member, flat_names) for member in _tool_list_of(tool)) - - -def _flattened_call(item: object, flat_names: Mapping[tuple[str, str], str]) -> object: - try: - call: Final = _NamespacedFunctionCall.model_validate(item) - except ValidationError: - return item - flat_name: Final = flat_names.get((call.namespace, call.name)) - if flat_name is None: - return item - fields: Final = _JSON_OBJECT.validate_python(item) - return {**{key: value for key, value in fields.items() if key != "namespace"}, "name": flat_name} - - -def with_namespaces_flattened( - request: Mapping[str, object], namespaced_names: Mapping[str, tuple[str, str]] -) -> Mapping[str, object]: - if not namespaced_names: - return request - flat_names: Final = MappingProxyType({namespaced: flat for flat, namespaced in namespaced_names.items()}) - tools: Final = chain.from_iterable(_members_as_tools(tool, flat_names) for tool in _tool_list_of(request)) - request_input: Final = request.get("input") - replayed: Final = ( - {"input": [_flattened_call(item, flat_names) for item in _ITEMS.validate_python(request_input)]} - if isinstance(request_input, list) - else {} - ) - return {**request, "tools": list(tools), **replayed} diff --git a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py index f05d6d62fcf..08c35989262 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py @@ -1099,13 +1099,14 @@ _CHATGPT_REASONING: Final = {"type": "reasoning", "summary": [], "encrypted_cont @pytest.fixture -def xai_and_openai_router(monkeypatch: pytest.MonkeyPatch) -> Router: +def xai_fireworks_and_openai_router(monkeypatch: pytest.MonkeyPatch) -> Router: router: Final = Router( model_list=[ { "model_name": "grok-4.7", "litellm_params": {"model": "xai/grok-4.7", "api_base": _XAI_BASE, "api_key": "test-xai-key"}, }, + _open_weights("open-weights"), { "model_name": "oai-luna", "litellm_params": { @@ -1124,21 +1125,21 @@ def xai_and_openai_router(monkeypatch: pytest.MonkeyPatch) -> Router: @pytest.mark.parametrize( ("model", "replayed", "expected"), [ - pytest.param("grok-4.7", _CHATGPT_REASONING, None, id="chatgpt reasoning to xai is dropped"), - pytest.param("grok-4.7", _LUNA_REASONING, None, id="another deployment's reasoning to xai is dropped"), + pytest.param("open-weights", _CHATGPT_REASONING, None, id="chatgpt reasoning to fireworks is dropped"), + pytest.param("open-weights", _LUNA_REASONING, None, id="another deployment's reasoning to fireworks is dropped"), pytest.param("oai-luna", _LUNA_REASONING, _LUNA_BLOB, id="the minting deployment's reasoning is kept"), ], ) def test_litellm_model_on_native_route_gets_only_the_reasoning_it_can_decrypt( client: TestClient, respx_mock: respx.MockRouter, - xai_and_openai_router: Router, + xai_fireworks_and_openai_router: Router, model: str, replayed: dict[str, object], expected: str | None, ) -> None: _account_catalog(respx_mock) - xai: Final = respx_mock.post(f"{_XAI_BASE}/responses").mock(side_effect=_provider_reply) + fireworks: Final = _open_weights_provider(respx_mock) luna: Final = respx_mock.post(f"{_OPENAI_BASE}/responses").mock(side_effect=_provider_reply) response: Final = client.post( "/backend-api/codex/responses", @@ -1151,7 +1152,7 @@ def test_litellm_model_on_native_route_gets_only_the_reasoning_it_can_decrypt( headers=_ACCOUNT_HEADERS, ) assert response.status_code == 200, response.text - provider: Final = xai if model == "grok-4.7" else luna + provider: Final = fireworks if model == "open-weights" else luna sent: Final = json.loads(provider.calls.last.request.content)["input"] assert [item.get("encrypted_content") for item in sent if item.get("type") == "reasoning"] == ( [] if expected is None else [expected] @@ -1179,10 +1180,10 @@ _ADDITIONAL_TOOLS: Final = { ("model", "expected_input", "expected_tool_names"), [ pytest.param( - "grok-4.7", + "open-weights", [_USER_TURN], - ["exec_command", "collaboration__followup_task", "mcp__docs____search"], - id="xai gets the item's tools as top-level functions", + ["exec_command", "collaboration", "mcp__docs__"], + id="fireworks gets the item's tools as top-level tools", ), pytest.param("oai-luna", [_ADDITIONAL_TOOLS, _USER_TURN], ["exec_command"], id="the openai api keeps the item"), ], @@ -1190,13 +1191,13 @@ _ADDITIONAL_TOOLS: Final = { def test_codex_additional_tools_item_reaches_each_provider_in_a_shape_it_accepts( client: TestClient, respx_mock: respx.MockRouter, - xai_and_openai_router: Router, + xai_fireworks_and_openai_router: Router, model: str, expected_input: list[dict[str, object]], expected_tool_names: list[str], ) -> None: _account_catalog(respx_mock) - xai: Final = respx_mock.post(f"{_XAI_BASE}/responses").mock(side_effect=_provider_reply) + fireworks: Final = _open_weights_provider(respx_mock) luna: Final = respx_mock.post(f"{_OPENAI_BASE}/responses").mock(side_effect=_provider_reply) response: Final = client.post( "/backend-api/codex/responses", @@ -1210,7 +1211,7 @@ def test_codex_additional_tools_item_reaches_each_provider_in_a_shape_it_accepts headers=_ACCOUNT_HEADERS, ) assert response.status_code == 200, response.text - sent: Final = json.loads((xai if model == "grok-4.7" else luna).calls.last.request.content) + sent: Final = json.loads((fireworks if model == "open-weights" else luna).calls.last.request.content) assert sent["input"] == expected_input assert [tool["name"] for tool in sent["tools"]] == expected_tool_names @@ -1270,17 +1271,6 @@ def test_codex_namespace_tool_called_through_the_chat_bridge_comes_back_with_its _CLOCK_NAMESPACE: Final = _namespace("clock", "sleep") _CLOCK_TOOLS: Final = {"type": "additional_tools", "role": "developer", "tools": [_CLOCK_NAMESPACE]} -_FLAT_SLEEP_TOOL: Final = {"type": "function", "name": "clock__sleep", "parameters": {"type": "object"}} -_FLAT_SLEEP_CALL: Final = { - "type": "function_call", - "id": "fc_sleep", - "call_id": "call_sleep", - "name": "clock__sleep", - "arguments": '{"seconds":1}', - "status": "completed", -} - - def test_codex_namespace_tool_reaches_fireworks_as_a_namespace( client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router ) -> None: @@ -1295,113 +1285,102 @@ def test_codex_namespace_tool_reaches_fireworks_as_a_namespace( assert json.loads(provider.calls.last.request.content)["tools"] == [_CLOCK_NAMESPACE] -def test_xai_namespace_call_round_trips_to_codex_with_its_namespace( - client: TestClient, respx_mock: respx.MockRouter, xai_and_openai_router: Router -) -> None: - _account_catalog(respx_mock) - xai: Final = respx_mock.post(f"{_XAI_BASE}/responses").respond( - json={**_provider_completed(""), "output": [_FLAT_SLEEP_CALL]} - ) - replayed_call: Final = {**_FLAT_SLEEP_CALL, "name": "sleep", "namespace": "clock"} - replayed_output: Final = {"type": "function_call_output", "call_id": "call_sleep", "output": "slept"} - response: Final = client.post( - "/backend-api/codex/responses", - json={ - "model": "grok-4.7", - "input": [_CLOCK_TOOLS, _USER_TURN, replayed_call, replayed_output], - "stream": False, - "store": False, - }, - headers=_ACCOUNT_HEADERS, - ) - assert response.status_code == 200, response.text - sent: Final = json.loads(xai.calls.last.request.content) - assert sent["tools"] == [_FLAT_SLEEP_TOOL] - assert sent["input"][1:] == [_FLAT_SLEEP_CALL, replayed_output] - (call,) = (item for item in response.json()["output"] if item["type"] == "function_call") - assert (call["name"], call["namespace"], call["call_id"]) == ("sleep", "clock", "call_sleep") - - -def _sse_events(*events: dict[str, object]) -> bytes: - return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events).encode() - - -def _xai_sleep_call_stream() -> bytes: - completed: Final = {**_provider_completed(""), "output": [_FLAT_SLEEP_CALL]} - return _sse_events( - { - "type": "response.created", - "sequence_number": 0, - "response": {**completed, "status": "in_progress", "output": []}, - }, - { - "type": "response.output_item.added", - "sequence_number": 1, - "output_index": 0, - "item": {**_FLAT_SLEEP_CALL, "arguments": "", "status": "in_progress"}, - }, - { - "type": "response.function_call_arguments.delta", - "sequence_number": 2, - "item_id": "fc_sleep", - "output_index": 0, - "delta": '{"seconds":1}', - }, - {"type": "response.output_item.done", "sequence_number": 3, "output_index": 0, "item": _FLAT_SLEEP_CALL}, - {"type": "response.completed", "sequence_number": 4, "response": completed}, +_CUSTOM_EXEC_TOOL: Final = {"type": "custom", "name": "exec", "description": "Run a command", "format": {"type": "text"}} +_XAI_SEARCH_CALL: Final = { + "id": "call_search", + "type": "function", + "function": {"name": "mcp__docs____search", "arguments": '{"query":"proxy"}'}, +} +_XAI_USAGE: Final = {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4} + + +def _xai_chat_chunk(choice: dict[str, object], **fields: object) -> str: + chunk: Final = { + "id": "chatcmpl-xai", + "object": "chat.completion.chunk", + "created": 1700000000, + "model": "grok-4.7", + "choices": [{"index": 0, **choice}], + **fields, + } + return f"data: {json.dumps(chunk)}\n\n" + + +def _xai_search_call_reply(request: httpx.Request) -> httpx.Response: + if json.loads(request.content).get("stream") is not True: + return httpx.Response( + 200, + json={ + "id": "chatcmpl-xai", + "object": "chat.completion", + "created": 1700000000, + "model": "grok-4.7", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": None, "tool_calls": [_XAI_SEARCH_CALL]}, + "finish_reason": "tool_calls", + } + ], + "usage": _XAI_USAGE, + }, + ) + stream: Final = ( + _xai_chat_chunk( + {"delta": {"role": "assistant", "tool_calls": [{"index": 0, **_XAI_SEARCH_CALL}]}, "finish_reason": None} + ) + + _xai_chat_chunk({"delta": {}, "finish_reason": "tool_calls"}, usage=_XAI_USAGE) + + "data: [DONE]\n\n" ) + return httpx.Response(200, content=stream.encode(), headers={"content-type": "text/event-stream"}) def _data_lines(stream: bytes) -> list[bytes]: return [line.removeprefix(b"data: ") for line in stream.splitlines() if line.startswith(b"data: {")] -def test_xai_streamed_namespace_call_reaches_codex_with_its_namespace_and_other_events_unchanged( - client: TestClient, respx_mock: respx.MockRouter, xai_and_openai_router: Router -) -> None: - _account_catalog(respx_mock) - respx_mock.post(f"{_XAI_BASE}/responses").respond( - content=_xai_sleep_call_stream(), headers={"content-type": "text/event-stream"} - ) - native: Final = client.post( - "/backend-api/codex/responses", - json={"model": "grok-4.7", "input": [_CLOCK_TOOLS, _USER_TURN], "stream": True, "store": False}, - headers=_ACCOUNT_HEADERS, - ) - v1: Final = client.post( - "/v1/responses", - json={"model": "grok-4.7", "input": [_USER_TURN], "tools": [_FLAT_SLEEP_TOOL], "stream": True, "store": False}, - headers={"authorization": f"Bearer {MASTER_KEY}"}, - ) - assert native.status_code == v1.status_code == 200 - native_lines: Final = _data_lines(native.content) - v1_lines: Final = _data_lines(v1.content) - native_events: Final = [json.loads(line) for line in native_lines] - v1_events: Final = [json.loads(line) for line in v1_lines] - assert ( - [event["type"] for event in native_events] - == [event["type"] for event in v1_events] - == [ - "response.created", - "response.output_item.added", - "response.function_call_arguments.delta", - "response.output_item.done", - "response.completed", +def _function_calls_of(response: httpx.Response, stream: bool) -> list[dict[str, object]]: + items: Final = ( + response.json()["output"] + if not stream + else [ + event["item"] + for event in map(json.loads, _data_lines(response.content)) + if event["type"] == "response.output_item.done" ] ) - restored: Final = {"name": "sleep", "namespace": "clock"} - for native_line, v1_line, native_event, v1_event in zip(native_lines, v1_lines, native_events, v1_events): - match v1_event["type"]: - case "response.output_item.added" | "response.output_item.done": - assert native_event == {**v1_event, "item": {**v1_event["item"], **restored}} - case "response.completed": - (v1_call,) = v1_event["response"]["output"] - assert native_event == { - **v1_event, - "response": {**v1_event["response"], "output": [{**v1_call, **restored}]}, - } - case _: - assert native_line == v1_line + return [item for item in items if item["type"] == "function_call"] + + +@pytest.mark.parametrize("stream", [False, True]) +def test_codex_turn_for_xai_goes_through_chat_completions_and_its_namespaced_call_comes_back_whole( + client: TestClient, respx_mock: respx.MockRouter, xai_fireworks_and_openai_router: Router, stream: bool +) -> None: + _account_catalog(respx_mock) + responses_endpoint: Final = respx_mock.post(f"{_XAI_BASE}/responses").respond(json=_provider_completed("Hi")) + chat_endpoint: Final = respx_mock.post(f"{_XAI_BASE}/chat/completions").mock(side_effect=_xai_search_call_reply) + response: Final = client.post( + "/backend-api/codex/responses", + json={ + "model": "grok-4.7", + "input": [_ADDITIONAL_TOOLS, _USER_TURN], + "tools": [_EXEC_TOOL, _CUSTOM_EXEC_TOOL], + "stream": stream, + "store": False, + }, + headers=_ACCOUNT_HEADERS, + ) + assert response.status_code == 200, response.text + assert not responses_endpoint.called + sent_tools: Final = json.loads(chat_endpoint.calls.last.request.content)["tools"] + assert [(tool["type"], tool["function"]["name"]) for tool in sent_tools] == [ + ("function", "exec_command"), + ("function", "exec"), + ("function", "collaboration__followup_task"), + ("function", "mcp__docs____search"), + ] + (call,) = _function_calls_of(response, stream) + assert (call["name"], call["namespace"], call["call_id"]) == ("search", "mcp__docs__", "call_search") def test_encrypted_reasoning_stays_only_when_every_deployment_the_router_may_pick_can_decrypt_it( From 69ceef4c3354052900731c758a0f598c7ff8a806 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:04:33 -0700 Subject: [PATCH 10/20] fix(proxy): drop reasoning ids a Codex turn's LiteLLM target cannot resolve A reasoning item another deployment produced, replayed with store false and no blob the target can decrypt, now reaches the target as its readable summary without id or status, and is dropped when nothing readable is left. OpenAI answered 400 on the status field and 404 on the id for a glm-5p3 item replayed whole to an openai deployment. Same-deployment replays and stored conversations keep the item whole --- .../codex_passthrough_endpoints.py | 12 +- litellm/responses/utils.py | 7 +- .../test_codex_passthrough_endpoints.py | 109 ++++++++++++++++++ 3 files changed, 126 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py index 7225c1fbf7e..988f5a1108e 100644 --- a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py @@ -41,6 +41,7 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import pass_thr from litellm.proxy.response_api_endpoints.endpoints import responses_api from litellm.responses.additional_tools import with_additional_tools_hoisted from litellm.responses.litellm_completion_transformation.reasoning_items import is_other_provider_reasoning_item +from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import EncryptedContentAffinityCheck from litellm.types.llms.custom_http import httpxSpecialProvider from litellm.types.router import Deployment @@ -359,7 +360,16 @@ def _with_reasoning_the_targets_can_decrypt( EncryptedContentAffinityCheck.strip_reasoning_the_targets_cannot_decrypt( llm_router, kept, None, group, unmarked_origin=None ) - return {**body, "input": kept} + if body.get("store") is True: + return {**body, "input": kept} + resolvable: Final = (_without_unpersisted_reasoning_reference(item) for item in kept) + return {**body, "input": [item for item in resolvable if item is not None]} + + +def _without_unpersisted_reasoning_reference(item: object) -> object | None: + if not _is_unpersisted_reasoning_reference(item): + return item + return ResponsesAPIRequestUtils.readable_reasoning(_JSON_OBJECT.validate_python(item)) def _model_and_provider(deployment: Deployment) -> tuple[str, str] | None: diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 05d6a36c1f6..fe8ba29ce0e 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -43,6 +43,7 @@ from litellm.types.utils import ( _PROVIDER_SPECIFIC_HEADER_ADAPTER: Final[ TypeAdapter[ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None] ] = TypeAdapter(ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None) +_REASONING_FIELDS_ONLY_ITS_ORIGIN_RESOLVES: Final = frozenset({"encrypted_content", "id", "status"}) class _UsageExtraFields(ExtraItemsTypedDict, total=False, extra_items=object): @@ -685,13 +686,17 @@ class ResponsesAPIRequestUtils: reasoning: Final = cast(Mapping[str, object], item) # cast-ok: untyped client json if reasoning.get("type") != "reasoning" or not reasoning.get("encrypted_content"): return reasoning + return ResponsesAPIRequestUtils.readable_reasoning(reasoning) + + @staticmethod + def readable_reasoning(reasoning: Mapping[str, object]) -> Mapping[str, object] | None: readable: Final = any( ResponsesAPIRequestUtils._has_readable_text(reasoning.get(key)) for key in ("summary", "content") ) if not readable: return None kept: Final[dict[str, object]] = { # mutable-ok: request item rebuilt without the undecryptable keys - key: value for key, value in reasoning.items() if key not in ("encrypted_content", "id") + key: value for key, value in reasoning.items() if key not in _REASONING_FIELDS_ONLY_ITS_ORIGIN_RESOLVES } return kept diff --git a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py index 08c35989262..158d4dffee2 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py @@ -1159,6 +1159,115 @@ def test_litellm_model_on_native_route_gets_only_the_reasoning_it_can_decrypt( ) +_WHOLE_FIREWORKS_REASONING: Final = {**_FIREWORKS_REASONING, "status": "completed"} +_SUMMARY_ONLY_REASONING: Final = {key: value for key, value in _FIREWORKS_REASONING.items() if key != "id"} +_OTHER_DEPLOYMENTS_BLOB: Final = ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id( + "gAAAAB-other-deployment-blob", "other-deployment" +) + + +@pytest.mark.parametrize( + ("model", "replayed", "store", "expected"), + [ + pytest.param( + "oai-luna", _WHOLE_FIREWORKS_REASONING, False, [_SUMMARY_ONLY_REASONING], id="whole fireworks item to openai" + ), + pytest.param( + "oai-luna", + {**_WHOLE_FIREWORKS_REASONING, "encrypted_content": _OTHER_DEPLOYMENTS_BLOB}, + False, + [_SUMMARY_ONLY_REASONING], + id="whole item with another deployment's blob to openai", + ), + pytest.param( + "oai-luna", + {**_SUMMARY_ONLY_REASONING, "content": [], "encrypted_content": _OTHER_DEPLOYMENTS_BLOB}, + False, + [{**_SUMMARY_ONLY_REASONING, "content": []}], + id="codex's replay of another deployment's item to openai", + ), + pytest.param( + "oai-luna", + {**_SUMMARY_ONLY_REASONING, "content": None, "encrypted_content": None}, + False, + [_SUMMARY_ONLY_REASONING], + id="codex's replay of a blobless item to openai", + ), + pytest.param( + "oai-luna", + {"type": "reasoning", "id": "rs_unreadable", "summary": []}, + False, + [], + id="an unresolvable id with nothing readable is dropped", + ), + pytest.param( + "open-weights", + {**_SUMMARY_ONLY_REASONING, "id": "rs_openai"}, + False, + [_SUMMARY_ONLY_REASONING], + id="openai summary-only item to fireworks", + ), + pytest.param( + "oai-luna", + {**_LUNA_REASONING, "id": "rs_luna", "content": []}, + False, + [{**_LUNA_REASONING, "id": "rs_luna", "content": [], "encrypted_content": _LUNA_BLOB}], + id="the minting deployment gets its item whole", + ), + pytest.param( + "oai-luna", _FIREWORKS_REASONING, True, [_FIREWORKS_REASONING], id="a stored conversation keeps the id" + ), + ], +) +def test_reasoning_the_target_cannot_resolve_reaches_it_without_its_id( + client: TestClient, + respx_mock: respx.MockRouter, + xai_fireworks_and_openai_router: Router, + model: str, + replayed: dict[str, object], + store: bool, + expected: list[dict[str, object]], +) -> None: + _account_catalog(respx_mock) + fireworks: Final = _open_weights_provider(respx_mock) + luna: Final = respx_mock.post(f"{_OPENAI_BASE}/responses").mock(side_effect=_provider_reply) + response: Final = client.post( + "/backend-api/codex/responses", + json={ + "model": model, + "input": [_USER_TURN, replayed, _ASSISTANT_TURN, _USER_TURN], + "stream": False, + "store": store, + }, + headers=_ACCOUNT_HEADERS, + ) + assert response.status_code == 200, response.text + sent: Final = json.loads((fireworks if model == "open-weights" else luna).calls.last.request.content)["input"] + assert [item for item in sent if item.get("type") == "reasoning"] == expected + + +def test_reasoning_the_chat_bridge_target_cannot_resolve_reaches_it_as_its_summary( + client: TestClient, respx_mock: respx.MockRouter, xai_fireworks_and_openai_router: Router +) -> None: + _account_catalog(respx_mock) + chat_endpoint: Final = respx_mock.post(f"{_XAI_BASE}/chat/completions").mock(side_effect=_xai_search_call_reply) + response: Final = client.post( + "/backend-api/codex/responses", + json={ + "model": "grok-4.7", + "input": [_USER_TURN, _WHOLE_FIREWORKS_REASONING, _ASSISTANT_TURN, _USER_TURN], + "stream": False, + "store": False, + }, + headers=_ACCOUNT_HEADERS, + ) + assert response.status_code == 200, response.text + messages: Final = json.loads(chat_endpoint.calls.last.request.content)["messages"] + assert [message.get("reasoning_content") for message in messages if message["role"] == "assistant"] == [ + "Thinking it over" + ] + + def _namespace(name: str, *members: str) -> dict[str, object]: return { "type": "namespace", From 0ad76f27da7badaa0d4fcbc6c3b73ef83e0b098b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 01:51:26 -0700 Subject: [PATCH 11/20] refactor(codex): reach encrypted reasoning stripping through the router registry The Codex route read Router.optional_callbacks and imported EncryptedContentAffinityCheck from litellm.router_utils, which the router surface fence forbids outside the router. RouterRegistry gains decryptable_reasoning(model_group, request_input, targets), which keeps the same rule: with encrypted content affinity on for the group, reasoning one target can decrypt stays, otherwise every target must decrypt it --- .../codex_passthrough_endpoints.py | 20 +---- litellm/router_api.py | 28 +++++++ .../test_router/test_router_entry_points.py | 75 +++++++++++++++++++ 3 files changed, 105 insertions(+), 18 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py index 38effce735f..a71a9e0755d 100644 --- a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py @@ -43,7 +43,6 @@ from litellm.proxy.response_api_endpoints.endpoints import responses_api from litellm.responses.additional_tools import with_additional_tools_hoisted from litellm.responses.litellm_completion_transformation.reasoning_items import is_other_provider_reasoning_item from litellm.responses.utils import ResponsesAPIRequestUtils -from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import EncryptedContentAffinityCheck from litellm.types.llms.custom_http import httpxSpecialProvider from litellm.types.router import Deployment from litellm.types.utils import LlmProviders @@ -339,13 +338,6 @@ def _with_wrapped_reasoning(body: Mapping[str, object]) -> Mapping[str, object]: } -def _router_pins_encrypted_reasoning(model: str, llm_router: "Router") -> bool: - return any( - isinstance(callback, EncryptedContentAffinityCheck) and callback.is_enabled_for_model_group(model) - for callback in llm_router.optional_callbacks or () - ) - - def _with_reasoning_the_targets_can_decrypt( body: Mapping[str, object], model: str, llm_router: "Router", deployments: tuple[Deployment, ...] ) -> Mapping[str, object]: @@ -353,17 +345,9 @@ def _with_reasoning_the_targets_can_decrypt( items: Final = _INPUT_ITEMS.validate_python(body.get("input")) except ValidationError: return body - targets: Final = tuple(deployment.model_dump(exclude_none=True) for deployment in deployments) - target_groups: Final = ( - (targets,) if _router_pins_encrypted_reasoning(model, llm_router) else tuple((target,) for target in targets) - ) - kept: Final = list(items) - for group in target_groups: - EncryptedContentAffinityCheck.strip_reasoning_the_targets_cannot_decrypt( - llm_router, kept, None, group, unmarked_origin=None - ) + kept: Final = llm_router.registry().decryptable_reasoning(model, items, deployments) if body.get("store") is True: - return {**body, "input": kept} + return {**body, "input": list(kept)} resolvable: Final = (_without_unpersisted_reasoning_reference(item) for item in kept) return {**body, "input": [item for item in resolvable if item is not None]} diff --git a/litellm/router_api.py b/litellm/router_api.py index d62999cf261..0254adcb785 100644 --- a/litellm/router_api.py +++ b/litellm/router_api.py @@ -931,6 +931,34 @@ class RouterRegistry: return r._core_prompt_caching_pin_cache raise ValueError(f"unknown plugin kind {kind!r}") + def decryptable_reasoning( + self, model_group: str, request_input: Sequence[object], targets: "Sequence[Deployment]" + ) -> tuple[object, ...]: + """ + ``request_input`` without the encrypted reasoning that ``targets``, the deployments a + ``model_group`` request may go to, cannot decrypt, keeping each item's readable summary. + When encrypted content affinity is on for the group, the router sends the request to the + reasoning's origin, so reasoning one target can decrypt stays; otherwise every target must + decrypt it (was reading ``optional_callbacks`` for ``EncryptedContentAffinityCheck``). + """ + from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( + EncryptedContentAffinityCheck, + ) + + r: Final = self._router + target_dicts: Final = tuple(target.model_dump(exclude_none=True) for target in targets) + pinned: Final = any( + isinstance(callback, EncryptedContentAffinityCheck) and callback.is_enabled_for_model_group(model_group) + for callback in r.optional_callbacks or () + ) + target_groups: Final = (target_dicts,) if pinned else tuple((target,) for target in target_dicts) + kept: Final = list(request_input) + for group in target_groups: + EncryptedContentAffinityCheck.strip_reasoning_the_targets_cannot_decrypt( + r, kept, None, group, unmarked_origin=None + ) + return tuple(kept) + # Settings a caller may read. Assigning one from outside the router goes through apply(ChangeSet(assign=...)). SETTING_FIELDS: Final[tuple[str, ...]] = ( diff --git a/tests/unit/test_router/test_router_entry_points.py b/tests/unit/test_router/test_router_entry_points.py index 2af7db930c6..07562f7930e 100644 --- a/tests/unit/test_router/test_router_entry_points.py +++ b/tests/unit/test_router/test_router_entry_points.py @@ -827,3 +827,78 @@ def test_attribute_errors_about_other_objects_pass_through_unchanged(router): other = AttributeError("'dict' object has no attribute 'x'", name="x", obj={}) assert _public_attribute_error(router, "_core_x", "x", other) is other + + +_MINTING_DEPLOYMENT = "dep-minting-org" +_OTHER_ORG_DEPLOYMENT = "dep-other-org" +_BOTH_ORGS = (_MINTING_DEPLOYMENT, _OTHER_ORG_DEPLOYMENT) +_GROUP_AFFINITY = ["encrypted_content_affinity"] + + +@pytest.mark.parametrize( + "affinity, target_ids, reasoning_stays", + [ + pytest.param({}, _BOTH_ORGS, False, id="no affinity, one target cannot decrypt"), + pytest.param({}, (_MINTING_DEPLOYMENT,), True, id="no affinity, the only target minted it"), + pytest.param( + {"optional_pre_call_checks": _GROUP_AFFINITY}, _BOTH_ORGS, True, id="global affinity, origin among targets" + ), + pytest.param( + {"optional_pre_call_checks": _GROUP_AFFINITY}, + (_OTHER_ORG_DEPLOYMENT,), + False, + id="global affinity, origin not among targets", + ), + pytest.param( + {"optional_pre_call_checks": [], "model_group_affinity_config": {"pair": _GROUP_AFFINITY}}, + _BOTH_ORGS, + True, + id="affinity on for this group", + ), + pytest.param( + {"optional_pre_call_checks": [], "model_group_affinity_config": {"other-group": _GROUP_AFFINITY}}, + _BOTH_ORGS, + False, + id="affinity on for another group only", + ), + ], +) +def test_decryptable_reasoning_keeps_encrypted_reasoning_only_when_the_router_sends_it_to_a_deployment_that_decrypts_it( + affinity, target_ids, reasoning_stays +): + from litellm.responses.utils import ResponsesAPIRequestUtils + + r = Router( + model_list=[ + { + "model_name": "pair", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_base": api_base, "api_key": "sk-pair"}, + "model_info": {"id": model_id}, + } + for api_base, model_id in ( + ("https://minting-org.test/v1", _MINTING_DEPLOYMENT), + ("https://other-org.test/v1", _OTHER_ORG_DEPLOYMENT), + ) + ], + **affinity, + ) + summary = [{"type": "summary_text", "text": "multiply 17 by 23"}] + user_turn = {"type": "message", "role": "user", "content": "What is 17*23?"} + reasoning = { + "type": "reasoning", + "id": "rs_minted", + "encrypted_content": ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id( + "gAAAAA-blob", _MINTING_DEPLOYMENT + ), + "summary": summary, + } + request_input = [user_turn, reasoning] + targets = tuple(r.registry().deployment(model_id) for model_id in target_ids) + try: + kept = r.registry().decryptable_reasoning("pair", request_input, targets) + finally: + r.retire() + + readable_only = {"type": "reasoning", "summary": summary} + assert kept == (user_turn, reasoning if reasoning_stays else readable_only) + assert request_input == [user_turn, reasoning], "the caller's input must not change" From eda8ac76d96fba08cc6a2b6773dd06a62fb5bd5d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 02:17:38 -0700 Subject: [PATCH 12/20] fix(proxy): keep Codex's priority tier off LiteLLM rows in the native model catalog --- litellm/proxy/common_utils/codex_model_catalog.py | 2 +- .../proxy/common_utils/test_codex_model_catalog.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/common_utils/codex_model_catalog.py b/litellm/proxy/common_utils/codex_model_catalog.py index 8412e612408..33c99f178bf 100644 --- a/litellm/proxy/common_utils/codex_model_catalog.py +++ b/litellm/proxy/common_utils/codex_model_catalog.py @@ -542,7 +542,7 @@ def merged_codex_catalog( slugs: Final = frozenset(model.get("slug") for model in upstream) added: Final = tuple(row for row in rows if _codex_can_drive(row.id, row.mode) and row.id not in slugs) first_priority: Final = max((_priority(model) for model in upstream), default=-1) + 1 - entries: Final = _entries(added, stock, instructions, first_priority, "codex", client_version) + entries: Final = _entries(added, stock, instructions, first_priority, "deployment", client_version) added_models: Final = tuple(_entry_json(entry) for entry in entries) serialized: Final = tuple(_compact_json(model) for model in added_models) budget: Final = byte_limit - len(_compact_json({**top, "models": upstream})) diff --git a/tests/unit/proxy/common_utils/test_codex_model_catalog.py b/tests/unit/proxy/common_utils/test_codex_model_catalog.py index f5749374f25..b612047ed1e 100644 --- a/tests/unit/proxy/common_utils/test_codex_model_catalog.py +++ b/tests/unit/proxy/common_utils/test_codex_model_catalog.py @@ -550,6 +550,20 @@ def test_merged_catalog_keeps_upstream_whole_and_adds_only_the_entries_that_fit_ assert merged(*rows, byte_limit=upstream_only) is None +def test_merged_catalog_gives_an_alias_no_service_tier_or_upgrade_of_codexs_own(): + upstream = json.dumps({"models": [{"slug": "upstream-model", "priority": 0}]}).encode() + rows = (_row("team-sol", upstream_model="openai/gpt-6-sol"), _row("team-55", upstream_model="openai/gpt-5.5")) + + added = json.loads(merged_codex_catalog(upstream, rows, stock=_STOCK, instructions=_PROMPT))["models"][1:] + + assert [ + (entry["slug"], entry["service_tiers"], entry["default_service_tier"], entry["upgrade"]) for entry in added + ] == [ + ("team-sol", [], None, None), + ("team-55", [], None, None), + ] + + def test_merged_catalog_warns_once_with_how_many_models_the_byte_limit_left_out(caplog, monkeypatch): upstream = json.dumps({"models": [{"slug": "upstream-model", "priority": 0}]}).encode() rows = tuple(_row(f"model-{index}") for index in range(4)) From 68248bf96694971d7ec5f6983d51ab7b1ccb6c7c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:59:56 -0800 Subject: [PATCH 13/20] fix(proxy): drop Codex's none effort for an alias whose native entry names no default --- litellm/proxy/common_utils/codex_model_catalog.py | 12 +++++++++--- .../proxy/common_utils/test_codex_model_catalog.py | 13 +++++++------ .../test_codex_passthrough_endpoints.py | 7 +++++-- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/common_utils/codex_model_catalog.py b/litellm/proxy/common_utils/codex_model_catalog.py index 33c99f178bf..f2177c5209a 100644 --- a/litellm/proxy/common_utils/codex_model_catalog.py +++ b/litellm/proxy/common_utils/codex_model_catalog.py @@ -170,6 +170,7 @@ class CodexStockCatalog(LiteLLMBaseModel): CodexCatalogEntry = CodexFallbackModel | CodexStockModel CodexAliasDefaults: TypeAlias = Literal["codex", "deployment"] +_NATIVE_ALIAS_DEFAULTS: Final[CodexAliasDefaults] = "deployment" @dataclass(frozen=True, slots=True) @@ -542,7 +543,7 @@ def merged_codex_catalog( slugs: Final = frozenset(model.get("slug") for model in upstream) added: Final = tuple(row for row in rows if _codex_can_drive(row.id, row.mode) and row.id not in slugs) first_priority: Final = max((_priority(model) for model in upstream), default=-1) + 1 - entries: Final = _entries(added, stock, instructions, first_priority, "deployment", client_version) + entries: Final = _entries(added, stock, instructions, first_priority, _NATIVE_ALIAS_DEFAULTS, client_version) added_models: Final = tuple(_entry_json(entry) for entry in entries) serialized: Final = tuple(_compact_json(model) for model in added_models) budget: Final = byte_limit - len(_compact_json({**top, "models": upstream})) @@ -650,11 +651,16 @@ def codex_model_list_body( def codex_entry_names_default_reasoning_level( model_id: str, llm_router: Router, team_id: str | None, client_version: str | None ) -> bool: - """Whether the catalog entry for `model_id` names a `default_reasoning_level`. Codex's model picker + """Whether the native catalog entry for `model_id` names a `default_reasoning_level`. Codex's model picker sends reasoning effort `none` for a model whose entry names none, while showing that choice as "default".""" listed: Final[ModelInfoResponse] = {"id": model_id, "object": "model", "created": 0, "owned_by": ""} (entry,) = _entries( - (_catalog_row(listed, model_id, model_id, llm_router, team_id),), None, "", 0, "codex", client_version + (_catalog_row(listed, model_id, model_id, llm_router, team_id),), + None, + "", + 0, + _NATIVE_ALIAS_DEFAULTS, + client_version, ) return _entry_json(entry).get("default_reasoning_level") is not None diff --git a/tests/unit/proxy/common_utils/test_codex_model_catalog.py b/tests/unit/proxy/common_utils/test_codex_model_catalog.py index b612047ed1e..6db3e3044f2 100644 --- a/tests/unit/proxy/common_utils/test_codex_model_catalog.py +++ b/tests/unit/proxy/common_utils/test_codex_model_catalog.py @@ -809,7 +809,7 @@ def test_catalog_rows_without_a_router_carry_only_the_listing(): @pytest.mark.parametrize("client_version", [None, "0.165.0"]) -def test_entry_names_a_default_reasoning_level_exactly_when_the_served_entry_carries_one(client_version): +def test_entry_names_a_default_reasoning_level_exactly_when_the_native_entry_carries_one(client_version): router = litellm.Router( model_list=[ {"model_name": "open-weights", "litellm_params": {"model": "fireworks_ai/open-weights"}}, @@ -825,14 +825,15 @@ def test_entry_names_a_default_reasoning_level_exactly_when_the_served_entry_car listing = tuple( {"id": name, "object": "model", "created": 0, "owned_by": "openai", "mode": "chat"} for name in names ) - served = json.loads( - codex_model_list_body(listing, tuple((name, name) for name in names), router, client_version=client_version) - )["models"] + upstream = json.dumps({"models": [{"slug": "upstream-model", "priority": 0}]}).encode() + rows = codex_catalog_rows(listing, tuple((name, name) for name in names), router) + served = json.loads(merged_codex_catalog(upstream, rows, client_version=client_version))["models"][1:] + served_names_one = [entry.get("default_reasoning_level") is not None for entry in served] assert [ codex_entry_names_default_reasoning_level(entry["slug"], router, None, client_version) for entry in served - ] == ["default_reasoning_level" in entry for entry in served] - assert ["default_reasoning_level" in entry for entry in served] == [False, True, client_version is not None] + ] == served_names_one + assert served_names_one == [False, False, client_version is not None] def test_codex_tool_search_is_offered_off_the_route_and_model_info_and_only_when_every_deployment_answers_it(): diff --git a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py index 49b781c47f2..0919fb72750 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py @@ -477,7 +477,7 @@ def test_litellm_model_on_native_route_keeps_chatgpt_credentials_out_of_hook_raw pytest.param("open-weights", "none", "deployment", None, id="none under the deployment's drop_params"), pytest.param("open-weights", "none", "request", None, id="none under the request's drop_params"), pytest.param("open-weights", "high", "global", "high", id="effort set for a model without a default effort"), - pytest.param("coding-model", "none", "global", "none", id="none on a model whose entry names a default effort"), + pytest.param("coding-model", "none", "global", None, id="none on an alias whose native entry names no effort"), pytest.param("api-model", "none", "global", "none", id="none on a model that supports it"), pytest.param("glm-model", "none", "global", None, id="none on a model that does not support it"), pytest.param("declared-model", "none", "global", "none", id="none on a deployment whose model_info allows it"), @@ -503,7 +503,10 @@ def test_litellm_model_on_native_route_gets_the_reasoning_effort_codex_m_would_s **open_weights, "litellm_params": {**open_weights["litellm_params"], "drop_params": drop_params == "deployment"}, }, - _open_weights("coding-model", upstream="gpt-5.5"), + { + **_open_weights("coding-model", upstream="gpt-5.5"), + "model_info": {"supports_none_reasoning_effort": False}, + }, _open_weights("glm-model", upstream="glm-5p3"), {**_open_weights("declared-model"), "model_info": {"supports_none_reasoning_effort": True}}, { From 1959361111b39e88f7785c6559f9a937bd73ee3d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 1 Nov 2026 22:14:28 -0800 Subject: [PATCH 14/20] fix(proxy): start a native Codex alias at its pinned reasoning effort and build the catalog off the event loop --- litellm/proxy/common_utils/codex_model_catalog.py | 10 ++++++++++ .../codex_passthrough_endpoints.py | 5 ++++- litellm/proxy/proxy_server.py | 4 +++- litellm/router_api.py | 6 +++++- litellm/router_python_core.py | 14 ++++++++++++++ .../proxy/common_utils/test_codex_model_catalog.py | 8 ++++++-- 6 files changed, 42 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/common_utils/codex_model_catalog.py b/litellm/proxy/common_utils/codex_model_catalog.py index f2177c5209a..048ed69f5bd 100644 --- a/litellm/proxy/common_utils/codex_model_catalog.py +++ b/litellm/proxy/common_utils/codex_model_catalog.py @@ -584,6 +584,11 @@ def _group_reasoning_efforts( return group.supported_reasoning_efforts if group is not None else None +def _shared_pinned_effort(pinned: tuple[object, ...]) -> str | None: + first: Final = next(iter(pinned), None) + return first if isinstance(first, str) and all(effort == first for effort in pinned) else None + + def _catalog_row( row: ModelInfoResponse, lookup_id: str, label_id: str, llm_router: Router | None, team_id: str | None ) -> CodexCatalogRow: @@ -594,6 +599,11 @@ def _catalog_row( max_input_tokens=row.get("max_input_tokens"), upstream_model=registry.resolve(lookup_id, kind="upstream", team_id=team_id) if registry is not None else None, display_name=registry.group(label_id, field="display_name") if registry is not None else None, + reasoning_effort=( + _shared_pinned_effort(registry.group(lookup_id, field="pinned_reasoning_efforts", team_id=team_id)) + if registry is not None + else None + ), service_tiers=registry.group(lookup_id, field="service_tiers", team_id=team_id) if registry is not None else (), supported_reasoning_efforts=_group_reasoning_efforts(row, lookup_id, registry, team_id), search_tool_support=( diff --git a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py index a71a9e0755d..8a61a116877 100644 --- a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py @@ -1,3 +1,4 @@ +import asyncio import hashlib import hmac import json @@ -567,7 +568,9 @@ async def _catalog( etag: Final = None if upstream_etag is None else await _remembered_models_etag(caller, upstream_etag, rows, cache) if not rows: return upstream - merged: Final = merged_codex_catalog(upstream_body, rows, client_version=request.query_params.get("client_version")) + merged: Final = await asyncio.to_thread( + merged_codex_catalog, upstream_body, rows, client_version=request.query_params.get("client_version") + ) return _with_etag(upstream, merged if merged is not None else upstream_body, etag) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 7ba46b76286..151981adf6a 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -12501,7 +12501,9 @@ async def codex_catalog_rows_for_key(user_api_key_dict: UserAPIKeyAuth) -> tuple await _caller_aliases(user_api_key_dict, None), await _listing_hidden_names(None), ) - return codex_catalog_rows(listing.rows, listing.entries, llm_router, user_api_key_dict.team_id, listing.labels) + return await asyncio.to_thread( + codex_catalog_rows, listing.rows, listing.entries, llm_router, user_api_key_dict.team_id, listing.labels + ) @router.get("/v1/models", dependencies=[Depends(user_api_key_auth)], tags=["model management"]) diff --git a/litellm/router_api.py b/litellm/router_api.py index 993731b7f3a..d2f0858f23c 100644 --- a/litellm/router_api.py +++ b/litellm/router_api.py @@ -155,6 +155,7 @@ GroupField: TypeAlias = Literal[ "display_name", "mode", "service_tiers", + "pinned_reasoning_efforts", "token_limits", "listing_info", ] @@ -850,7 +851,7 @@ class RouterRegistry: self, name: str, *, - field: Literal["service_tiers"], + field: Literal["service_tiers", "pinned_reasoning_efforts"], include_hidden: bool = False, team_id: str | None = None, ) -> tuple[object, ...]: ... @@ -894,6 +895,7 @@ class RouterRegistry: (were ``get_configured_display_name``, ``get_configured_mode``, ``get_configured_token_limits``, ``get_model_listing_info``) - ``service_tiers``: (was ``get_configured_service_tiers``) + - ``pinned_reasoning_efforts``: the ``reasoning_effort`` each routable deployment pins """ r: Final = self._router if field == "info": @@ -906,6 +908,8 @@ class RouterRegistry: return r._core_get_configured_mode(name) if field == "service_tiers": return r._core_get_configured_service_tiers(name, team_id=team_id) + if field == "pinned_reasoning_efforts": + return r._core_get_pinned_reasoning_efforts(name, team_id=team_id) if field == "token_limits": return r._core_get_configured_token_limits(name) if field == "listing_info": diff --git a/litellm/router_python_core.py b/litellm/router_python_core.py index 30b05456987..a4b544756d2 100644 --- a/litellm/router_python_core.py +++ b/litellm/router_python_core.py @@ -12680,6 +12680,20 @@ class PythonCore(DeprecatedRouterMembers): for deployment in self._routable_deployments(model_name, team_id) ) + def _core_get_pinned_reasoning_efforts(self, model_name: str, team_id: str | None = None) -> tuple[object, ...]: + """ + Return the litellm_params.reasoning_effort each deployment a request + from team_id to model_name can route to (see _routable_deployments) + pins, unvalidated and in model_list order, None for a deployment that + pins none; the caller decides how the deployments combine. + + Returns an empty tuple for wildcard-expanded or unknown names. + """ + return tuple( + _LITELLM_PARAMS_ADAPTER.validate_python(deployment["litellm_params"]).get("reasoning_effort") + for deployment in self._routable_deployments(model_name, team_id) + ) + def _core_get_search_tool_support(self, model_name: str, team_id: str | None = None) -> tuple[bool | None, ...]: """ Whether each deployment a request from team_id to model_name can route diff --git a/tests/unit/proxy/common_utils/test_codex_model_catalog.py b/tests/unit/proxy/common_utils/test_codex_model_catalog.py index 6db3e3044f2..750d8a5901d 100644 --- a/tests/unit/proxy/common_utils/test_codex_model_catalog.py +++ b/tests/unit/proxy/common_utils/test_codex_model_catalog.py @@ -814,6 +814,9 @@ def test_entry_names_a_default_reasoning_level_exactly_when_the_native_entry_car model_list=[ {"model_name": "open-weights", "litellm_params": {"model": "fireworks_ai/open-weights"}}, {"model_name": "coding-model", "litellm_params": {"model": "openai/gpt-5.5"}}, + {"model_name": "pinned-model", "litellm_params": {"model": "openai/gpt-5.5", "reasoning_effort": "high"}}, + {"model_name": "split-pin", "litellm_params": {"model": "openai/gpt-5.5", "reasoning_effort": "high"}}, + {"model_name": "split-pin", "litellm_params": {"model": "openai/gpt-5.5", "reasoning_effort": "low"}}, { "model_name": "max-only", "litellm_params": {"model": "hosted_vllm/qa-local-coder", "api_base": "https://vllm.example/v1"}, @@ -821,7 +824,7 @@ def test_entry_names_a_default_reasoning_level_exactly_when_the_native_entry_car }, ] ) - names = ("open-weights", "coding-model", "max-only") + names = ("open-weights", "coding-model", "pinned-model", "split-pin", "max-only") listing = tuple( {"id": name, "object": "model", "created": 0, "owned_by": "openai", "mode": "chat"} for name in names ) @@ -833,7 +836,8 @@ def test_entry_names_a_default_reasoning_level_exactly_when_the_native_entry_car assert [ codex_entry_names_default_reasoning_level(entry["slug"], router, None, client_version) for entry in served ] == served_names_one - assert served_names_one == [False, False, client_version is not None] + assert served_names_one == [False, False, True, False, client_version is not None] + assert served[2]["default_reasoning_level"] == "high" def test_codex_tool_search_is_offered_off_the_route_and_model_info_and_only_when_every_deployment_answers_it(): From 19107fb361d7003ff7fb850507427d16d19fa16a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 1 Nov 2026 23:08:33 -0800 Subject: [PATCH 15/20] test(proxy): map the pinned reasoning efforts field in the router registry stub --- tests/unit/proxy/router_entry_point_stubs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/proxy/router_entry_point_stubs.py b/tests/unit/proxy/router_entry_point_stubs.py index 6912049d225..4fdcea43fda 100644 --- a/tests/unit/proxy/router_entry_point_stubs.py +++ b/tests/unit/proxy/router_entry_point_stubs.py @@ -123,6 +123,8 @@ def route_entry_points_to_stubs(router: _RouterDouble) -> _RouterDouble: return router.get_configured_display_name(name) if field == "service_tiers": return router.get_configured_service_tiers(name, **_given(team_id=team_id)) + if field == "pinned_reasoning_efforts": + return router.get_pinned_reasoning_efforts(name, **_given(team_id=team_id)) if field == "listing_info": return router.get_model_listing_info(name) if field == "mode": From 6bf42fc8d85e90986691c9043655f3b3ba0fc317 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 2 Nov 2026 10:35:10 -0800 Subject: [PATCH 16/20] fix(proxy): honor admin-only /v1/responses and keep Codex's default tier off native aliases Codex native routes now send a key to LiteLLM only when admin_only_routes allows it, a native alias with configured service tiers starts with no tier picked, and the account slug cache declares its Redis key family. --- .../proxy/common_utils/codex_model_catalog.py | 7 ++- .../codex_passthrough_endpoints.py | 7 ++- .../common_utils/test_codex_model_catalog.py | 9 ++++ .../test_codex_passthrough_endpoints.py | 46 +++++++++++++++++-- 4 files changed, 63 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/common_utils/codex_model_catalog.py b/litellm/proxy/common_utils/codex_model_catalog.py index 048ed69f5bd..e8b76651129 100644 --- a/litellm/proxy/common_utils/codex_model_catalog.py +++ b/litellm/proxy/common_utils/codex_model_catalog.py @@ -393,6 +393,7 @@ def _entry( stock_model.upgrade if stock_model.upgrade is not None and stock_model.upgrade.model in served else None ) is_alias: Final = row.id != stock_model.slug + deployment_decides: Final = is_alias and alias_defaults == "deployment" stock_name: Final = row.id if is_alias else stock_model.display_name listing: Final = { "slug": row.id, @@ -402,7 +403,7 @@ def _entry( "supported_in_api": True, "upgrade": upgrade, **({} if search_tool is None else {"supports_search_tool": search_tool}), - **(_deployment_defaults(row, stock_model) if is_alias and alias_defaults == "deployment" else {}), + **(_deployment_defaults(row, stock_model) if deployment_decides else {}), } if configured is None: return stock_model.model_copy(update=listing) @@ -412,7 +413,9 @@ def _entry( **listing, "service_tiers": configured, "default_service_tier": ( - stock_model.default_service_tier if stock_model.default_service_tier in configured_ids else None + stock_model.default_service_tier + if not deployment_decides and stock_model.default_service_tier in configured_ids + else None ), } ) diff --git a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py index 8a61a116877..f2c24728bba 100644 --- a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py @@ -14,6 +14,7 @@ from pydantic import TypeAdapter, ValidationError from starlette.types import Message import litellm +from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params @@ -22,7 +23,7 @@ from litellm.llms.chatgpt.common_utils import CHATGPT_API_BASE from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared client factory has untyped params ) -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.auth_checks import check_model_access_helper from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -75,6 +76,7 @@ _SHARED_CACHE_TTL_SECONDS: Final = 3600 _ACCOUNT_SLUGS_UNAVAILABLE: Final = "unavailable" _ACCOUNT_SLUGS_UNAVAILABLE_TTL_SECONDS: Final = 60 _ACCOUNT_SLUGS_CACHE_PREFIX: Final = "codex_account_slugs:" +_ACCOUNT_SLUGS_TARGET: Final = "codex_account_slugs" _MODELS_ETAG_CACHE_PREFIX: Final = "codex_models_etag:" _MODELS_ETAG_TTL_SECONDS: Final = 300 _OPENAI_API_PROVIDERS: Final = frozenset({LlmProviders.OPENAI.value, LlmProviders.AZURE.value}) @@ -171,6 +173,8 @@ def codex_shared_cache() -> _SharedCache: def _may_call_litellm_responses(user_api_key_dict: UserAPIKeyAuth) -> bool: try: _ = RouteChecks.should_call_route(route=_LITELLM_RESPONSES_PATH, valid_token=user_api_key_dict) + if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: + RouteChecks.custom_admin_only_route_check(route=_LITELLM_RESPONSES_PATH) except HTTPException: return False return True @@ -223,6 +227,7 @@ async def _fetched_account_slugs(headers: Mapping[str, str], client_version: str return upstream_codex_slugs(response.content) if response.status_code == 200 else None +@with_service_target(_ACCOUNT_SLUGS_TARGET) async def _account_slugs(headers: Mapping[str, str], cache: _SharedCache) -> frozenset[str] | None: cache_key: Final = _account_slugs_cache_key(headers) if cache_key is None: diff --git a/tests/unit/proxy/common_utils/test_codex_model_catalog.py b/tests/unit/proxy/common_utils/test_codex_model_catalog.py index 750d8a5901d..a3dd1fcd514 100644 --- a/tests/unit/proxy/common_utils/test_codex_model_catalog.py +++ b/tests/unit/proxy/common_utils/test_codex_model_catalog.py @@ -453,6 +453,15 @@ def test_default_service_tier_survives_only_while_still_offered(configured, expe assert entry["default_service_tier"] == expected + +@pytest.mark.parametrize(("alias_defaults", "expected"), [("codex", "priority"), ("deployment", None)]) +def test_alias_with_offered_tiers_takes_codexs_default_tier_only_under_codex_defaults(alias_defaults, expected): + (entry,) = _models( + _row("team-sol", upstream_model="gpt-6-sol", service_tiers=["priority"]), alias_defaults=alias_defaults + ) + + assert (_tiers(entry), entry["default_service_tier"]) == ([("priority", "Fast", "2x speed, increased usage")], expected) + def test_non_chat_and_wildcard_rows_are_left_out_of_the_catalog(): body = _body( _row("gpt-6-astra"), diff --git a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py index 0919fb72750..a297dc11c0f 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py @@ -597,11 +597,15 @@ def test_chatgpt_model_keeps_passthrough_without_other_providers_reasoning( def restricted_key(monkeypatch: pytest.MonkeyPatch) -> Callable[..., str]: monkeypatch.setattr(proxy_server, "prisma_client", "test-database") - def mint(models: list[str], allowed_routes: tuple[str, ...] = ()) -> str: - key: Final = f"sk-test-{'-'.join((*models, *allowed_routes))}" + def mint( + models: list[str], allowed_routes: tuple[str, ...] = (), user_role: LitellmUserRoles | None = None + ) -> str: + key: Final = f"sk-test-{'-'.join((*models, *allowed_routes, str(user_role)))}" proxy_server.user_api_key_cache.set_cache( key=hash_token(key), - value=UserAPIKeyAuth(token=hash_token(key), models=models, allowed_routes=list(allowed_routes)), + value=UserAPIKeyAuth( + token=hash_token(key), models=models, allowed_routes=list(allowed_routes), user_role=user_role + ), ) return key @@ -744,6 +748,42 @@ def test_native_responses_reach_litellm_models_only_for_a_key_that_may_call_v1_r assert chatgpt.called is not through_litellm + +@pytest.mark.parametrize( + ("user_role", "through_litellm"), + [ + pytest.param(LitellmUserRoles.INTERNAL_USER, False, id="non-admin key"), + pytest.param(LitellmUserRoles.PROXY_ADMIN, True, id="proxy admin key"), + ], +) +def test_admin_only_v1_responses_keeps_litellm_models_to_proxy_admins_on_native_routes( + client: TestClient, + respx_mock: respx.MockRouter, + mixed_router: Router, + restricted_key: Callable[..., str], + monkeypatch: pytest.MonkeyPatch, + user_role: LitellmUserRoles, + through_litellm: bool, +) -> None: + monkeypatch.setattr(proxy_server, "general_settings", {"admin_only_routes": ["/v1/responses"]}) + monkeypatch.setattr(proxy_server, "premium_user", True) + _account_catalog(respx_mock) + provider: Final = _open_weights_provider(respx_mock) + chatgpt: Final = _chatgpt_responses(respx_mock) + key: Final = restricted_key(["chatgpt-native", "open-weights"], user_role=user_role) + catalog: Final = client.get( + "/backend-api/codex/models?client_version=1.0.0", headers={**_ACCOUNT_HEADERS, "x-litellm-api-key": key} + ) + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": "open-weights", "input": "Hello", "stream": True}, + headers={**_ACCOUNT_HEADERS, "x-litellm-api-key": key}, + ) + assert (catalog.status_code, response.status_code) == (200, 200), response.text + assert ("open-weights" in [model["slug"] for model in catalog.json()["models"]]) is through_litellm + assert provider.called is through_litellm + assert chatgpt.called is not through_litellm + _USER_TURN: Final = {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Hello"}]} _ASSISTANT_TURN: Final = {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "Hi"}]} _FIREWORKS_REASONING: Final = { From 5c73389c25e1fa8c77fd646a23d4870bb53e4006 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 2 Nov 2026 13:29:07 -0800 Subject: [PATCH 17/20] fix(proxy): check the key's tool allowlist on native Codex LiteLLM turns and additional_tools items --- .../guardrail_translation/handler.py | 10 +++- litellm/proxy/auth/auth_checks.py | 6 +-- .../codex_passthrough_endpoints.py | 8 ++- litellm/responses/additional_tools.py | 7 ++- .../test_codex_passthrough_endpoints.py | 52 +++++++++++++++++-- .../proxy/test_tools_allowlist_enforcement.py | 36 +++++++++++++ 6 files changed, 109 insertions(+), 10 deletions(-) diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index a18cbe13925..27270439b35 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -76,6 +76,7 @@ from litellm.llms.openai.responses.guardrail_translation.tool_merge import ( append_returned_tools, merge_guardrailed_tools, ) +from litellm.responses.additional_tools import additional_tools_in_input from litellm.responses.litellm_completion_transformation.compaction_items import ( compaction_summary, decode_compaction_item, @@ -953,9 +954,14 @@ class OpenAIResponsesHandler(BaseTranslation): def extract_request_tool_names(self, data: Mapping[str, object]) -> list[str]: """Extract tool names from Responses API request (tools[].name for function - and custom, tools[].server_label for mcp).""" + and custom, tools[].server_label for mcp), including tools carried by + ``additional_tools`` input items.""" names: Final[list[str]] = [] - for tool in list_field(data, "tools"): + request_tools: Final[tuple[object, ...]] = ( + *list_field(data, "tools"), + *additional_tools_in_input(data.get("input")), + ) + for tool in request_tools: if not isinstance(tool, dict): continue if tool.get("type") in ("function", "custom") and tool.get("name"): diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index f5d3a80ee91..c8e03dc1136 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -1017,9 +1017,6 @@ async def check_tools_allowlist( call_types: Final = get_call_types_for_route(route) if not call_types or not any(ct.value in TOOL_CAPABLE_CALL_TYPES for ct in call_types): return - tool_names: Final = extract_request_tool_names(route, request_body) - if not tool_names: - return key_meta: Final = (valid_token.metadata or {}) if isinstance(valid_token.metadata, dict) else {} team_meta: Final = (valid_token.team_metadata or {}) if isinstance(valid_token.team_metadata, dict) else {} key_allowed: Final = key_meta.get("allowed_tools") @@ -1027,6 +1024,9 @@ async def check_tools_allowlist( effective: Final = key_allowed if (isinstance(key_allowed, list) and len(key_allowed) > 0) else team_allowed if not isinstance(effective, list) or len(effective) == 0: return + tool_names: Final = extract_request_tool_names(route, request_body) + if not tool_names: + return allowed_set: Final = {str(t) for t in effective} disallowed: Final = [n for n in tool_names if n not in allowed_set] if disallowed: diff --git a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py index f2c24728bba..2f9f78a7ae2 100644 --- a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py @@ -24,7 +24,7 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared client factory has untyped params ) from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth -from litellm.proxy.auth.auth_checks import check_model_access_helper +from litellm.proxy.auth.auth_checks import check_model_access_helper, check_tools_allowlist from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.codex_model_catalog import ( @@ -592,6 +592,12 @@ async def _responses( body: Final = _JSON_OBJECT.validate_python(await read_request_body(request)) if await _serves_through_litellm(body.get("model"), llm_router, user_api_key_dict, headers, cache): litellm_body: Final = _litellm_body(body, llm_router, user_api_key_dict.team_id, headers) + await check_tools_allowlist( + request_body=dict(litellm_body), + valid_token=user_api_key_dict, + team_object=None, + route=_LITELLM_RESPONSES_PATH, + ) return await responses_api( _as_litellm_responses_request(request, json.dumps(litellm_body).encode(), litellm_body), fastapi_response, diff --git a/litellm/responses/additional_tools.py b/litellm/responses/additional_tools.py index bdbeb3bed1e..72263967b30 100644 --- a/litellm/responses/additional_tools.py +++ b/litellm/responses/additional_tools.py @@ -1,7 +1,7 @@ import json from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass -from itertools import groupby +from itertools import chain, groupby from types import MappingProxyType from typing import Final, cast # noqa: TID251 # validating the openai tool union strips vendor keys from raw tools @@ -88,6 +88,11 @@ def _unique_tools(tools: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, o return tuple(_merged(copies_by_identity[identity]) for identity in first_seen_order) +def additional_tools_in_input(input: object) -> tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]: + items: Final = _ITEMS.validate_python(input) if isinstance(input, list) else () + return tuple(chain.from_iterable(_tools_of_item(item) for item in items if _is_additional_tools_item(item))) + + def hoist_additional_tools( input: str | ResponseInputParam, tools: Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None, diff --git a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py index a297dc11c0f..3e5034b68d2 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py @@ -598,13 +598,20 @@ def restricted_key(monkeypatch: pytest.MonkeyPatch) -> Callable[..., str]: monkeypatch.setattr(proxy_server, "prisma_client", "test-database") def mint( - models: list[str], allowed_routes: tuple[str, ...] = (), user_role: LitellmUserRoles | None = None + models: list[str], + allowed_routes: tuple[str, ...] = (), + user_role: LitellmUserRoles | None = None, + allowed_tools: tuple[str, ...] = (), ) -> str: - key: Final = f"sk-test-{'-'.join((*models, *allowed_routes, str(user_role)))}" + key: Final = f"sk-test-{'-'.join((*models, *allowed_routes, str(user_role), *allowed_tools))}" proxy_server.user_api_key_cache.set_cache( key=hash_token(key), value=UserAPIKeyAuth( - token=hash_token(key), models=models, allowed_routes=list(allowed_routes), user_role=user_role + token=hash_token(key), + models=models, + allowed_routes=list(allowed_routes), + user_role=user_role, + metadata={"allowed_tools": list(allowed_tools)} if allowed_tools else {}, ), ) return key @@ -1394,6 +1401,45 @@ def test_codex_additional_tools_item_reaches_each_provider_in_a_shape_it_accepts assert [tool["name"] for tool in sent["tools"]] == expected_tool_names +_FORBIDDEN_TOOL: Final = {"type": "function", "name": "delete_user", "parameters": {"type": "object"}} + + +@pytest.mark.parametrize( + ("tools", "input_items", "allowed"), + [ + pytest.param([_EXEC_TOOL, _FORBIDDEN_TOOL], [_USER_TURN], False, id="a forbidden top-level tool"), + pytest.param( + [_EXEC_TOOL], + [{"type": "additional_tools", "role": "developer", "tools": [_FORBIDDEN_TOOL]}, _USER_TURN], + False, + id="a forbidden tool in an additional_tools item", + ), + pytest.param([_EXEC_TOOL], [_USER_TURN], True, id="only allowed tools"), + ], +) +def test_native_litellm_turn_enforces_the_key_tool_allowlist( + client: TestClient, + respx_mock: respx.MockRouter, + xai_fireworks_and_openai_router: Router, + restricted_key: Callable[..., str], + tools: list[dict[str, object]], + input_items: list[dict[str, object]], + allowed: bool, +) -> None: + _account_catalog(respx_mock) + fireworks: Final = _open_weights_provider(respx_mock) + key: Final = restricted_key( + ["open-weights"], (*_NATIVE_ROUTES, "/v1/responses"), allowed_tools=(str(_EXEC_TOOL["name"]),) + ) + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": "open-weights", "input": input_items, "tools": tools, "stream": False, "store": False}, + headers={**_ACCOUNT_HEADERS, "x-litellm-api-key": key}, + ) + assert response.status_code == (200 if allowed else 403), response.text + assert fireworks.called is allowed + + _ANTHROPIC_BASE: Final = "https://anthropic-api.test" diff --git a/tests/unit/proxy/test_tools_allowlist_enforcement.py b/tests/unit/proxy/test_tools_allowlist_enforcement.py index e306f7be4f8..efaf25c480c 100644 --- a/tests/unit/proxy/test_tools_allowlist_enforcement.py +++ b/tests/unit/proxy/test_tools_allowlist_enforcement.py @@ -242,6 +242,42 @@ class TestCheckToolsAllowlist: assert exc_info.value.type == ProxyErrorTypes.tool_access_denied assert "restricted_tool" in str(exc_info.value.message) + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("allowed_tools", "denied"), + [ + pytest.param(["get_weather"], True, id="forbidden"), + pytest.param(["get_weather", "delete_user"], False, id="allowed"), + ], + ) + async def test_tool_defined_in_an_additional_tools_input_item_is_checked_on_responses_route( + self, allowed_tools: list[str], denied: bool + ) -> None: + body = { + "model": "fireworks_ai/glm-5p3", + "input": [ + { + "type": "additional_tools", + "role": "developer", + "tools": [{"type": "function", "name": "delete_user", "parameters": {"type": "object"}}], + }, + {"type": "message", "role": "user", "content": "Delete user 42"}, + ], + } + check = check_tools_allowlist( + request_body=body, + valid_token=_token(metadata={"allowed_tools": allowed_tools}), + team_object=None, + route="/v1/responses", + ) + if not denied: + await check + return + with pytest.raises(ProxyException) as exc_info: + await check + assert exc_info.value.type == ProxyErrorTypes.tool_access_denied + assert "delete_user" in str(exc_info.value.message) + @pytest.mark.parametrize( "loaded_tool", [ From 9dd7a83f3e83a82d78b43772dc93bea41215a4be Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 2 Nov 2026 14:47:15 -0800 Subject: [PATCH 18/20] fix(proxy): check vector store grants on file_search tools inside additional_tools items A Responses request can define file_search inside an additional_tools input item, which the native Codex route hoists for LiteLLM-served models and OpenAI runs as is on /v1/responses. The vector store check now reads those tools in both strict and legacy modes. The input item scan uses a Mapping guard instead of a pydantic error per item, since auth now runs it on every list input. --- litellm/proxy/auth/auth_checks.py | 19 +++++++----- litellm/responses/additional_tools.py | 16 +++++----- ...st_auth_checks_object_access_and_lookup.py | 31 +++++++++++++++++++ 3 files changed, 51 insertions(+), 15 deletions(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index c8e03dc1136..adb1e1ab88e 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -166,6 +166,7 @@ from litellm.repositories.table_repositories import ( ) from litellm.repositories.team_repository import TeamRepository from litellm.repositories.user_repository import UserRepository +from litellm.responses.additional_tools import additional_tools_in_input from litellm.router import Router from litellm.router_api import RegistryReader from litellm.types.proxy.auth.auth_checks import UserNotFoundError @@ -7954,6 +7955,10 @@ def _tools(tools: object) -> tuple[object, ...]: return () +def _request_tools(request_body: Mapping[str, object]) -> tuple[object, ...]: + return (*_tools(request_body.get("tools")), *additional_tools_in_input(request_body.get("input"))) + + def _tool_vector_store_ids(tool: object) -> tuple[str, ...]: try: tool_fields: Final = _TOOL_ADAPTER.validate_python(tool, strict=True) @@ -7964,14 +7969,14 @@ def _tool_vector_store_ids(tool: object) -> tuple[str, ...]: def _strict_requested_vector_store_ids(request_body: Mapping[str, object]) -> tuple[str, ...]: """ - Same fields VectorStoreRegistry.get_vector_store_ids_to_run reads, but a vector_store_ids that is - not a list of strings is a 400 instead of being skipped or iterated, and tools that are not - objects name no store. + Same fields VectorStoreRegistry.get_vector_store_ids_to_run reads, plus the tools of `additional_tools` + input items, but a vector_store_ids that is not a list of strings is a 400 instead of being skipped or + iterated, and tools that are not objects name no store. """ return tuple( chain( _validated_vector_store_ids(request_body.get("vector_store_ids")), - chain.from_iterable(_tool_vector_store_ids(tool) for tool in _tools(request_body.get("tools"))), + chain.from_iterable(_tool_vector_store_ids(tool) for tool in _request_tools(request_body)), ) ) @@ -8010,7 +8015,7 @@ def _legacy_requested_vector_store_ids( return tuple( chain( top_level, - chain.from_iterable(vector_store_ids_in_tool(tool) for tool in _tools(request_body.get("tools"))), + chain.from_iterable(vector_store_ids_in_tool(tool) for tool in _request_tools(request_body)), ) ) @@ -8079,8 +8084,8 @@ async def vector_store_access_check( """ Checks whether the caller may use every vector store the request names. - Requested stores come from `vector_store_ids`, `tools[].vector_store_ids` and the RAG - `retrieval_config.vector_store_id`. Grants come from each identity's + Requested stores come from `vector_store_ids`, `tools[].vector_store_ids`, the tools of + `additional_tools` input items and the RAG `retrieval_config.vector_store_id`. Grants come from each identity's `object_permission.vector_stores`. Both modes read the same request fields, with or without a vector store registry, and answer 400 diff --git a/litellm/responses/additional_tools.py b/litellm/responses/additional_tools.py index 72263967b30..af6322c62de 100644 --- a/litellm/responses/additional_tools.py +++ b/litellm/responses/additional_tools.py @@ -6,6 +6,7 @@ from types import MappingProxyType from typing import Final, cast # noqa: TID251 # validating the openai tool union strips vendor keys from raw tools from pydantic import TypeAdapter, ValidationError +from typing_extensions import TypeIs # noqa: TID251 # narrows a client input item before its type is read from litellm._logging import verbose_logger from litellm.types.llms.base import LiteLLMBaseModel @@ -16,10 +17,6 @@ _NAMESPACE_TOOL_TYPE: Final = "namespace" _ITEMS: Final = TypeAdapter(tuple[object, ...]) -class _InputItemType(LiteLLMBaseModel): - type: str = "" - - class _ToolList(LiteLLMBaseModel): tools: tuple[Mapping[str, object], ...] = () @@ -31,11 +28,14 @@ class HoistedAdditionalTools: hoisted: tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...] +def _is_an_input_item( + item: object, +) -> TypeIs[Mapping[str, object]]: # guard-ok: auth scans every list input, and a pydantic error per item is slow + return isinstance(item, Mapping) + + def _is_additional_tools_item(item: object) -> bool: - try: - return _InputItemType.model_validate(item).type == ADDITIONAL_TOOLS_INPUT_ITEM_TYPE - except ValidationError: - return False + return _is_an_input_item(item) and item.get("type") == ADDITIONAL_TOOLS_INPUT_ITEM_TYPE def _tool_list_of(value: object) -> tuple[Mapping[str, object], ...]: diff --git a/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py b/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py index 7a060a214f7..a765237cb8b 100644 --- a/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py +++ b/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py @@ -2397,6 +2397,37 @@ async def test_deny_by_default_ignores_tools_that_name_no_vector_store( ) +@pytest.mark.asyncio +@pytest.mark.parametrize("deny_by_default", [False, True], ids=["legacy", "deny-by-default"]) +@pytest.mark.parametrize( + ("key_vector_stores", "denied_by"), + [(["KBSTOREA"], _KEY_DENIED), (["KBSTOREB"], None)], + ids=["ungranted", "granted"], +) +async def test_file_search_inside_additional_tools_input_item_needs_a_vector_store_grant( + deny_by_default: bool, key_vector_stores: list[str], denied_by: ProxyErrorTypes | None +): + await _assert_rag_query_outcome( + _common_checks_for_rag_query( + {"vector_store_deny_by_default": deny_by_default}, + _virtual_key(object_permission_id="key-permission"), + SimpleNamespace(vector_stores=key_vector_stores), + request_body={ + "model": "gpt-4o-mini", + "input": [ + { + "type": "additional_tools", + "role": "developer", + "tools": [{"type": "file_search", "vector_store_ids": ["KBSTOREB"]}], + }, + {"type": "message", "role": "user", "content": "what is in this KB?"}, + ], + }, + ), + denied_by, + ) + + @pytest.mark.asyncio @pytest.mark.parametrize( "request_body", From ed3a2e2569badd3ce174b65ce142a0b747c66ee1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 2 Nov 2026 15:16:13 -0800 Subject: [PATCH 19/20] fix(proxy): apply the proxy-wide allowed_routes to LiteLLM models on native Codex routes --- litellm/proxy/auth/auth_utils.py | 21 ++++------ litellm/proxy/auth/route_checks.py | 13 ++++++ .../codex_passthrough_endpoints.py | 2 + .../test_codex_passthrough_endpoints.py | 41 +++++++++++++++++++ 4 files changed, 63 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 9350c9c935c..99dd395a99a 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -821,7 +821,7 @@ async def pre_db_read_auth_checks( Raises: - HTTPException if request fails initial auth checks """ - from litellm.proxy.proxy_server import general_settings, llm_router, premium_user + from litellm.proxy.proxy_server import general_settings, llm_router # Check 1. request size await check_if_request_size_is_safe(request=request) @@ -849,19 +849,12 @@ async def pre_db_read_auth_checks( ) # Check 4. Check if request route is an allowed route on the proxy - if "allowed_routes" in general_settings: - _allowed_routes: Final = general_settings["allowed_routes"] - if premium_user is not True: - verbose_proxy_logger.error( - "Trying to set allowed_routes. This is an Enterprise feature. %s", - CommonProxyErrors.not_premium_user.value, - ) - if route not in _allowed_routes: - verbose_proxy_logger.error("Route %s not in allowed_routes=%s", route, _allowed_routes) - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail=f"Access forbidden: Route {route} not allowed", - ) + if not RouteChecks.is_allowed_on_proxy(route): + verbose_proxy_logger.error("Route %s not in allowed_routes=%s", route, general_settings["allowed_routes"]) + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=f"Access forbidden: Route {route} not allowed", + ) def route_in_additonal_public_routes(current_route: str): diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index 125bfab4edf..0aa2528d46b 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -439,6 +439,19 @@ class RouteChecks: detail=f"user not allowed to access this route. Route={route} is an admin only route", ) + @staticmethod + def is_allowed_on_proxy(route: str) -> bool: + from litellm.proxy.proxy_server import general_settings, premium_user + + if "allowed_routes" not in general_settings: + return True + if premium_user is not True: + verbose_proxy_logger.error( + "Trying to set allowed_routes. This is an Enterprise feature. %s", + CommonProxyErrors.not_premium_user.value, + ) + return route in general_settings["allowed_routes"] + @staticmethod def is_unbilled_route(route: str) -> bool: resource, _, action = route.rsplit("/", 1)[-1].partition(":") diff --git a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py index 2f9f78a7ae2..191315d16d9 100644 --- a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py @@ -171,6 +171,8 @@ def codex_shared_cache() -> _SharedCache: def _may_call_litellm_responses(user_api_key_dict: UserAPIKeyAuth) -> bool: + if not RouteChecks.is_allowed_on_proxy(_LITELLM_RESPONSES_PATH): + return False try: _ = RouteChecks.should_call_route(route=_LITELLM_RESPONSES_PATH, valid_token=user_api_key_dict) if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: diff --git a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py index 3e5034b68d2..8e42edf506d 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py @@ -791,6 +791,47 @@ def test_admin_only_v1_responses_keeps_litellm_models_to_proxy_admins_on_native_ assert provider.called is through_litellm assert chatgpt.called is not through_litellm + +@pytest.mark.parametrize( + ("allowed_routes", "through_litellm"), + [ + pytest.param(["/backend-api/codex/models", "/backend-api/codex/responses"], False, id="native routes only"), + pytest.param( + ["/backend-api/codex/models", "/backend-api/codex/responses", "/v1/responses"], + True, + id="v1 responses allowed too", + ), + ], +) +def test_proxy_allowed_routes_keep_litellm_models_off_native_routes_unless_v1_responses_is_allowed( + client: TestClient, + respx_mock: respx.MockRouter, + mixed_router: Router, + restricted_key: Callable[..., str], + monkeypatch: pytest.MonkeyPatch, + allowed_routes: list[str], + through_litellm: bool, +) -> None: + monkeypatch.setattr(proxy_server, "general_settings", {"allowed_routes": allowed_routes}) + monkeypatch.setattr(proxy_server, "premium_user", True) + _account_catalog(respx_mock) + provider: Final = _open_weights_provider(respx_mock) + chatgpt: Final = _chatgpt_responses(respx_mock) + key: Final = restricted_key(["chatgpt-native", "open-weights"]) + catalog: Final = client.get( + "/backend-api/codex/models?client_version=1.0.0", headers={**_ACCOUNT_HEADERS, "x-litellm-api-key": key} + ) + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": "open-weights", "input": "Hello", "stream": True}, + headers={**_ACCOUNT_HEADERS, "x-litellm-api-key": key}, + ) + assert (catalog.status_code, response.status_code) == (200, 200), response.text + assert ("open-weights" in [model["slug"] for model in catalog.json()["models"]]) is through_litellm + assert provider.called is through_litellm + assert chatgpt.called is not through_litellm + + _USER_TURN: Final = {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Hello"}]} _ASSISTANT_TURN: Final = {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "Hi"}]} _FIREWORKS_REASONING: Final = { From 9b63bda0b539e3cb40bcfd95e83b51e0c394573b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 2 Nov 2026 15:42:18 -0800 Subject: [PATCH 20/20] fix(proxy): cap native Codex account catalog reads per LiteLLM caller before rate limits apply --- .../codex_passthrough_endpoints.py | 21 ++++++++--- .../test_codex_passthrough_endpoints.py | 36 ++++++++++++++++++- 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py index 191315d16d9..c9f609ad662 100644 --- a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py @@ -77,6 +77,9 @@ _ACCOUNT_SLUGS_UNAVAILABLE: Final = "unavailable" _ACCOUNT_SLUGS_UNAVAILABLE_TTL_SECONDS: Final = 60 _ACCOUNT_SLUGS_CACHE_PREFIX: Final = "codex_account_slugs:" _ACCOUNT_SLUGS_TARGET: Final = "codex_account_slugs" +_ACCOUNT_SLUGS_FETCHES_CACHE_PREFIX: Final = "codex_account_slugs_fetches:" +_ACCOUNT_SLUGS_FETCHES_PER_CALLER: Final = 5 +_ACCOUNT_SLUGS_FETCHES_WINDOW_SECONDS: Final = 60 _MODELS_ETAG_CACHE_PREFIX: Final = "codex_models_etag:" _MODELS_ETAG_TTL_SECONDS: Final = 300 _OPENAI_API_PROVIDERS: Final = frozenset({LlmProviders.OPENAI.value, LlmProviders.AZURE.value}) @@ -163,6 +166,8 @@ class _SharedCache(Protocol): async def async_set_cache(self, key: str, value: object, *, ttl: int) -> None: ... + async def async_increment_cache(self, key: str, value: float, *, ttl: int) -> float | None: ... + def codex_shared_cache() -> _SharedCache: from litellm.proxy.proxy_server import proxy_logging_obj @@ -229,8 +234,15 @@ async def _fetched_account_slugs(headers: Mapping[str, str], client_version: str return upstream_codex_slugs(response.content) if response.status_code == 200 else None +async def _admits_account_slugs_fetch(caller: str, cache: _SharedCache) -> bool: + fetches: Final = await cache.async_increment_cache( + f"{_ACCOUNT_SLUGS_FETCHES_CACHE_PREFIX}{caller}", 1, ttl=_ACCOUNT_SLUGS_FETCHES_WINDOW_SECONDS + ) + return fetches is not None and fetches <= _ACCOUNT_SLUGS_FETCHES_PER_CALLER + + @with_service_target(_ACCOUNT_SLUGS_TARGET) -async def _account_slugs(headers: Mapping[str, str], cache: _SharedCache) -> frozenset[str] | None: +async def _account_slugs(headers: Mapping[str, str], caller: str, cache: _SharedCache) -> frozenset[str] | None: cache_key: Final = _account_slugs_cache_key(headers) if cache_key is None: return None @@ -241,7 +253,7 @@ async def _account_slugs(headers: Mapping[str, str], cache: _SharedCache) -> fro if cached is not None: return cached client_version: Final = _codex_client_version(headers) - if client_version is None: + if client_version is None or not await _admits_account_slugs_fetch(caller, cache): return None fetched: Final = await _fetched_account_slugs(headers, client_version) await _remember_account_slugs(cache_key, fetched, cache) @@ -252,6 +264,7 @@ async def _serves_through_litellm( model: object, llm_router: "Router | None", user_api_key_dict: UserAPIKeyAuth, + caller: str, headers: Mapping[str, str], cache: _SharedCache, ) -> bool: @@ -259,7 +272,7 @@ async def _serves_through_litellm( user_api_key_dict ): return False - account_slugs: Final = await _account_slugs(headers, cache) + account_slugs: Final = await _account_slugs(headers, caller, cache) return account_slugs is not None and model not in account_slugs @@ -592,7 +605,7 @@ async def _responses( from litellm.proxy.proxy_server import llm_router body: Final = _JSON_OBJECT.validate_python(await read_request_body(request)) - if await _serves_through_litellm(body.get("model"), llm_router, user_api_key_dict, headers, cache): + if await _serves_through_litellm(body.get("model"), llm_router, user_api_key_dict, caller, headers, cache): litellm_body: Final = _litellm_body(body, llm_router, user_api_key_dict.team_id, headers) await check_tools_allowlist( request_body=dict(litellm_body), diff --git a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py index 8e42edf506d..c64540bf2c7 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py @@ -19,7 +19,10 @@ from litellm.proxy import proxy_server from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.codex_model_catalog import bundled_codex_models -from litellm.proxy.pass_through_endpoints.codex_passthrough_endpoints import codex_shared_cache +from litellm.proxy.pass_through_endpoints.codex_passthrough_endpoints import ( + _ACCOUNT_SLUGS_FETCHES_PER_CALLER, + codex_shared_cache, +) from litellm.proxy.utils import hash_token from litellm.responses.litellm_completion_transformation.reasoning_items import ( encode_thinking_blocks, @@ -981,6 +984,37 @@ def test_a_failed_account_catalog_read_with_another_login_leaves_the_accounts_ow ] +def test_rotating_chatgpt_logins_on_one_key_reads_the_account_catalog_a_bounded_number_of_times_per_key( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router, restricted_key: Callable[..., str] +) -> None: + catalog: Final = respx_mock.get(f"{_BASE}/models").respond(401) + provider: Final = _open_weights_provider(respx_mock) + chatgpt: Final = _chatgpt_responses(respx_mock) + turn: Final = {"model": "open-weights", "input": "Hello", "stream": True} + rotated: Final = tuple( + client.post( + "/backend-api/codex/responses", + json=turn, + headers={**_ACCOUNT_HEADERS, "authorization": f"Bearer test-rotated-login-{attempt}"}, + ) + for attempt in range(_ACCOUNT_SLUGS_FETCHES_PER_CALLER + 2) + ) + teammate: Final = client.post( + "/backend-api/codex/responses", + json=turn, + headers={ + **_ACCOUNT_HEADERS, + "authorization": "Bearer test-teammate-login", + "x-litellm-api-key": restricted_key(["open-weights"]), + }, + ) + assert {response.status_code for response in (*rotated, teammate)} == {200} + assert chatgpt.call_count == len(rotated) + 1 + assert not provider.called + assert [call.request.headers["authorization"] for call in catalog.calls][-1] == "Bearer test-teammate-login" + assert catalog.call_count == _ACCOUNT_SLUGS_FETCHES_PER_CALLER + 1 + + def test_account_catalog_lookup_sends_the_callers_chatgpt_login_and_codex_version( client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router ) -> None: