From e6350d6394eedba1a3daa80cfba77dbf9da4f225 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 07:17:29 -0700 Subject: [PATCH] fix(proxy): keep Codex native turns working when the key's model listing fails A listing callback that raises no longer turns chatgpt.com's catalog or responses answer into a 500. The catalog keeps upstream's body and ETag, and responses keep upstream's X-Models-Etag. Under drop_params, a `none` reasoning effort is now dropped only when a candidate deployment does not explicitly support it, so an alias to a model like openai/gpt-5.4 keeps it. The account slug list and its unavailable marker are keyed on the account id plus the caller's ChatGPT Authorization header, so a failed fetch with one bearer cannot turn off LiteLLM routing for another caller on the same account. The auth step checks the Codex route before importing the Codex module. --- .../chatgpt_subscription_passthrough.md | 2 +- litellm/proxy/auth/user_api_key_auth.py | 11 +-- .../codex_passthrough_endpoints.py | 61 ++++++++++----- .../test_codex_passthrough_endpoints.py | 74 ++++++++++++++++++- 4 files changed, 120 insertions(+), 28 deletions(-) diff --git a/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md b/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md index 034b58e8642..538a5f8a404 100644 --- a/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md +++ b/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md @@ -52,7 +52,7 @@ For a native Codex model catalog and request format, change Codex's `base_url` t On the native route, Codex's model picker shows OpenAI's catalog first and then the other chat and responses models the virtual key can use. Pick one of those and Codex's requests go through LiteLLM's `/v1/responses` with the key's usual routing, budgets, guardrails, and logging. The proxy removes the ChatGPT bearer and account header from that request, so the other provider and your callbacks never receive them. A name is routed this way when another provider serves it as an exact `model_name`, through a router `model_group_alias`, or as a team's public model name. A ChatGPT model, a name that only a wildcard deployment matches, or a name OpenAI's catalog lists for the caller's ChatGPT account keeps the passthrough, so a model OpenAI adds later still uses the subscription even when a deployment of another provider has the same name. The proxy learns the account's list from the catalog Codex fetches, or asks OpenAI with the caller's login. When it cannot get the list, for example because OpenAI is down, the request times out, or the client is not Codex, every request keeps the passthrough for the next minute, as it did before LiteLLM models were added to the picker. When the key can use many models, the picker lists the ones that fit in Codex's catalog size limit, and the proxy logs a warning with how many it left out. -Codex's picker sends a `none` reasoning effort for a LiteLLM model whose catalog entry lists no effort levels. With `litellm_settings: drop_params: true`, the proxy drops it and the provider applies its own default. Without `drop_params`, the effort reaches the provider unchanged. +Codex's picker sends a `none` reasoning effort for a LiteLLM model whose catalog entry lists no effort levels. With `litellm_settings: drop_params: true`, the proxy drops it for a model that does not support `none`, and the provider applies its own default. A model whose cost map entry or `model_info` sets `supports_none_reasoning_effort: true` keeps it. Without `drop_params`, the effort reaches the provider unchanged. Codex also sends its ChatGPT app tools with every turn, which on a LiteLLM model adds a few hundred thousand prompt tokens to each turn. Set `apps = false` under `[features]` in Codex's `config.toml` to turn them off. diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 32d4bfe91de..ff29eca8da3 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -351,16 +351,13 @@ def _resolve_codex_native_chatgpt_model( request: Request | None, route: str, ) -> None: + requested: Final = request_data.get("model") + if route != _CODEX_NATIVE_RESPONSES_ROUTE or request is None or not isinstance(requested, str): + return from litellm.proxy.pass_through_endpoints.codex_passthrough_endpoints import native_chatgpt_model from litellm.proxy.proxy_server import llm_router - requested: Final = request_data.get("model") - if ( - route != _CODEX_NATIVE_RESPONSES_ROUTE - or request is None - or llm_router is None - or not isinstance(requested, str) - ): + if llm_router is None: return target: Final = native_chatgpt_model(requested, llm_router, valid_token) if target is None: diff --git a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py index 4963b324dc7..97fddf2b5ce 100644 --- a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py @@ -13,6 +13,7 @@ from pydantic import TypeAdapter, ValidationError from starlette.types import Message import litellm +from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params from litellm.llms.chatgpt.common_utils import CHATGPT_API_BASE from litellm.llms.custom_httpx.http_handler import ( @@ -39,6 +40,7 @@ from litellm.proxy.response_api_endpoints.endpoints import responses_api from litellm.responses.litellm_completion_transformation.reasoning_items import is_other_provider_reasoning_item from litellm.types.llms.custom_http import httpxSpecialProvider from litellm.types.router import Deployment +from litellm.utils import supports_none_reasoning_effort if TYPE_CHECKING: from litellm.router import Router @@ -52,6 +54,7 @@ _MODELS_ETAG_HEADER: Final = "x-models-etag" _ENCRYPTED_CONTENT_AFFINITY_FLAG: Final = "encrypted_content_affinity_enabled" _LITELLM_RESPONSES_PATH: Final = "/v1/responses" _CODEX_NO_DEFAULT_EFFORT: Final = "none" +_NONE_EFFORT_SUPPORT_FLAG: Final = "supports_none_reasoning_effort" _JSON_OBJECT: Final = TypeAdapter(dict[str, object]) _INPUT_ITEMS: Final = TypeAdapter(tuple[object, ...]) _CACHED_SLUGS: Final = TypeAdapter(frozenset[str]) @@ -160,8 +163,12 @@ def _may_call_litellm_responses(user_api_key_dict: UserAPIKeyAuth) -> bool: return True -def _account_slugs_cache_key(account_id: str) -> str: - return f"{_ACCOUNT_SLUGS_CACHE_PREFIX}{hashlib.sha256(account_id.encode()).hexdigest()}" +def _account_slugs_cache_key(headers: Mapping[str, str]) -> str | None: + account_id: Final = headers.get("chatgpt-account-id") + if not account_id: + return None + caller: Final = json.dumps([account_id, headers.get("authorization", "")]) + return f"{_ACCOUNT_SLUGS_CACHE_PREFIX}{hashlib.sha256(caller.encode()).hexdigest()}" def _cached_slugs(cached: object) -> frozenset[str] | None: @@ -173,17 +180,11 @@ def _cached_slugs(cached: object) -> frozenset[str] | None: return None -async def _remember_account_slugs(account_id: str, slugs: frozenset[str] | None, cache: _SharedCache) -> None: +async def _remember_account_slugs(cache_key: str, slugs: frozenset[str] | None, cache: _SharedCache) -> None: if slugs is None: - await cache.async_set_cache( - _account_slugs_cache_key(account_id), - _ACCOUNT_SLUGS_UNAVAILABLE, - ttl=_ACCOUNT_SLUGS_UNAVAILABLE_TTL_SECONDS, - ) + await cache.async_set_cache(cache_key, _ACCOUNT_SLUGS_UNAVAILABLE, ttl=_ACCOUNT_SLUGS_UNAVAILABLE_TTL_SECONDS) return - await cache.async_set_cache( - _account_slugs_cache_key(account_id), tuple(sorted(slugs)), ttl=_SHARED_CACHE_TTL_SECONDS - ) + await cache.async_set_cache(cache_key, tuple(sorted(slugs)), ttl=_SHARED_CACHE_TTL_SECONDS) def _codex_client_version(headers: Mapping[str, str]) -> str | None: @@ -210,10 +211,10 @@ async def _fetched_account_slugs(headers: Mapping[str, str], client_version: str async def _account_slugs(headers: Mapping[str, str], cache: _SharedCache) -> frozenset[str] | None: - account_id: Final = headers.get("chatgpt-account-id") - if not account_id: + cache_key: Final = _account_slugs_cache_key(headers) + if cache_key is None: return None - cached_value: Final = await cache.async_get_cache(_account_slugs_cache_key(account_id)) + cached_value: Final = await cache.async_get_cache(cache_key) if cached_value == _ACCOUNT_SLUGS_UNAVAILABLE: return None cached: Final = _cached_slugs(cached_value) @@ -223,7 +224,7 @@ async def _account_slugs(headers: Mapping[str, str], cache: _SharedCache) -> fro if client_version is None: return None fetched: Final = await _fetched_account_slugs(headers, client_version) - await _remember_account_slugs(account_id, fetched, cache) + await _remember_account_slugs(cache_key, fetched, cache) return fetched @@ -284,6 +285,20 @@ def _drops_params(body: Mapping[str, object], model: str, llm_router: "Router", ) +def _deployment_supports_no_effort(deployment: Deployment) -> bool: + configured: Final = _json_object(deployment.model_info.model_extra).get(_NONE_EFFORT_SUPPORT_FLAG) + if isinstance(configured, bool): + return configured + return supports_none_reasoning_effort( + model=deployment.litellm_params.model, custom_llm_provider=deployment.litellm_params.custom_llm_provider + ) + + +def _supports_no_effort(model: str, llm_router: "Router", team_id: str | None) -> bool: + deployments: Final = _candidate_deployments(model, llm_router, team_id) + return bool(deployments) and all(_deployment_supports_no_effort(deployment) for deployment in deployments) + + def _with_provider_default_reasoning( body: Mapping[str, object], llm_router: "Router | None", team_id: str | None ) -> Mapping[str, object]: @@ -298,6 +313,7 @@ def _with_provider_default_reasoning( or llm_router is None or not _drops_params(body, model, llm_router, team_id) or codex_entry_names_default_reasoning_level(model, llm_router, team_id) + or _supports_no_effort(model, llm_router, team_id) ): return body return {**body, "reasoning": {key: value for key, value in reasoning.items() if key != "effort"}} @@ -358,7 +374,14 @@ async def _litellm_catalog_rows( if llm_router is None or not _may_call_litellm_responses(user_api_key_dict): return () - rows: Final = await codex_catalog_rows_for_key(user_api_key_dict) + try: + rows: Final = await codex_catalog_rows_for_key(user_api_key_dict) + except Exception as error: # noqa: BLE001 # a failed listing must never fail the ChatGPT answer it extends + verbose_proxy_logger.warning( + "Codex catalog keeps only ChatGPT's models: listing the key's LiteLLM models raised %s", + type(error).__name__, + ) + return () return tuple(row for row in rows if _routes_to_litellm(row.id, llm_router, user_api_key_dict.team_id)) @@ -431,9 +454,9 @@ async def _catalog( slugs: Final = upstream_codex_slugs(upstream_body) if slugs is None: return upstream - account_id: Final = headers.get("chatgpt-account-id") - if account_id: - await _remember_account_slugs(account_id, slugs, cache) + account_slugs_key: Final = _account_slugs_cache_key(headers) + if account_slugs_key is not None: + await _remember_account_slugs(account_slugs_key, slugs, cache) rows: Final = await _litellm_catalog_rows(user_api_key_dict, llm_router) upstream_etag: Final = upstream.headers.get(_ETAG_HEADER) etag: Final = ( diff --git a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py index f9e8cca4198..02c6229b9df 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py @@ -1,7 +1,7 @@ import asyncio import base64 import json -from collections.abc import Callable, Iterator +from collections.abc import Callable, Iterator, Sequence from datetime import datetime from typing import Final @@ -451,6 +451,9 @@ def test_litellm_model_on_native_route_keeps_chatgpt_credentials_out_of_hook_raw pytest.param("open-weights", "none", "request", None, id="none under the request's drop_params"), pytest.param("open-weights", "high", "global", "high", id="effort set for a model without a default effort"), pytest.param("coding-model", "none", "global", "none", id="none on a model whose entry names a default effort"), + pytest.param("api-model", "none", "global", "none", id="none on a model that supports it"), + pytest.param("glm-model", "none", "global", None, id="none on a model that does not support it"), + pytest.param("declared-model", "none", "global", "none", id="none on a deployment whose model_info allows it"), ], ) def test_litellm_model_on_native_route_gets_the_reasoning_effort_codex_m_would_send( @@ -474,6 +477,12 @@ def test_litellm_model_on_native_route_gets_the_reasoning_effort_codex_m_would_s "litellm_params": {**open_weights["litellm_params"], "drop_params": drop_params == "deployment"}, }, _open_weights("coding-model", upstream="gpt-5.5"), + _open_weights("glm-model", upstream="glm-5p3"), + {**_open_weights("declared-model"), "model_info": {"supports_none_reasoning_effort": True}}, + { + "model_name": "api-model", + "litellm_params": {"model": "openai/gpt-5.4", "api_base": _PROVIDER_BASE, "api_key": _PROVIDER_KEY}, + }, ] ), ) @@ -504,6 +513,7 @@ def test_litellm_model_on_native_route_gets_the_reasoning_effort_codex_m_would_s assert native.status_code == v1.status_code == 200 native_request, v1_request = (json.loads(call.request.content) for call in provider.calls) assert native_request == v1_request + assert native_request["reasoning"].get("effort") == expected @pytest.mark.parametrize( @@ -824,6 +834,35 @@ def test_a_failed_account_catalog_read_keeps_the_next_turns_on_the_subscription_ assert catalog.call_count == 1 +def test_a_failed_account_catalog_read_with_another_login_leaves_the_accounts_own_list_reachable( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router +) -> None: + own_login: Final = f"Bearer {_CHATGPT_JWT}" + catalog: Final = respx_mock.get(f"{_BASE}/models").mock( + side_effect=lambda request: ( + httpx.Response(200, json=_UPSTREAM_CATALOG) + if request.headers["authorization"] == own_login + else httpx.Response(401) + ) + ) + provider: Final = _open_weights_provider(respx_mock) + chatgpt: Final = _chatgpt_responses(respx_mock) + turn: Final = {"model": "open-weights", "input": "Hello", "stream": True} + other_login: Final = client.post( + "/backend-api/codex/responses", + json=turn, + headers={**_ACCOUNT_HEADERS, "authorization": "Bearer test-someone-elses-login"}, + ) + own: Final = client.post("/backend-api/codex/responses", json=turn, headers=_ACCOUNT_HEADERS) + assert other_login.status_code == own.status_code == 200 + assert chatgpt.call_count == 1 + assert provider.call_count == 1 + assert [call.request.headers["authorization"] for call in catalog.calls] == [ + "Bearer test-someone-elses-login", + own_login, + ] + + def test_account_catalog_lookup_sends_the_callers_chatgpt_login_and_codex_version( client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router ) -> None: @@ -911,6 +950,39 @@ def test_native_models_etag_changes_with_the_keys_litellm_models_and_with_upstre assert baseline not in (None, '"upstream-v1"') +class FailingModelListing(CustomLogger): + async def async_filter_listed_models( + self, user_api_key_dict: UserAPIKeyAuth, model_names: Sequence[str] + ) -> Sequence[str]: + raise RuntimeError("model listing backend unavailable") + + +def test_native_catalog_relays_chatgpts_answer_when_listing_the_keys_models_fails( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "callbacks", [FailingModelListing()]) + respx_mock.get(f"{_BASE}/models").respond(json=_UPSTREAM_CATALOG, headers={"etag": '"upstream-v1"'}) + response: Final = client.get("/backend-api/codex/models?client_version=0.165.0", headers=_HEADERS) + assert response.status_code == 200 + assert response.json() == _UPSTREAM_CATALOG + assert response.headers["etag"] == '"upstream-v1"' + + +def test_native_responses_keep_chatgpts_models_etag_when_listing_the_keys_models_fails( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "callbacks", [FailingModelListing()]) + chatgpt: Final = _chatgpt_responses(respx_mock) + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": "gpt-unlisted", "input": "Hello", "stream": True}, + headers=_HEADERS, + ) + assert response.status_code == 200 + assert chatgpt.called + assert response.headers["x-models-etag"] == _UPSTREAM_MODELS_ETAG + + _OPENAI_BASE: Final = "https://openai-api.test/v1" _OPENAI_BLOB: Final = "gAAAAB-openai-reasoning-blob" _OPENAI_REASONING: Final = {"type": "reasoning", "id": "rs_openai", "summary": [], "encrypted_content": _OPENAI_BLOB}