fix(proxy): keep Codex native turns working when the key's model listing fails

A listing callback that raises no longer turns chatgpt.com's catalog or
responses answer into a 500. The catalog keeps upstream's body and ETag, and
responses keep upstream's X-Models-Etag.

Under drop_params, a `none` reasoning effort is now dropped only when a
candidate deployment does not explicitly support it, so an alias to a model
like openai/gpt-5.4 keeps it.

The account slug list and its unavailable marker are keyed on the account id
plus the caller's ChatGPT Authorization header, so a failed fetch with one
bearer cannot turn off LiteLLM routing for another caller on the same account.

The auth step checks the Codex route before importing the Codex module.
This commit is contained in:
mateo-berri 2026-10-29 07:17:29 -07:00
parent 062cfada8b
commit e6350d6394
4 changed files with 120 additions and 28 deletions

View file

@ -52,7 +52,7 @@ For a native Codex model catalog and request format, change Codex's `base_url` t
On the native route, Codex's model picker shows OpenAI's catalog first and then the other chat and responses models the virtual key can use. Pick one of those and Codex's requests go through LiteLLM's `/v1/responses` with the key's usual routing, budgets, guardrails, and logging. The proxy removes the ChatGPT bearer and account header from that request, so the other provider and your callbacks never receive them. A name is routed this way when another provider serves it as an exact `model_name`, through a router `model_group_alias`, or as a team's public model name. A ChatGPT model, a name that only a wildcard deployment matches, or a name OpenAI's catalog lists for the caller's ChatGPT account keeps the passthrough, so a model OpenAI adds later still uses the subscription even when a deployment of another provider has the same name. The proxy learns the account's list from the catalog Codex fetches, or asks OpenAI with the caller's login. When it cannot get the list, for example because OpenAI is down, the request times out, or the client is not Codex, every request keeps the passthrough for the next minute, as it did before LiteLLM models were added to the picker. When the key can use many models, the picker lists the ones that fit in Codex's catalog size limit, and the proxy logs a warning with how many it left out.
Codex's picker sends a `none` reasoning effort for a LiteLLM model whose catalog entry lists no effort levels. With `litellm_settings: drop_params: true`, the proxy drops it and the provider applies its own default. Without `drop_params`, the effort reaches the provider unchanged.
Codex's picker sends a `none` reasoning effort for a LiteLLM model whose catalog entry lists no effort levels. With `litellm_settings: drop_params: true`, the proxy drops it for a model that does not support `none`, and the provider applies its own default. A model whose cost map entry or `model_info` sets `supports_none_reasoning_effort: true` keeps it. Without `drop_params`, the effort reaches the provider unchanged.
Codex also sends its ChatGPT app tools with every turn, which on a LiteLLM model adds a few hundred thousand prompt tokens to each turn. Set `apps = false` under `[features]` in Codex's `config.toml` to turn them off.

View file

@ -351,16 +351,13 @@ def _resolve_codex_native_chatgpt_model(
request: Request | None,
route: str,
) -> None:
requested: Final = request_data.get("model")
if route != _CODEX_NATIVE_RESPONSES_ROUTE or request is None or not isinstance(requested, str):
return
from litellm.proxy.pass_through_endpoints.codex_passthrough_endpoints import native_chatgpt_model
from litellm.proxy.proxy_server import llm_router
requested: Final = request_data.get("model")
if (
route != _CODEX_NATIVE_RESPONSES_ROUTE
or request is None
or llm_router is None
or not isinstance(requested, str)
):
if llm_router is None:
return
target: Final = native_chatgpt_model(requested, llm_router, valid_token)
if target is None:

View file

@ -13,6 +13,7 @@ from pydantic import TypeAdapter, ValidationError
from starlette.types import Message
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params
from litellm.llms.chatgpt.common_utils import CHATGPT_API_BASE
from litellm.llms.custom_httpx.http_handler import (
@ -39,6 +40,7 @@ from litellm.proxy.response_api_endpoints.endpoints import responses_api
from litellm.responses.litellm_completion_transformation.reasoning_items import is_other_provider_reasoning_item
from litellm.types.llms.custom_http import httpxSpecialProvider
from litellm.types.router import Deployment
from litellm.utils import supports_none_reasoning_effort
if TYPE_CHECKING:
from litellm.router import Router
@ -52,6 +54,7 @@ _MODELS_ETAG_HEADER: Final = "x-models-etag"
_ENCRYPTED_CONTENT_AFFINITY_FLAG: Final = "encrypted_content_affinity_enabled"
_LITELLM_RESPONSES_PATH: Final = "/v1/responses"
_CODEX_NO_DEFAULT_EFFORT: Final = "none"
_NONE_EFFORT_SUPPORT_FLAG: Final = "supports_none_reasoning_effort"
_JSON_OBJECT: Final = TypeAdapter(dict[str, object])
_INPUT_ITEMS: Final = TypeAdapter(tuple[object, ...])
_CACHED_SLUGS: Final = TypeAdapter(frozenset[str])
@ -160,8 +163,12 @@ def _may_call_litellm_responses(user_api_key_dict: UserAPIKeyAuth) -> bool:
return True
def _account_slugs_cache_key(account_id: str) -> str:
return f"{_ACCOUNT_SLUGS_CACHE_PREFIX}{hashlib.sha256(account_id.encode()).hexdigest()}"
def _account_slugs_cache_key(headers: Mapping[str, str]) -> str | None:
account_id: Final = headers.get("chatgpt-account-id")
if not account_id:
return None
caller: Final = json.dumps([account_id, headers.get("authorization", "")])
return f"{_ACCOUNT_SLUGS_CACHE_PREFIX}{hashlib.sha256(caller.encode()).hexdigest()}"
def _cached_slugs(cached: object) -> frozenset[str] | None:
@ -173,17 +180,11 @@ def _cached_slugs(cached: object) -> frozenset[str] | None:
return None
async def _remember_account_slugs(account_id: str, slugs: frozenset[str] | None, cache: _SharedCache) -> None:
async def _remember_account_slugs(cache_key: str, slugs: frozenset[str] | None, cache: _SharedCache) -> None:
if slugs is None:
await cache.async_set_cache(
_account_slugs_cache_key(account_id),
_ACCOUNT_SLUGS_UNAVAILABLE,
ttl=_ACCOUNT_SLUGS_UNAVAILABLE_TTL_SECONDS,
)
await cache.async_set_cache(cache_key, _ACCOUNT_SLUGS_UNAVAILABLE, ttl=_ACCOUNT_SLUGS_UNAVAILABLE_TTL_SECONDS)
return
await cache.async_set_cache(
_account_slugs_cache_key(account_id), tuple(sorted(slugs)), ttl=_SHARED_CACHE_TTL_SECONDS
)
await cache.async_set_cache(cache_key, tuple(sorted(slugs)), ttl=_SHARED_CACHE_TTL_SECONDS)
def _codex_client_version(headers: Mapping[str, str]) -> str | None:
@ -210,10 +211,10 @@ async def _fetched_account_slugs(headers: Mapping[str, str], client_version: str
async def _account_slugs(headers: Mapping[str, str], cache: _SharedCache) -> frozenset[str] | None:
account_id: Final = headers.get("chatgpt-account-id")
if not account_id:
cache_key: Final = _account_slugs_cache_key(headers)
if cache_key is None:
return None
cached_value: Final = await cache.async_get_cache(_account_slugs_cache_key(account_id))
cached_value: Final = await cache.async_get_cache(cache_key)
if cached_value == _ACCOUNT_SLUGS_UNAVAILABLE:
return None
cached: Final = _cached_slugs(cached_value)
@ -223,7 +224,7 @@ async def _account_slugs(headers: Mapping[str, str], cache: _SharedCache) -> fro
if client_version is None:
return None
fetched: Final = await _fetched_account_slugs(headers, client_version)
await _remember_account_slugs(account_id, fetched, cache)
await _remember_account_slugs(cache_key, fetched, cache)
return fetched
@ -284,6 +285,20 @@ def _drops_params(body: Mapping[str, object], model: str, llm_router: "Router",
)
def _deployment_supports_no_effort(deployment: Deployment) -> bool:
configured: Final = _json_object(deployment.model_info.model_extra).get(_NONE_EFFORT_SUPPORT_FLAG)
if isinstance(configured, bool):
return configured
return supports_none_reasoning_effort(
model=deployment.litellm_params.model, custom_llm_provider=deployment.litellm_params.custom_llm_provider
)
def _supports_no_effort(model: str, llm_router: "Router", team_id: str | None) -> bool:
deployments: Final = _candidate_deployments(model, llm_router, team_id)
return bool(deployments) and all(_deployment_supports_no_effort(deployment) for deployment in deployments)
def _with_provider_default_reasoning(
body: Mapping[str, object], llm_router: "Router | None", team_id: str | None
) -> Mapping[str, object]:
@ -298,6 +313,7 @@ def _with_provider_default_reasoning(
or llm_router is None
or not _drops_params(body, model, llm_router, team_id)
or codex_entry_names_default_reasoning_level(model, llm_router, team_id)
or _supports_no_effort(model, llm_router, team_id)
):
return body
return {**body, "reasoning": {key: value for key, value in reasoning.items() if key != "effort"}}
@ -358,7 +374,14 @@ async def _litellm_catalog_rows(
if llm_router is None or not _may_call_litellm_responses(user_api_key_dict):
return ()
rows: Final = await codex_catalog_rows_for_key(user_api_key_dict)
try:
rows: Final = await codex_catalog_rows_for_key(user_api_key_dict)
except Exception as error: # noqa: BLE001 # a failed listing must never fail the ChatGPT answer it extends
verbose_proxy_logger.warning(
"Codex catalog keeps only ChatGPT's models: listing the key's LiteLLM models raised %s",
type(error).__name__,
)
return ()
return tuple(row for row in rows if _routes_to_litellm(row.id, llm_router, user_api_key_dict.team_id))
@ -431,9 +454,9 @@ async def _catalog(
slugs: Final = upstream_codex_slugs(upstream_body)
if slugs is None:
return upstream
account_id: Final = headers.get("chatgpt-account-id")
if account_id:
await _remember_account_slugs(account_id, slugs, cache)
account_slugs_key: Final = _account_slugs_cache_key(headers)
if account_slugs_key is not None:
await _remember_account_slugs(account_slugs_key, slugs, cache)
rows: Final = await _litellm_catalog_rows(user_api_key_dict, llm_router)
upstream_etag: Final = upstream.headers.get(_ETAG_HEADER)
etag: Final = (

View file

@ -1,7 +1,7 @@
import asyncio
import base64
import json
from collections.abc import Callable, Iterator
from collections.abc import Callable, Iterator, Sequence
from datetime import datetime
from typing import Final
@ -451,6 +451,9 @@ def test_litellm_model_on_native_route_keeps_chatgpt_credentials_out_of_hook_raw
pytest.param("open-weights", "none", "request", None, id="none under the request's drop_params"),
pytest.param("open-weights", "high", "global", "high", id="effort set for a model without a default effort"),
pytest.param("coding-model", "none", "global", "none", id="none on a model whose entry names a default effort"),
pytest.param("api-model", "none", "global", "none", id="none on a model that supports it"),
pytest.param("glm-model", "none", "global", None, id="none on a model that does not support it"),
pytest.param("declared-model", "none", "global", "none", id="none on a deployment whose model_info allows it"),
],
)
def test_litellm_model_on_native_route_gets_the_reasoning_effort_codex_m_would_send(
@ -474,6 +477,12 @@ def test_litellm_model_on_native_route_gets_the_reasoning_effort_codex_m_would_s
"litellm_params": {**open_weights["litellm_params"], "drop_params": drop_params == "deployment"},
},
_open_weights("coding-model", upstream="gpt-5.5"),
_open_weights("glm-model", upstream="glm-5p3"),
{**_open_weights("declared-model"), "model_info": {"supports_none_reasoning_effort": True}},
{
"model_name": "api-model",
"litellm_params": {"model": "openai/gpt-5.4", "api_base": _PROVIDER_BASE, "api_key": _PROVIDER_KEY},
},
]
),
)
@ -504,6 +513,7 @@ def test_litellm_model_on_native_route_gets_the_reasoning_effort_codex_m_would_s
assert native.status_code == v1.status_code == 200
native_request, v1_request = (json.loads(call.request.content) for call in provider.calls)
assert native_request == v1_request
assert native_request["reasoning"].get("effort") == expected
@pytest.mark.parametrize(
@ -824,6 +834,35 @@ def test_a_failed_account_catalog_read_keeps_the_next_turns_on_the_subscription_
assert catalog.call_count == 1
def test_a_failed_account_catalog_read_with_another_login_leaves_the_accounts_own_list_reachable(
client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router
) -> None:
own_login: Final = f"Bearer {_CHATGPT_JWT}"
catalog: Final = respx_mock.get(f"{_BASE}/models").mock(
side_effect=lambda request: (
httpx.Response(200, json=_UPSTREAM_CATALOG)
if request.headers["authorization"] == own_login
else httpx.Response(401)
)
)
provider: Final = _open_weights_provider(respx_mock)
chatgpt: Final = _chatgpt_responses(respx_mock)
turn: Final = {"model": "open-weights", "input": "Hello", "stream": True}
other_login: Final = client.post(
"/backend-api/codex/responses",
json=turn,
headers={**_ACCOUNT_HEADERS, "authorization": "Bearer test-someone-elses-login"},
)
own: Final = client.post("/backend-api/codex/responses", json=turn, headers=_ACCOUNT_HEADERS)
assert other_login.status_code == own.status_code == 200
assert chatgpt.call_count == 1
assert provider.call_count == 1
assert [call.request.headers["authorization"] for call in catalog.calls] == [
"Bearer test-someone-elses-login",
own_login,
]
def test_account_catalog_lookup_sends_the_callers_chatgpt_login_and_codex_version(
client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router
) -> None:
@ -911,6 +950,39 @@ def test_native_models_etag_changes_with_the_keys_litellm_models_and_with_upstre
assert baseline not in (None, '"upstream-v1"')
class FailingModelListing(CustomLogger):
async def async_filter_listed_models(
self, user_api_key_dict: UserAPIKeyAuth, model_names: Sequence[str]
) -> Sequence[str]:
raise RuntimeError("model listing backend unavailable")
def test_native_catalog_relays_chatgpts_answer_when_listing_the_keys_models_fails(
client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "callbacks", [FailingModelListing()])
respx_mock.get(f"{_BASE}/models").respond(json=_UPSTREAM_CATALOG, headers={"etag": '"upstream-v1"'})
response: Final = client.get("/backend-api/codex/models?client_version=0.165.0", headers=_HEADERS)
assert response.status_code == 200
assert response.json() == _UPSTREAM_CATALOG
assert response.headers["etag"] == '"upstream-v1"'
def test_native_responses_keep_chatgpts_models_etag_when_listing_the_keys_models_fails(
client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "callbacks", [FailingModelListing()])
chatgpt: Final = _chatgpt_responses(respx_mock)
response: Final = client.post(
"/backend-api/codex/responses",
json={"model": "gpt-unlisted", "input": "Hello", "stream": True},
headers=_HEADERS,
)
assert response.status_code == 200
assert chatgpt.called
assert response.headers["x-models-etag"] == _UPSTREAM_MODELS_ETAG
_OPENAI_BASE: Final = "https://openai-api.test/v1"
_OPENAI_BLOB: Final = "gAAAAB-openai-reasoning-blob"
_OPENAI_REASONING: Final = {"type": "reasoning", "id": "rs_openai", "summary": [], "encrypted_content": _OPENAI_BLOB}