mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(proxy): keep Codex native turns working when the key's model listing fails
A listing callback that raises no longer turns chatgpt.com's catalog or responses answer into a 500. The catalog keeps upstream's body and ETag, and responses keep upstream's X-Models-Etag. Under drop_params, a `none` reasoning effort is now dropped only when a candidate deployment does not explicitly support it, so an alias to a model like openai/gpt-5.4 keeps it. The account slug list and its unavailable marker are keyed on the account id plus the caller's ChatGPT Authorization header, so a failed fetch with one bearer cannot turn off LiteLLM routing for another caller on the same account. The auth step checks the Codex route before importing the Codex module.
This commit is contained in:
parent
062cfada8b
commit
e6350d6394
4 changed files with 120 additions and 28 deletions
|
|
@ -52,7 +52,7 @@ For a native Codex model catalog and request format, change Codex's `base_url` t
|
|||
|
||||
On the native route, Codex's model picker shows OpenAI's catalog first and then the other chat and responses models the virtual key can use. Pick one of those and Codex's requests go through LiteLLM's `/v1/responses` with the key's usual routing, budgets, guardrails, and logging. The proxy removes the ChatGPT bearer and account header from that request, so the other provider and your callbacks never receive them. A name is routed this way when another provider serves it as an exact `model_name`, through a router `model_group_alias`, or as a team's public model name. A ChatGPT model, a name that only a wildcard deployment matches, or a name OpenAI's catalog lists for the caller's ChatGPT account keeps the passthrough, so a model OpenAI adds later still uses the subscription even when a deployment of another provider has the same name. The proxy learns the account's list from the catalog Codex fetches, or asks OpenAI with the caller's login. When it cannot get the list, for example because OpenAI is down, the request times out, or the client is not Codex, every request keeps the passthrough for the next minute, as it did before LiteLLM models were added to the picker. When the key can use many models, the picker lists the ones that fit in Codex's catalog size limit, and the proxy logs a warning with how many it left out.
|
||||
|
||||
Codex's picker sends a `none` reasoning effort for a LiteLLM model whose catalog entry lists no effort levels. With `litellm_settings: drop_params: true`, the proxy drops it and the provider applies its own default. Without `drop_params`, the effort reaches the provider unchanged.
|
||||
Codex's picker sends a `none` reasoning effort for a LiteLLM model whose catalog entry lists no effort levels. With `litellm_settings: drop_params: true`, the proxy drops it for a model that does not support `none`, and the provider applies its own default. A model whose cost map entry or `model_info` sets `supports_none_reasoning_effort: true` keeps it. Without `drop_params`, the effort reaches the provider unchanged.
|
||||
|
||||
Codex also sends its ChatGPT app tools with every turn, which on a LiteLLM model adds a few hundred thousand prompt tokens to each turn. Set `apps = false` under `[features]` in Codex's `config.toml` to turn them off.
|
||||
|
||||
|
|
|
|||
|
|
@ -351,16 +351,13 @@ def _resolve_codex_native_chatgpt_model(
|
|||
request: Request | None,
|
||||
route: str,
|
||||
) -> None:
|
||||
requested: Final = request_data.get("model")
|
||||
if route != _CODEX_NATIVE_RESPONSES_ROUTE or request is None or not isinstance(requested, str):
|
||||
return
|
||||
from litellm.proxy.pass_through_endpoints.codex_passthrough_endpoints import native_chatgpt_model
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
requested: Final = request_data.get("model")
|
||||
if (
|
||||
route != _CODEX_NATIVE_RESPONSES_ROUTE
|
||||
or request is None
|
||||
or llm_router is None
|
||||
or not isinstance(requested, str)
|
||||
):
|
||||
if llm_router is None:
|
||||
return
|
||||
target: Final = native_chatgpt_model(requested, llm_router, valid_token)
|
||||
if target is None:
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from pydantic import TypeAdapter, ValidationError
|
|||
from starlette.types import Message
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params
|
||||
from litellm.llms.chatgpt.common_utils import CHATGPT_API_BASE
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
|
|
@ -39,6 +40,7 @@ from litellm.proxy.response_api_endpoints.endpoints import responses_api
|
|||
from litellm.responses.litellm_completion_transformation.reasoning_items import is_other_provider_reasoning_item
|
||||
from litellm.types.llms.custom_http import httpxSpecialProvider
|
||||
from litellm.types.router import Deployment
|
||||
from litellm.utils import supports_none_reasoning_effort
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router
|
||||
|
|
@ -52,6 +54,7 @@ _MODELS_ETAG_HEADER: Final = "x-models-etag"
|
|||
_ENCRYPTED_CONTENT_AFFINITY_FLAG: Final = "encrypted_content_affinity_enabled"
|
||||
_LITELLM_RESPONSES_PATH: Final = "/v1/responses"
|
||||
_CODEX_NO_DEFAULT_EFFORT: Final = "none"
|
||||
_NONE_EFFORT_SUPPORT_FLAG: Final = "supports_none_reasoning_effort"
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, object])
|
||||
_INPUT_ITEMS: Final = TypeAdapter(tuple[object, ...])
|
||||
_CACHED_SLUGS: Final = TypeAdapter(frozenset[str])
|
||||
|
|
@ -160,8 +163,12 @@ def _may_call_litellm_responses(user_api_key_dict: UserAPIKeyAuth) -> bool:
|
|||
return True
|
||||
|
||||
|
||||
def _account_slugs_cache_key(account_id: str) -> str:
|
||||
return f"{_ACCOUNT_SLUGS_CACHE_PREFIX}{hashlib.sha256(account_id.encode()).hexdigest()}"
|
||||
def _account_slugs_cache_key(headers: Mapping[str, str]) -> str | None:
|
||||
account_id: Final = headers.get("chatgpt-account-id")
|
||||
if not account_id:
|
||||
return None
|
||||
caller: Final = json.dumps([account_id, headers.get("authorization", "")])
|
||||
return f"{_ACCOUNT_SLUGS_CACHE_PREFIX}{hashlib.sha256(caller.encode()).hexdigest()}"
|
||||
|
||||
|
||||
def _cached_slugs(cached: object) -> frozenset[str] | None:
|
||||
|
|
@ -173,17 +180,11 @@ def _cached_slugs(cached: object) -> frozenset[str] | None:
|
|||
return None
|
||||
|
||||
|
||||
async def _remember_account_slugs(account_id: str, slugs: frozenset[str] | None, cache: _SharedCache) -> None:
|
||||
async def _remember_account_slugs(cache_key: str, slugs: frozenset[str] | None, cache: _SharedCache) -> None:
|
||||
if slugs is None:
|
||||
await cache.async_set_cache(
|
||||
_account_slugs_cache_key(account_id),
|
||||
_ACCOUNT_SLUGS_UNAVAILABLE,
|
||||
ttl=_ACCOUNT_SLUGS_UNAVAILABLE_TTL_SECONDS,
|
||||
)
|
||||
await cache.async_set_cache(cache_key, _ACCOUNT_SLUGS_UNAVAILABLE, ttl=_ACCOUNT_SLUGS_UNAVAILABLE_TTL_SECONDS)
|
||||
return
|
||||
await cache.async_set_cache(
|
||||
_account_slugs_cache_key(account_id), tuple(sorted(slugs)), ttl=_SHARED_CACHE_TTL_SECONDS
|
||||
)
|
||||
await cache.async_set_cache(cache_key, tuple(sorted(slugs)), ttl=_SHARED_CACHE_TTL_SECONDS)
|
||||
|
||||
|
||||
def _codex_client_version(headers: Mapping[str, str]) -> str | None:
|
||||
|
|
@ -210,10 +211,10 @@ async def _fetched_account_slugs(headers: Mapping[str, str], client_version: str
|
|||
|
||||
|
||||
async def _account_slugs(headers: Mapping[str, str], cache: _SharedCache) -> frozenset[str] | None:
|
||||
account_id: Final = headers.get("chatgpt-account-id")
|
||||
if not account_id:
|
||||
cache_key: Final = _account_slugs_cache_key(headers)
|
||||
if cache_key is None:
|
||||
return None
|
||||
cached_value: Final = await cache.async_get_cache(_account_slugs_cache_key(account_id))
|
||||
cached_value: Final = await cache.async_get_cache(cache_key)
|
||||
if cached_value == _ACCOUNT_SLUGS_UNAVAILABLE:
|
||||
return None
|
||||
cached: Final = _cached_slugs(cached_value)
|
||||
|
|
@ -223,7 +224,7 @@ async def _account_slugs(headers: Mapping[str, str], cache: _SharedCache) -> fro
|
|||
if client_version is None:
|
||||
return None
|
||||
fetched: Final = await _fetched_account_slugs(headers, client_version)
|
||||
await _remember_account_slugs(account_id, fetched, cache)
|
||||
await _remember_account_slugs(cache_key, fetched, cache)
|
||||
return fetched
|
||||
|
||||
|
||||
|
|
@ -284,6 +285,20 @@ def _drops_params(body: Mapping[str, object], model: str, llm_router: "Router",
|
|||
)
|
||||
|
||||
|
||||
def _deployment_supports_no_effort(deployment: Deployment) -> bool:
|
||||
configured: Final = _json_object(deployment.model_info.model_extra).get(_NONE_EFFORT_SUPPORT_FLAG)
|
||||
if isinstance(configured, bool):
|
||||
return configured
|
||||
return supports_none_reasoning_effort(
|
||||
model=deployment.litellm_params.model, custom_llm_provider=deployment.litellm_params.custom_llm_provider
|
||||
)
|
||||
|
||||
|
||||
def _supports_no_effort(model: str, llm_router: "Router", team_id: str | None) -> bool:
|
||||
deployments: Final = _candidate_deployments(model, llm_router, team_id)
|
||||
return bool(deployments) and all(_deployment_supports_no_effort(deployment) for deployment in deployments)
|
||||
|
||||
|
||||
def _with_provider_default_reasoning(
|
||||
body: Mapping[str, object], llm_router: "Router | None", team_id: str | None
|
||||
) -> Mapping[str, object]:
|
||||
|
|
@ -298,6 +313,7 @@ def _with_provider_default_reasoning(
|
|||
or llm_router is None
|
||||
or not _drops_params(body, model, llm_router, team_id)
|
||||
or codex_entry_names_default_reasoning_level(model, llm_router, team_id)
|
||||
or _supports_no_effort(model, llm_router, team_id)
|
||||
):
|
||||
return body
|
||||
return {**body, "reasoning": {key: value for key, value in reasoning.items() if key != "effort"}}
|
||||
|
|
@ -358,7 +374,14 @@ async def _litellm_catalog_rows(
|
|||
|
||||
if llm_router is None or not _may_call_litellm_responses(user_api_key_dict):
|
||||
return ()
|
||||
rows: Final = await codex_catalog_rows_for_key(user_api_key_dict)
|
||||
try:
|
||||
rows: Final = await codex_catalog_rows_for_key(user_api_key_dict)
|
||||
except Exception as error: # noqa: BLE001 # a failed listing must never fail the ChatGPT answer it extends
|
||||
verbose_proxy_logger.warning(
|
||||
"Codex catalog keeps only ChatGPT's models: listing the key's LiteLLM models raised %s",
|
||||
type(error).__name__,
|
||||
)
|
||||
return ()
|
||||
return tuple(row for row in rows if _routes_to_litellm(row.id, llm_router, user_api_key_dict.team_id))
|
||||
|
||||
|
||||
|
|
@ -431,9 +454,9 @@ async def _catalog(
|
|||
slugs: Final = upstream_codex_slugs(upstream_body)
|
||||
if slugs is None:
|
||||
return upstream
|
||||
account_id: Final = headers.get("chatgpt-account-id")
|
||||
if account_id:
|
||||
await _remember_account_slugs(account_id, slugs, cache)
|
||||
account_slugs_key: Final = _account_slugs_cache_key(headers)
|
||||
if account_slugs_key is not None:
|
||||
await _remember_account_slugs(account_slugs_key, slugs, cache)
|
||||
rows: Final = await _litellm_catalog_rows(user_api_key_dict, llm_router)
|
||||
upstream_etag: Final = upstream.headers.get(_ETAG_HEADER)
|
||||
etag: Final = (
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
from collections.abc import Callable, Iterator
|
||||
from collections.abc import Callable, Iterator, Sequence
|
||||
from datetime import datetime
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -451,6 +451,9 @@ def test_litellm_model_on_native_route_keeps_chatgpt_credentials_out_of_hook_raw
|
|||
pytest.param("open-weights", "none", "request", None, id="none under the request's drop_params"),
|
||||
pytest.param("open-weights", "high", "global", "high", id="effort set for a model without a default effort"),
|
||||
pytest.param("coding-model", "none", "global", "none", id="none on a model whose entry names a default effort"),
|
||||
pytest.param("api-model", "none", "global", "none", id="none on a model that supports it"),
|
||||
pytest.param("glm-model", "none", "global", None, id="none on a model that does not support it"),
|
||||
pytest.param("declared-model", "none", "global", "none", id="none on a deployment whose model_info allows it"),
|
||||
],
|
||||
)
|
||||
def test_litellm_model_on_native_route_gets_the_reasoning_effort_codex_m_would_send(
|
||||
|
|
@ -474,6 +477,12 @@ def test_litellm_model_on_native_route_gets_the_reasoning_effort_codex_m_would_s
|
|||
"litellm_params": {**open_weights["litellm_params"], "drop_params": drop_params == "deployment"},
|
||||
},
|
||||
_open_weights("coding-model", upstream="gpt-5.5"),
|
||||
_open_weights("glm-model", upstream="glm-5p3"),
|
||||
{**_open_weights("declared-model"), "model_info": {"supports_none_reasoning_effort": True}},
|
||||
{
|
||||
"model_name": "api-model",
|
||||
"litellm_params": {"model": "openai/gpt-5.4", "api_base": _PROVIDER_BASE, "api_key": _PROVIDER_KEY},
|
||||
},
|
||||
]
|
||||
),
|
||||
)
|
||||
|
|
@ -504,6 +513,7 @@ def test_litellm_model_on_native_route_gets_the_reasoning_effort_codex_m_would_s
|
|||
assert native.status_code == v1.status_code == 200
|
||||
native_request, v1_request = (json.loads(call.request.content) for call in provider.calls)
|
||||
assert native_request == v1_request
|
||||
assert native_request["reasoning"].get("effort") == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -824,6 +834,35 @@ def test_a_failed_account_catalog_read_keeps_the_next_turns_on_the_subscription_
|
|||
assert catalog.call_count == 1
|
||||
|
||||
|
||||
def test_a_failed_account_catalog_read_with_another_login_leaves_the_accounts_own_list_reachable(
|
||||
client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router
|
||||
) -> None:
|
||||
own_login: Final = f"Bearer {_CHATGPT_JWT}"
|
||||
catalog: Final = respx_mock.get(f"{_BASE}/models").mock(
|
||||
side_effect=lambda request: (
|
||||
httpx.Response(200, json=_UPSTREAM_CATALOG)
|
||||
if request.headers["authorization"] == own_login
|
||||
else httpx.Response(401)
|
||||
)
|
||||
)
|
||||
provider: Final = _open_weights_provider(respx_mock)
|
||||
chatgpt: Final = _chatgpt_responses(respx_mock)
|
||||
turn: Final = {"model": "open-weights", "input": "Hello", "stream": True}
|
||||
other_login: Final = client.post(
|
||||
"/backend-api/codex/responses",
|
||||
json=turn,
|
||||
headers={**_ACCOUNT_HEADERS, "authorization": "Bearer test-someone-elses-login"},
|
||||
)
|
||||
own: Final = client.post("/backend-api/codex/responses", json=turn, headers=_ACCOUNT_HEADERS)
|
||||
assert other_login.status_code == own.status_code == 200
|
||||
assert chatgpt.call_count == 1
|
||||
assert provider.call_count == 1
|
||||
assert [call.request.headers["authorization"] for call in catalog.calls] == [
|
||||
"Bearer test-someone-elses-login",
|
||||
own_login,
|
||||
]
|
||||
|
||||
|
||||
def test_account_catalog_lookup_sends_the_callers_chatgpt_login_and_codex_version(
|
||||
client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router
|
||||
) -> None:
|
||||
|
|
@ -911,6 +950,39 @@ def test_native_models_etag_changes_with_the_keys_litellm_models_and_with_upstre
|
|||
assert baseline not in (None, '"upstream-v1"')
|
||||
|
||||
|
||||
class FailingModelListing(CustomLogger):
|
||||
async def async_filter_listed_models(
|
||||
self, user_api_key_dict: UserAPIKeyAuth, model_names: Sequence[str]
|
||||
) -> Sequence[str]:
|
||||
raise RuntimeError("model listing backend unavailable")
|
||||
|
||||
|
||||
def test_native_catalog_relays_chatgpts_answer_when_listing_the_keys_models_fails(
|
||||
client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "callbacks", [FailingModelListing()])
|
||||
respx_mock.get(f"{_BASE}/models").respond(json=_UPSTREAM_CATALOG, headers={"etag": '"upstream-v1"'})
|
||||
response: Final = client.get("/backend-api/codex/models?client_version=0.165.0", headers=_HEADERS)
|
||||
assert response.status_code == 200
|
||||
assert response.json() == _UPSTREAM_CATALOG
|
||||
assert response.headers["etag"] == '"upstream-v1"'
|
||||
|
||||
|
||||
def test_native_responses_keep_chatgpts_models_etag_when_listing_the_keys_models_fails(
|
||||
client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "callbacks", [FailingModelListing()])
|
||||
chatgpt: Final = _chatgpt_responses(respx_mock)
|
||||
response: Final = client.post(
|
||||
"/backend-api/codex/responses",
|
||||
json={"model": "gpt-unlisted", "input": "Hello", "stream": True},
|
||||
headers=_HEADERS,
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert chatgpt.called
|
||||
assert response.headers["x-models-etag"] == _UPSTREAM_MODELS_ETAG
|
||||
|
||||
|
||||
_OPENAI_BASE: Final = "https://openai-api.test/v1"
|
||||
_OPENAI_BLOB: Final = "gAAAAB-openai-reasoning-blob"
|
||||
_OPENAI_REASONING: Final = {"type": "reasoning", "id": "rs_openai", "summary": [], "encrypted_content": _OPENAI_BLOB}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue