diff --git a/litellm/proxy/common_utils/codex_model_catalog.py b/litellm/proxy/common_utils/codex_model_catalog.py index b40f1f467ba..fa9e4c36731 100644 --- a/litellm/proxy/common_utils/codex_model_catalog.py +++ b/litellm/proxy/common_utils/codex_model_catalog.py @@ -44,7 +44,6 @@ from pathlib import Path from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal, TypeAlias -from packaging.version import Version from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from litellm._logging import verbose_proxy_logger @@ -59,6 +58,7 @@ from litellm.proxy.common_utils.codex_model_translation import ( no_priced_deployment_lacks_tier, tier_price_keys, ) +from litellm.proxy.common_utils.codex_version import codex_version_at_least from litellm.types.llms.base_model import LiteLLMBaseModel if TYPE_CHECKING: @@ -77,6 +77,7 @@ CODEX_BEDROCK_MODELS_PATHS: Final[Mapping[CodexBedrockEndpoint, Path]] = Mapping } ) CODEX_CATALOG_BYTE_LIMIT: Final = 1024 * 1024 +_FIRST_CODEX_WITHOUT_CATALOG_BYTE_LIMIT: Final = "0.166.0-alpha.1" CODEX_CHAT_MODES: Final = frozenset({"chat", "responses"}) _BODY_PREFIX: Final = '{"models":[' @@ -179,9 +180,10 @@ def bundled_codex_models() -> Mapping[str, CodexStockModel]: def _with_minimal_client_version(model: CodexStockModel, built_from: str | tuple[int, ...] | None) -> CodexStockModel: source_minimum: Final = built_from if not isinstance(built_from, tuple) else ".".join(map(str, built_from)) - minimum: Final = max( - (version for version in (CODEX_BEDROCK_MODELS_MINIMAL_CLIENT_VERSION, source_minimum) if version is not None), - key=Version, + minimum: Final = ( + source_minimum + if codex_version_at_least(source_minimum, CODEX_BEDROCK_MODELS_MINIMAL_CLIENT_VERSION) + else CODEX_BEDROCK_MODELS_MINIMAL_CLIENT_VERSION ) return model.model_copy(update={"minimal_client_version": minimum}) @@ -396,10 +398,7 @@ def _readable_by(stock_model: CodexStockModel, client_version: str | None) -> bo minimum: Final = stock_model.minimal_client_version if minimum is None or client_version is None: return True - try: - return Version(client_version) >= Version(minimum if isinstance(minimum, str) else ".".join(map(str, minimum))) - except ValueError: - return False + return codex_version_at_least(client_version, minimum if isinstance(minimum, str) else ".".join(map(str, minimum))) def _tier_fields(stock_model: CodexStockModel, tiers: tuple[CodexServiceTier, ...]) -> Mapping[str, object]: @@ -601,6 +600,16 @@ class CodexCatalogBody: left_out: tuple[str, ...] +def codex_catalog_byte_limit(client_version: str | None) -> int | None: + """None, no limit, for a Codex that reads a catalog of any size since https://github.com/openai/codex/pull/56525. + An older Codex, or one sending no version, drops a catalog over 1 MiB. 0.165.0 still has the limit.""" + return ( + None + if codex_version_at_least(client_version, _FIRST_CODEX_WITHOUT_CATALOG_BYTE_LIMIT) + else CODEX_CATALOG_BYTE_LIMIT + ) + + def codex_models_response_json( rows: Sequence[CodexCatalogRow], *, @@ -659,7 +668,7 @@ def merged_codex_catalog( *, stock: Mapping[str, CodexStockModel] | None = None, instructions: str | None = None, - byte_limit: int = CODEX_CATALOG_BYTE_LIMIT, + byte_limit: int | None = CODEX_CATALOG_BYTE_LIMIT, client_version: str | None = None, ) -> str | None: try: @@ -673,7 +682,7 @@ def merged_codex_catalog( entries: Final = _entries(added, stock, instructions, first_priority, _NATIVE_ALIAS_DEFAULTS, client_version) added_models: Final = tuple(_entry_json(entry) for entry in entries) serialized: Final = tuple(_compact_json(model) for model in added_models) - budget: Final = byte_limit - len(_compact_json({**top, "models": upstream})) + budget: Final = None if byte_limit is None else byte_limit - len(_compact_json({**top, "models": upstream})) kept: Final = tuple(sorted(_kept_positions(entries, serialized, budget))) if len(kept) < len(entries): verbose_proxy_logger.warning( @@ -819,17 +828,19 @@ def codex_model_list_body( ) -> str: """The `/v1/models?client_version=...` body, logging the models Codex's byte limit left out. `stock` and `instructions` are the requesting Codex release's catalog and prompt, the bundled ones when None.""" + byte_limit: Final = codex_catalog_byte_limit(client_version) body: Final = codex_models_response_json( codex_catalog_rows(rows, entries, llm_router, team_id, label_entries), stock=stock, instructions=instructions, + byte_limit=byte_limit, client_version=client_version, ) if body.left_out: verbose_proxy_logger.warning( "Codex model catalog cut at %d bytes, left out: %s. Models offering a service tier are kept first, " "then model_list order; list the models Codex users need first in model_list", - CODEX_CATALOG_BYTE_LIMIT, + byte_limit, ", ".join(body.left_out), ) return body.json diff --git a/litellm/proxy/common_utils/codex_model_translation.py b/litellm/proxy/common_utils/codex_model_translation.py index 49e2615f9fa..7b63bb4fa42 100644 --- a/litellm/proxy/common_utils/codex_model_translation.py +++ b/litellm/proxy/common_utils/codex_model_translation.py @@ -15,10 +15,10 @@ from functools import cache from types import NoneType from typing import Annotated, Final, Literal, TypeAlias -from packaging.version import Version from pydantic import ConfigDict, Field, StringConstraints, field_serializer from litellm.litellm_core_utils.reasoning_effort_strength import nearest_declared_reasoning_effort +from litellm.proxy.common_utils.codex_version import codex_version_at_least from litellm.types.llms.base_model import LiteLLMBaseModel TierId: TypeAlias = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)] @@ -27,7 +27,7 @@ CodexInputModality: TypeAlias = Literal["text", "image"] _TEXT_ONLY: Final[tuple[CodexInputModality, ...]] = ("text",) _TEXT_AND_IMAGE: Final[tuple[CodexInputModality, ...]] = ("text", "image") TOKEN_PRICE_KEY: Final = "input_cost_per_token" -_FIRST_CODEX_WITH_CUSTOM_EFFORTS: Final = Version("0.138.0") +_FIRST_CODEX_WITH_CUSTOM_EFFORTS: Final = "0.138.0" _EFFORTS_EVERY_CODEX_PARSES: Final = frozenset({"none", "minimal", "low", "medium", "high", "xhigh"}) _PREFERRED_DEFAULT_EFFORT: Final = "medium" _CODEX_UNSET_EFFORT: Final = "none" @@ -191,13 +191,6 @@ def combined_capabilities( ) -def _parses_custom_efforts(client_version: str | None) -> bool: - try: - return client_version is not None and Version(client_version) >= _FIRST_CODEX_WITH_CUSTOM_EFFORTS - except ValueError: - return False - - def codex_reasoning_levels( efforts: Sequence[str], client_version: str | None ) -> tuple[tuple[CodexFallbackReasoningLevel, ...], str | None]: @@ -209,7 +202,7 @@ def codex_reasoning_levels( group accepts it, so a user who picks no level sends what Codex sent before, else medium, else the nearest weaker declared level, so picking no level never costs more reasoning than Codex's usual medium, else the weakest declared level, else the first one.""" - parses_any: Final = _parses_custom_efforts(client_version) + parses_any: Final = codex_version_at_least(client_version, _FIRST_CODEX_WITH_CUSTOM_EFFORTS) offered: Final = tuple(effort for effort in efforts if parses_any or effort in _EFFORTS_EVERY_CODEX_PARSES) if not offered: return (), None diff --git a/litellm/proxy/common_utils/codex_version.py b/litellm/proxy/common_utils/codex_version.py new file mode 100644 index 00000000000..dfe801d33b8 --- /dev/null +++ b/litellm/proxy/common_utils/codex_version.py @@ -0,0 +1,24 @@ +"""Compare Codex `client_version` strings, which are semver tags such as `0.166.0`, `0.166.0-alpha.5` and the +patch build `0.166.0-alpha.5.1`.""" + +from __future__ import annotations + +import re +from typing import Final + +_CODEX_VERSION: Final = re.compile(r"(\d{1,9})\.(\d{1,9})\.(\d{1,9})(?:-alpha\.(\d{1,9})(?:\.(\d{1,9}))?)?") + + +def _sort_key(version: str) -> tuple[int, int, int, bool, int, int] | None: + match: Final = _CODEX_VERSION.fullmatch(version) + if match is None: + return None + major, minor, patch, alpha, alpha_patch = match.groups() + return (int(major), int(minor), int(patch), alpha is None, int(alpha or 0), int(alpha_patch or 0)) + + +def codex_version_at_least(client_version: str | None, first: str) -> bool: + """Whether `client_version` is `first` or newer. A missing or unreadable version counts as older.""" + current: Final = None if client_version is None else _sort_key(client_version) + threshold: Final = _sort_key(first) + return current is not None and threshold is not None and current >= threshold diff --git a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py index d321f5b5861..d4abd610246 100644 --- a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py @@ -32,6 +32,7 @@ from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.codex_model_catalog import ( CodexCatalogRow, bundled_codex_models, + codex_catalog_byte_limit, codex_catalog_etag, codex_entry_names_default_reasoning_level, merged_codex_catalog, @@ -685,8 +686,13 @@ async def _catalog( etag: Final = None if upstream_etag is None else await _remembered_models_etag(caller, upstream_etag, rows, cache) if not rows: return upstream + client_version: Final = request.query_params.get("client_version") merged: Final = await asyncio.to_thread( - merged_codex_catalog, upstream_body, rows, client_version=request.query_params.get("client_version") + merged_codex_catalog, + upstream_body, + rows, + byte_limit=codex_catalog_byte_limit(client_version), + client_version=client_version, ) return _with_etag(upstream, merged if merged is not None else upstream_body, etag) diff --git a/tests/integration/compatibility/test_codex_model_catalog.py b/tests/integration/compatibility/test_codex_model_catalog.py index a513088d082..b229d7e3311 100644 --- a/tests/integration/compatibility/test_codex_model_catalog.py +++ b/tests/integration/compatibility/test_codex_model_catalog.py @@ -37,6 +37,7 @@ pytestmark: Final = pytest.mark.shares_worker_state T = TypeVar("T") CLIENT_VERSION: Final = "0.166.0" +BYTE_LIMITED_CODEX_PARAMS: Final = {"client_version": "0.165.0"} CONSISTENT_READS: Final = 6 CONVERGENCE_SECONDS: Final = 60 RELOAD_MARGIN_SECONDS: Final = 5.0 @@ -513,10 +514,10 @@ def test_wildcard_deployment_never_listed(gateway: Gateway) -> None: def test_a_tiered_model_listed_last_survives_the_byte_cut(gateway: Gateway) -> None: - """Codex 0.159.3 rejects a `model_catalog_url` body over `MAX_MODEL_CATALOG_BYTES` (1 MiB, - `codex-rs/model-provider/src/models_endpoint.rs`, read on 2026-10-03), so the proxy keeps the + """Codex up to 0.165.0 rejects a `model_catalog_url` body over `MAX_MODEL_CATALOG_BYTES` (1 MiB, + `codex-rs/model-provider/src/models_endpoint.rs`, removed by openai/codex#56525), so for it the proxy keeps the entries offering a tier first and then listing order; fallback entries carry Codex's base - prompt, so some fifty of them overrun the limit.""" + prompt, so some fifty of them overrun the limit. A newer Codex gets every entry.""" with gateway.scenario() as scenario: plain: Final = tuple(scenario.model() for _ in range(52)) tiered: Final = scenario.model(model_info={"service_tiers": ["ultrafast"]}) @@ -526,7 +527,7 @@ def test_a_tiered_model_listed_last_survives_the_byte_cut(gateway: Gateway) -> N ) assert listing.index(tiered) == len(plain), listing response: Final = eventually( - lambda: _catalog_response(gateway, key=key), + lambda: _catalog_response(gateway, key=key, params=BYTE_LIMITED_CODEX_PARAMS), lambda value: value.status_code == 200 and tiered in {entry["slug"] for entry in _entries(value)}, seconds=CONVERGENCE_SECONDS, ) @@ -538,6 +539,9 @@ def test_a_tiered_model_listed_last_survives_the_byte_cut(gateway: Gateway) -> N assert slugs[-1] == tiered and _tier_ids(entries[-1]) == ("ultrafast",), entries[-1] assert slugs[:-1] == list(plain[: len(slugs) - 1]), slugs assert [entry["priority"] for entry in entries] == [listing.index(slug) for slug in slugs], slugs + whole: Final = _catalog_response(gateway, key=key) + assert len(whole.content) > 1024 * 1024, len(whole.content) + assert [string_value(entry["slug"]) for entry in _entries(whole)] == listing def test_an_oversized_tiered_model_is_left_out_alone(gateway: Gateway) -> None: @@ -557,7 +561,7 @@ def test_an_oversized_tiered_model_is_left_out_alone(gateway: Gateway) -> None: assert listing == [plain, oversized, tiered], listing def kept() -> tuple[int, tuple[tuple[JsonValue, JsonValue, tuple[str, ...]], ...]]: - response: Final = _catalog_response(gateway, key=key) + response: Final = _catalog_response(gateway, key=key, params=BYTE_LIMITED_CODEX_PARAMS) return len(response.content), tuple( (entry["slug"], entry["priority"], _tier_ids(entry)) for entry in _entries(response) ) diff --git a/tests/unit/proxy/common_utils/test_codex_model_catalog.py b/tests/unit/proxy/common_utils/test_codex_model_catalog.py index abdce1a5663..b61bdab4016 100644 --- a/tests/unit/proxy/common_utils/test_codex_model_catalog.py +++ b/tests/unit/proxy/common_utils/test_codex_model_catalog.py @@ -31,6 +31,7 @@ from litellm.proxy.common_utils.codex_model_catalog import ( merged_codex_catalog, ) from litellm.proxy.common_utils.codex_model_translation import CodexDeploymentCapabilities, CodexServiceTier +from litellm.types.proxy.model_listing import ModelInfoResponse _PROMPT = "base prompt" _FAST = {"id": "priority", "name": "Fast", "description": "2x speed, increased usage"} @@ -193,6 +194,7 @@ def test_a_model_codex_knows_keeps_its_own_reasoning_levels(levels): [ ("0.153.0", "GPT-6.1 Terra", ["low", "max"]), ("0.166.0", "GPT-6.1 Terra", ["low", "max"]), + ("0.166.0-alpha.5.1", "GPT-6.1 Terra", ["low", "max"]), (None, "GPT-6.1 Terra", ["low", "max"]), ("0.152.9", "gpt-6.1-terra", ["low", "max"]), ("0.137.0", "gpt-6.1-terra", ["low"]), @@ -782,6 +784,43 @@ def test_unlimited_body_keeps_every_entry(): assert CODEX_CATALOG_BYTE_LIMIT == 1024 * 1024 +_LIMITED_CODEX: Final = "a codex that drops a catalog over 1 MiB gets one cut to fit" +_UNLIMITED_CODEX: Final = "a codex that reads any size gets every model" + + +@pytest.mark.parametrize( + ("client_version", "cut"), + [ + pytest.param("0.166.0-alpha.1", False, id=f"0.166.0-alpha.1, {_UNLIMITED_CODEX}"), + pytest.param("0.166.0", False, id=f"0.166.0, {_UNLIMITED_CODEX}"), + pytest.param("0.169.0-alpha.7", False, id=f"0.169.0-alpha.7, {_UNLIMITED_CODEX}"), + pytest.param("0.166.0-alpha.5.1", False, id=f"0.166.0-alpha.5.1, {_UNLIMITED_CODEX}"), + pytest.param("0.165.0", True, id=f"0.165.0, {_LIMITED_CODEX}"), + pytest.param("0.165.0-alpha.1.2", True, id=f"0.165.0-alpha.1.2, {_LIMITED_CODEX}"), + pytest.param("0.164.0", True, id=f"0.164.0, {_LIMITED_CODEX}"), + pytest.param(None, True, id=f"no version, {_LIMITED_CODEX}"), + pytest.param("not-a-version", True, id=f"unparseable version, {_LIMITED_CODEX}"), + pytest.param("9" * 5000 + ".0.0", True, id=f"absurdly long version, {_LIMITED_CODEX}"), + ], +) +def test_model_list_body_is_cut_to_1_mib_only_for_a_codex_that_drops_a_larger_catalog( + client_version: str | None, cut: bool +) -> None: + names: Final = tuple(f"model-{index}" for index in range(60)) + listing: Final[tuple[ModelInfoResponse, ...]] = tuple( + {"id": name, "object": "model", "created": 0, "owned_by": "openai", "mode": "chat"} for name in names + ) + + body: Final = codex_model_list_body( + listing, tuple((name, name) for name in names), None, client_version=client_version, instructions="x" * 20_000 + ) + slugs: Final[list[str]] = [entry["slug"] for entry in json.loads(body)["models"]] + + assert slugs == list(names[: len(slugs)]) + assert (len(slugs) < len(names)) is cut + assert (len(body.encode()) <= CODEX_CATALOG_BYTE_LIMIT) is cut + + def test_vendored_codex_catalog_entries_decode_on_codex(): """Every stock entry served keeps the `model_messages` Codex's `ModelsResponse` decoder needs in place of `base_instructions`, and the catalog is read from the vendored file.""" diff --git a/tests/unit/proxy/common_utils/test_codex_version.py b/tests/unit/proxy/common_utils/test_codex_version.py new file mode 100644 index 00000000000..8cd445ea527 --- /dev/null +++ b/tests/unit/proxy/common_utils/test_codex_version.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import pytest + +from litellm.proxy.common_utils.codex_version import codex_version_at_least + + +@pytest.mark.parametrize( + ("client_version", "first", "at_least"), + [ + ("0.166.0", "0.166.0", True), + ("0.166.1", "0.166.0", True), + ("0.167.0", "0.166.9", True), + ("1.0.0", "0.999.0", True), + ("0.165.9", "0.166.0", False), + ("0.166.0-alpha.1", "0.166.0", False), + ("0.166.0", "0.166.0-alpha.9", True), + ("0.166.0-alpha.2", "0.166.0-alpha.1", True), + ("0.166.0-alpha.10", "0.166.0-alpha.9", True), + ("0.166.0-alpha.5.1", "0.166.0-alpha.5", True), + ("0.166.0-alpha.5.1", "0.166.0-alpha.6", False), + ("0.166.0-alpha.5.2", "0.166.0-alpha.5.1", True), + ("0.165.0", "0.165.0-alpha.2", True), + ("0.165.0-alpha.1.2", "0.166.0-alpha.1", False), + ], +) +def test_codex_versions_order_as_semver(client_version: str, first: str, at_least: bool) -> None: + assert codex_version_at_least(client_version, first) is at_least + + +@pytest.mark.parametrize( + "client_version", [None, "", "dev", "0.166", "v0.166.0", "0.166.0-beta.1", "9" * 5000 + ".0.0"] +) +def test_a_missing_or_unreadable_version_counts_as_older(client_version: str | None) -> None: + assert codex_version_at_least(client_version, "0.0.0") is False + + +def test_an_unreadable_threshold_admits_no_version() -> None: + assert codex_version_at_least("0.166.0", "latest") is False diff --git a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py index 322752782fd..0352fc6520d 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py @@ -324,6 +324,33 @@ def test_native_catalog_offers_the_declared_reasoning_efforts_the_requesting_cod assert [level["effort"] for level in added["supported_reasoning_levels"]] == expected_efforts +@pytest.mark.parametrize( + ("query", "cut"), + [ + pytest.param("?client_version=0.166.0-alpha.1", False, id="first codex that reads any size gets every model"), + pytest.param("?client_version=0.166.0", False, id="newer codex gets every model"), + pytest.param("?client_version=0.165.0", True, id="last codex with the 1 MiB limit gets a cut catalog"), + pytest.param("", True, id="no version gets a cut catalog"), + ], +) +def test_native_catalog_is_cut_to_1_mib_only_for_a_codex_that_drops_a_larger_catalog( + client: TestClient, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, query: str, cut: bool +) -> None: + names: Final = tuple(f"open-weights-{index}" for index in range(60)) + monkeypatch.setattr(proxy_server, "llm_router", Router(model_list=[_open_weights(name) for name in names])) + respx_mock.get(f"{_BASE}/models").respond(json=_UPSTREAM_CATALOG) + + response: Final = client.get(f"/backend-api/codex/models{query}", headers=_HEADERS) + models: Final = response.json()["models"] + added: Final = [model["slug"] for model in models[len(_UPSTREAM_CATALOG["models"]) :]] + + assert response.status_code == 200 + assert models[: len(_UPSTREAM_CATALOG["models"])] == _UPSTREAM_CATALOG["models"] + assert added == list(names[: len(added)]) + assert (len(added) < len(names)) is cut + assert (len(response.content) <= 1024 * 1024) is cut + + @pytest.mark.parametrize( "upstream", [