diff --git a/litellm/proxy/common_utils/codex_catalog_source.py b/litellm/proxy/common_utils/codex_catalog_source.py index a6dd6cb238b..c7d4723e96c 100644 --- a/litellm/proxy/common_utils/codex_catalog_source.py +++ b/litellm/proxy/common_utils/codex_catalog_source.py @@ -8,9 +8,12 @@ parallel tool calls, a generic prompt). openai/codex publishes every release's e fetched once and kept for the life of the process, since a release tag never changes. A version without a tag (before 0.120, or not released yet), a fetch that fails, a body that does not parse, a request the fetch outlasts by ``CODEX_CATALOG_RESPONSE_DEADLINE_SECONDS`` (Codex gives up on its catalog after five), -``LITELLM_OFFLINE_MODE`` and ``LITELLM_LOCAL_CODEX_CATALOG=True`` all serve the vendored pair instead, a -failed fetch again after ``CODEX_CATALOG_RETRY_SECONDS``. ``LITELLM_CODEX_CATALOG_BASE_URL`` points the -fetch at a mirror of the openai/codex raw tree. +``LITELLM_OFFLINE_MODE`` and ``LITELLM_LOCAL_CODEX_CATALOG=True`` all serve the vendored pair instead. A +failed fetch is tried again in the background after ``CODEX_CATALOG_RETRY_SECONDS`` while its vendored pair +keeps answering, and at most ``CODEX_CATALOG_CACHED_VERSIONS`` fetches start per retry window, so made-up +versions cannot turn requests into outbound calls. ``LITELLM_CODEX_CATALOG_BASE_URL`` points the fetch at a +mirror of the openai/codex raw tree, and an empty ``LITELLM_CODEX_CATALOG_OPENAI_URL`` keeps the OpenAI key +home by skipping OpenAI's catalog. """ from __future__ import annotations @@ -19,7 +22,7 @@ import asyncio import os import re import time -from collections import Counter, OrderedDict +from collections import OrderedDict from collections.abc import Awaitable, Callable from dataclasses import dataclass from functools import cache @@ -44,7 +47,7 @@ CODEX_CATALOG_OPENAI_URL: Final = "https://chatgpt.com/backend-api/codex/models" CODEX_CATALOG_BASE_URL: Final = "https://raw.githubusercontent.com/openai/codex" CODEX_CATALOG_RETRY_SECONDS: Final = 300.0 CODEX_CATALOG_FETCH_TIMEOUT_SECONDS: Final = 10.0 -CODEX_CATALOG_RESPONSE_DEADLINE_SECONDS: Final = 4.0 +CODEX_CATALOG_RESPONSE_DEADLINE_SECONDS: Final = 3.0 CODEX_CATALOG_MAX_BYTES: Final = 16 * 1024 * 1024 CODEX_CATALOG_CACHED_VERSIONS: Final = 16 _RELEASE_VERSION: Final = re.compile(r"^\d+\.\d+\.\d+(?:-[0-9A-Za-z.]+)?$") @@ -139,6 +142,7 @@ class CodexCatalogSource: self._cached_versions = cached_versions self._cache: OrderedDict[_CacheSlot, _Cached] = OrderedDict() # mutable-ok: filled as releases are asked for self._pending: dict[_CacheSlot, asyncio.Task[CodexStock]] = {} # mutable-ok: the fetches in flight + self._fetch_starts: tuple[float, ...] = () async def stock(self, client_version: str | None, *, openai_api_key: str | None = None) -> CodexStock: """The catalog for ``client_version``: OpenAI's live one when an OpenAI key is at hand, @@ -148,24 +152,37 @@ class CodexCatalogSource: return vendored_codex_stock() slot: Final = (tag, openai_api_key) cached: Final = self._cache.get(slot) - if cached is not None and (cached.retry_at is None or cached.retry_at > self._clock()): + if cached is not None: self._cache.move_to_end(slot) + if cached.retry_at is not None and cached.retry_at <= self._clock(): + self._fetching(slot, tag, client_version, openai_api_key) return cached.stock + fetch: Final = self._fetching(slot, tag, client_version, openai_api_key) + return vendored_codex_stock() if fetch is None else await self._within_deadline(tag, fetch) + + def _fetching( + self, slot: _CacheSlot, tag: str, version: str, openai_api_key: str | None + ) -> asyncio.Task[CodexStock] | None: pending: Final = self._pending.get(slot) if pending is not None: - return await self._within_deadline(tag, pending) - if len(self._pending) >= self._cached_versions: + return pending + now: Final = self._clock() + recent: Final = tuple(start for start in self._fetch_starts if start > now - self._retry_seconds) + if len(recent) >= self._cached_versions: verbose_proxy_logger.warning( - "Codex catalog %s not fetched, %d fetches already in flight, serving the %s one", + "Codex catalog %s not fetched, %d fetches started in the last %.0fs, serving the %s one", tag, - len(self._pending), + len(recent), + self._retry_seconds, vendored_codex_stock().source, ) - return vendored_codex_stock() - task: Final = asyncio.ensure_future(self._resolve(slot, tag, client_version, openai_api_key)) + self._fetch_starts = recent + return None + self._fetch_starts = (*recent, now) + task: Final = asyncio.ensure_future(self._resolve(slot, tag, version, openai_api_key)) self._pending[slot] = task task.add_done_callback(lambda _: self._pending.pop(slot, None)) - return await self._within_deadline(tag, task) + return task async def _within_deadline(self, tag: str, fetch: asyncio.Task[CodexStock]) -> CodexStock: done, _ = await asyncio.wait((fetch,), timeout=self._response_deadline) @@ -205,7 +222,7 @@ class CodexCatalogSource: return entry.stock async def _fetch(self, tag: str, version: str, openai_api_key: str | None) -> CodexStock | _FetchFailed: - if openai_api_key: + if openai_api_key and self._openai_url: fetched: Final = await self._fetch_openai(version, openai_api_key) if not isinstance(fetched, _FetchFailed): return fetched @@ -223,12 +240,11 @@ class CodexCatalogSource: catalog: Final = CodexStockCatalog.model_validate_json(body) except ValidationError as error: return _FetchFailed(reason=f"{url} body does not parse: {type(error).__name__}") - prompts: Final = Counter(model.base_instructions for model in catalog.models if model.base_instructions.strip()) - if not catalog.models or not prompts: + if not catalog.models: return _FetchFailed(reason=f"{url} catalog is empty") return CodexStock( models=MappingProxyType({model.slug: model for model in catalog.models}), - instructions=prompts.most_common(1)[0][0], + instructions=codex_base_instructions(), source=f"openai {version}", ) diff --git a/litellm/proxy/common_utils/codex_model_catalog.py b/litellm/proxy/common_utils/codex_model_catalog.py index d1f6395f941..f5a0f9ac331 100644 --- a/litellm/proxy/common_utils/codex_model_catalog.py +++ b/litellm/proxy/common_utils/codex_model_catalog.py @@ -56,6 +56,7 @@ _TierId = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)] _BODY_PREFIX: Final = '{"models":[' _BODY_SUFFIX: Final = "]}" _FIRST_CODEX_WITH_CUSTOM_EFFORTS: Final = Version("0.138.0") +_FIRST_CODEX_WITHOUT_CATALOG_BYTE_LIMIT: Final = Version("0.166.0") _EFFORTS_EVERY_CODEX_PARSES: Final = frozenset({"none", "minimal", "low", "medium", "high", "xhigh"}) _PREFERRED_DEFAULT_EFFORT: Final = "medium" _CODEX_UNSET_EFFORT: Final = "none" @@ -316,6 +317,18 @@ def _parses_custom_efforts(client_version: str | None) -> bool: return False +def codex_catalog_byte_limit(client_version: str | None) -> int | None: + """The cap the asking Codex puts on a `model_catalog_url` body: 1 MiB through 0.165, none from 0.166.0, + which dropped it (openai/codex#56525).""" + try: + uncapped: Final = ( + client_version is not None and Version(client_version) >= _FIRST_CODEX_WITHOUT_CATALOG_BYTE_LIMIT + ) + except ValueError: + return CODEX_CATALOG_BYTE_LIMIT + return None if uncapped else CODEX_CATALOG_BYTE_LIMIT + + def codex_reasoning_levels( efforts: Sequence[str], client_version: str | None ) -> tuple[tuple[CodexFallbackReasoningLevel, ...], str | None]: @@ -511,8 +524,8 @@ def codex_models_response_json( """Codex's `ModelsResponse` body for `rows`, in listing order, cut to Codex's byte limit. Only chat-capable rows are listed (Codex cannot drive an embedding or image model, and a - wildcard id is no model), their order is their `priority`. Codex drops a catalog over its - byte limit silently and keeps its bundled one, so the body holds only the entries that fit, + wildcard id is no model), their order is their `priority`. Codex before 0.166.0 drops a catalog over + its byte limit silently and keeps its bundled one, so the body holds only the entries that fit, the ones offering a service tier taken first and the rest in listing order, an entry too large for the bytes left passed over, every kept entry at its listing position; `byte_limit=None` keeps every entry. A row whose deployments set `model_info.max_input_tokens` gets the smallest @@ -586,13 +599,28 @@ def _is_openai_origin(api_base: object) -> bool: return host is not None and (host == OPENAI_API_HOST or host.endswith(f".{OPENAI_HOST_SUFFIX}")) +def _deployment_provider(params: Mapping[str, object]) -> str: + """The provider litellm routes a deployment to: its `custom_llm_provider`, else its model's prefix, else + `openai` for a bare OpenAI model name such as `gpt-5.5`.""" + import litellm # noqa: PLC0415 # litellm imports this module while it is initializing + + custom: Final = params.get("custom_llm_provider") + model: Final = params.get("model") + if isinstance(custom, str) and custom: + return custom + if not isinstance(model, str): + return "" + prefix, slash, _ = model.partition("/") + if slash: + return prefix + return "openai" if model in litellm.open_ai_chat_completion_models else "" + + def _deployment_openai_key(deployment: DeploymentTypedDict) -> str | None: params: Final = MappingProxyType[str, object](deployment["litellm_params"]) - model: Final = params.get("model") - provider: Final = params.get("custom_llm_provider") or (model.partition("/")[0] if isinstance(model, str) else "") api_key: Final = params.get("api_key") if ( - provider != "openai" + _deployment_provider(params) != "openai" or not _is_openai_origin(params.get("api_base")) or not isinstance(api_key, str) or not api_key @@ -615,7 +643,7 @@ def codex_catalog_openai_key( team_id: str | None = None, ) -> str | None: """The OpenAI API key the catalog fetch authenticates with: the first one among the listed entries' - deployments that is `openai/` with no api_base or OpenAI's own, else ``OPENAI_API_KEY`` from the + deployments that routes to OpenAI with no api_base or OpenAI's own, else ``OPENAI_API_KEY`` from the environment unless ``OPENAI_BASE_URL`` points elsewhere. A key for an OpenAI-compatible server never leaves for OpenAI.""" registry: Final = llm_router.registry() if llm_router is not None else None @@ -672,6 +700,7 @@ def codex_model_list_body( codex_catalog_rows(rows, entries, llm_router, team_id, label_entries), stock=stock.models if stock is not None else None, instructions=stock.instructions if stock is not None else None, + byte_limit=codex_catalog_byte_limit(client_version), client_version=client_version, ) if body.left_out: diff --git a/tests/unit/proxy/common_utils/test_codex_catalog_source.py b/tests/unit/proxy/common_utils/test_codex_catalog_source.py index 5502fef7e19..2a73b04122e 100644 --- a/tests/unit/proxy/common_utils/test_codex_catalog_source.py +++ b/tests/unit/proxy/common_utils/test_codex_catalog_source.py @@ -1,5 +1,6 @@ import asyncio import json +from collections import ChainMap from collections.abc import Mapping from typing import Final @@ -19,6 +20,7 @@ from litellm.proxy.common_utils.codex_catalog_source import ( from litellm.proxy.common_utils.codex_model_catalog import ( CodexCatalogRow, CodexStockModel, + codex_base_instructions, codex_models_response_json, ) @@ -78,6 +80,7 @@ def _fetching_allowed(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("LITELLM_LOCAL_CODEX_CATALOG", raising=False) monkeypatch.delenv("LITELLM_OFFLINE_MODE", raising=False) monkeypatch.delenv("LITELLM_CODEX_CATALOG_BASE_URL", raising=False) + monkeypatch.delenv("LITELLM_CODEX_CATALOG_OPENAI_URL", raising=False) def _source( @@ -121,29 +124,61 @@ def test_a_release_is_fetched_once_and_concurrent_requests_share_the_fetch() -> source: Final = _source(tree) async def burst() -> list[str]: - stocks = await asyncio.gather(*(source.stock(RELEASE) for _ in range(5))) - again = await source.stock(RELEASE) + stocks: Final = await asyncio.gather(*(source.stock(RELEASE) for _ in range(5))) + again: Final = await source.stock(RELEASE) return [stock.source for stock in (*stocks, again)] assert asyncio.run(burst()) == [f"github rust-v{RELEASE}"] * 6 assert len(tree.calls) == 2, tree.calls -def test_a_release_without_a_tag_serves_the_vendored_pair_and_is_retried_after_the_window() -> None: +def test_a_release_without_a_tag_serves_the_vendored_pair_and_is_fetched_again_behind_it_after_the_window() -> None: tree: Final = _RawTree({}) clock: Final = _Clock() source: Final = _source(tree, clock, retry_seconds=300) - first: Final = asyncio.run(source.stock(RELEASE)) - assert first.source == vendored_codex_stock().source and first.models == vendored_codex_stock().models - assert _entry(first.models, first.instructions, FRESH_SLUG)["shell_type"] == "unified_exec" - calls_after_first = len(tree.calls) - clock.now += 299 - assert asyncio.run(source.stock(RELEASE)).source == first.source - assert len(tree.calls) == calls_after_first, "a failed fetch was retried inside its window" - clock.now += 2 - tree.replies = _tree(RELEASE) - assert asyncio.run(source.stock(RELEASE)).source == f"github rust-v{RELEASE}" - assert len(tree.calls) > calls_after_first, "a failed fetch was never retried" + vendored: Final = vendored_codex_stock().source + + async def scenario() -> tuple[str, ...]: + first: Final = await source.stock(RELEASE) + assert first.models == vendored_codex_stock().models + assert _entry(first.models, first.instructions, FRESH_SLUG)["shell_type"] == "unified_exec" + calls_after_first: Final = len(tree.calls) + clock.now += 299 + inside_window: Final = await source.stock(RELEASE) + assert len(tree.calls) == calls_after_first, "a failed fetch was retried inside its window" + clock.now += 2 + tree.replies = _tree(RELEASE) + past_window: Final = await source.stock(RELEASE) + await asyncio.gather(*(asyncio.all_tasks() - {asyncio.current_task()})) + refetched: Final = await source.stock(RELEASE) + return first.source, inside_window.source, past_window.source, refetched.source + + assert asyncio.run(asyncio.wait_for(scenario(), timeout=30)) == ( + vendored, + vendored, + vendored, + f"github rust-v{RELEASE}", + ) + + +def test_fetches_past_the_budget_inside_one_window_serve_the_vendored_pair_without_a_fetch() -> None: + tree: Final = _RawTree({}) + clock: Final = _Clock() + source: Final = _source(tree, clock, retry_seconds=300, cached_versions=2) + + async def scenario() -> tuple[str, tuple[str, ...], tuple[str, ...]]: + await source.stock("0.150.0") + await source.stock("0.151.0") + over_budget: Final = await source.stock("0.152.0") + calls_inside_window: Final = tuple(tree.calls) + clock.now += 301 + await source.stock("0.152.0") + return over_budget.source, calls_inside_window, tuple(tree.calls) + + over_budget, calls_inside_window, calls = asyncio.run(asyncio.wait_for(scenario(), timeout=30)) + assert over_budget == vendored_codex_stock().source + assert not any("0.152.0" in url for url in calls_inside_window), calls_inside_window + assert any("0.152.0" in url for url in calls), calls @pytest.mark.parametrize( @@ -207,7 +242,7 @@ def test_local_and_offline_settings_keep_the_vendored_pair_without_a_fetch( def test_the_base_url_setting_points_the_fetch_at_a_mirror(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("LITELLM_CODEX_CATALOG_BASE_URL", "http://mirror.internal/codex/") - mirrored = { + mirrored: Final = { url.replace(CODEX_CATALOG_BASE_URL, "http://mirror.internal/codex"): reply for url, reply in _tree(RELEASE).items() } @@ -217,19 +252,19 @@ def test_the_base_url_setting_points_the_fetch_at_a_mirror(monkeypatch: pytest.M def test_the_cache_keeps_only_the_newest_releases() -> None: - versions = ("0.164.0", "0.165.0", "0.166.0") - tree: Final = _RawTree({url: reply for version in versions for url, reply in _tree(version).items()}) - source: Final = _source(tree, cached_versions=2) + versions: Final = ("0.164.0", "0.165.0", "0.166.0") + tree: Final = _RawTree(ChainMap(*map(_tree, versions))) + clock: Final = _Clock() + source: Final = _source(tree, clock, retry_seconds=300, cached_versions=2) async def walk() -> None: - for version in versions: + for version in (*versions, versions[-1], versions[0]): + clock.now += 301 await source.stock(version) - await source.stock(versions[-1]) - await source.stock(versions[0]) asyncio.run(walk()) - oldest_models_url = next(url for url in _tree(versions[0]) if url.endswith("models.json")) - newest_models_url = next(url for url in _tree(versions[-1]) if url.endswith("models.json")) + oldest_models_url: Final = next(url for url in _tree(versions[0]) if url.endswith("models.json")) + newest_models_url: Final = next(url for url in _tree(versions[-1]) if url.endswith("models.json")) assert tree.calls.count(oldest_models_url) == 2, tree.calls assert tree.calls.count(newest_models_url) == 1, tree.calls @@ -247,7 +282,7 @@ def test_an_openai_key_fetches_the_live_catalog_from_openai_with_that_key_and_sk tree: Final = _RawTree({**_tree(RELEASE), _openai_url(RELEASE): (200, OPENAI_CATALOG)}) stock: Final = asyncio.run(_source(tree).stock(RELEASE, openai_api_key=OPENAI_KEY)) entry: Final = _entry(stock.models, stock.instructions, FRESH_SLUG) - assert stock.source == f"openai {RELEASE}" and stock.instructions == "# OpenAI live prompt", stock.source + assert stock.source == f"openai {RELEASE}" and stock.instructions == codex_base_instructions(), stock.source assert entry["shell_type"] == "unified_exec", entry assert tree.calls == [_openai_url(RELEASE)] and tree.headers == [{"Authorization": f"Bearer {OPENAI_KEY}"}] @@ -260,6 +295,16 @@ def test_a_refused_openai_fetch_falls_back_to_the_release_tag_on_github(status: assert tree.calls[0] == _openai_url(RELEASE) and sorted(tree.calls[1:]) == sorted(_tree(RELEASE)), tree.calls +def test_an_empty_openai_url_setting_keeps_the_key_home_and_fetches_from_github( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("LITELLM_CODEX_CATALOG_OPENAI_URL", "") + tree: Final = _RawTree({**_tree(RELEASE), _openai_url(RELEASE): (200, OPENAI_CATALOG)}) + stock: Final = asyncio.run(_source(tree).stock(RELEASE, openai_api_key=OPENAI_KEY)) + assert stock.source == f"github rust-v{RELEASE}" + assert sorted(tree.calls) == sorted(_tree(RELEASE)) and tree.headers == [None, None], tree.calls + + def test_without_an_openai_key_openai_is_never_asked() -> None: tree: Final = _RawTree({**_tree(RELEASE), _openai_url(RELEASE): (200, OPENAI_CATALOG)}) stock: Final = asyncio.run(_source(tree).stock(RELEASE)) diff --git a/tests/unit/proxy/common_utils/test_codex_model_catalog.py b/tests/unit/proxy/common_utils/test_codex_model_catalog.py index 994590ea9df..aee0865a72f 100644 --- a/tests/unit/proxy/common_utils/test_codex_model_catalog.py +++ b/tests/unit/proxy/common_utils/test_codex_model_catalog.py @@ -14,6 +14,7 @@ from litellm.proxy.common_utils.codex_model_catalog import ( CodexStockModel, CodexStockUpgrade, bundled_codex_models, + codex_base_instructions, codex_catalog_openai_key, codex_catalog_rows, codex_model_list_body, @@ -673,6 +674,25 @@ def test_unlimited_body_keeps_every_entry(): assert CODEX_CATALOG_BYTE_LIMIT == 1024 * 1024 +@pytest.mark.parametrize( + ("client_version", "capped"), + [(None, True), ("0.165.0", True), ("0.166.0", False), ("0.167.0-alpha.4", False), ("not-a-version", True)], +) +def test_the_catalog_is_cut_at_the_byte_limit_only_for_a_codex_that_enforces_one( + client_version: str | None, capped: bool +) -> None: + names: Final = tuple( + f"chat-model-{index}" for index in range(CODEX_CATALOG_BYTE_LIMIT // len(codex_base_instructions()) + 2) + ) + listing: Final = tuple( + {"id": name, "object": "model", "created": 0, "owned_by": "x", "mode": "chat"} for name in names + ) + body: Final = codex_model_list_body(listing, tuple((name, name) for name in names), None, client_version=client_version) + slugs: Final = [entry["slug"] for entry in json.loads(body)["models"]] + assert (len(body.encode()) <= CODEX_CATALOG_BYTE_LIMIT) is capped, len(body.encode()) + assert (slugs == list(names)) is not capped, slugs + + def test_vendored_codex_catalog_entries_decode_on_codex(): """Every stock entry served keeps the `model_messages` Codex's `ModelsResponse` decoder needs in place of `base_instructions`, and the catalog is read from the vendored file.""" @@ -971,6 +991,21 @@ def test_the_catalog_key_is_the_listed_openai_deployments_key_before_the_environ assert codex_catalog_openai_key(router, entries[:1]) == "sk-from-env" +def test_a_bare_openai_model_name_supplies_its_key_and_a_bare_alias_of_another_provider_does_not( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + router: Final = _router_with( + {"model": "claude-sonnet-4-5", "api_key": "sk-ant-bare-alias"}, + {"model": "gpt-5.5", "api_key": "sk-bare-openai"}, + {"model": "gpt-5.5", "custom_llm_provider": "azure", "api_key": "sk-azure-bare"}, + ) + entries: Final = [(f"catalog-model-{index}", f"catalog-model-{index}") for index in range(3)] + assert codex_catalog_openai_key(router, entries) == "sk-bare-openai" + assert codex_catalog_openai_key(router, entries[:1]) is None + assert codex_catalog_openai_key(router, entries[2:]) is None + + def test_the_catalog_key_is_absent_without_an_openai_deployment_or_environment_key( monkeypatch: pytest.MonkeyPatch, ) -> None: